drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/of_net.h>
   6 #include <linux/pci.h>
   7 #include <linux/bpf.h>
   8
   9 /* Local includes */
  10 #include "i40e.h"
  11 #include "i40e_diag.h"
  12 #include "i40e_xsk.h"
  13 #include <net/udp_tunnel.h>
  14 #include <net/xdp_sock.h>
  15 /* All i40e tracepoints are defined by the include below, which
  16  * must be included exactly once across the whole kernel with
  17  * CREATE_TRACE_POINTS defined
  18  */
  19 #define CREATE_TRACE_POINTS
  20 #include "i40e_trace.h"
  21
  22 const char i40e_driver_name[] = "i40e";
  23 static const char i40e_driver_string[] =
  24                         "Intel(R) Ethernet Connection XL710 Network Driver";
  25
  26 #define DRV_KERN "-k"
  27
  28 #define DRV_VERSION_MAJOR 2
  29 #define DRV_VERSION_MINOR 8
  30 #define DRV_VERSION_BUILD 20
  31 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  32              __stringify(DRV_VERSION_MINOR) "." \
  33              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  34 const char i40e_driver_version_str[] = DRV_VERSION;
  35 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
  36
  37 /* a bit of forward declarations */
  38 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  39 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  40 static int i40e_add_vsi(struct i40e_vsi *vsi);
  41 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  42 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  43 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  44 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  45 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  46 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  47 static int i40e_reset(struct i40e_pf *pf);
  48 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  49 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
  50 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
  51 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
  52 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
  53 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  54 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  55 static int i40e_get_capabilities(struct i40e_pf *pf,
  56                                  enum i40e_admin_queue_opc list_type);
  57
  58
  59 /* i40e_pci_tbl - PCI Device ID Table
  60  *
  61  * Last entry must be all 0s
  62  *
  63  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  64  *   Class, Class Mask, private data (not used) }
  65  */
  66 static const struct pci_device_id i40e_pci_tbl[] = {
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  91         /* required last entry */
  92         {0, }
  93 };
  94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  95
  96 #define I40E_MAX_VF_COUNT 128
  97 static int debug = -1;
  98 module_param(debug, uint, 0);
  99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 100
 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 103 MODULE_LICENSE("GPL v2");
 104 MODULE_VERSION(DRV_VERSION);
 105
 106 static struct workqueue_struct *i40e_wq;
 107
 108 /**
 109  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 110  * @hw:   pointer to the HW structure
 111  * @mem:  ptr to mem struct to fill out
 112  * @size: size of memory requested
 113  * @alignment: what to align the allocation to
 114  **/
 115 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 116                             u64 size, u32 alignment)
 117 {
 118         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 119
 120         mem->size = ALIGN(size, alignment);
 121         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
 122                                      GFP_KERNEL);
 123         if (!mem->va)
 124                 return -ENOMEM;
 125
 126         return 0;
 127 }
 128
 129 /**
 130  * i40e_free_dma_mem_d - OS specific memory free for shared code
 131  * @hw:   pointer to the HW structure
 132  * @mem:  ptr to mem struct to free
 133  **/
 134 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 135 {
 136         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 137
 138         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 139         mem->va = NULL;
 140         mem->pa = 0;
 141         mem->size = 0;
 142
 143         return 0;
 144 }
 145
 146 /**
 147  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 148  * @hw:   pointer to the HW structure
 149  * @mem:  ptr to mem struct to fill out
 150  * @size: size of memory requested
 151  **/
 152 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 153                              u32 size)
 154 {
 155         mem->size = size;
 156         mem->va = kzalloc(size, GFP_KERNEL);
 157
 158         if (!mem->va)
 159                 return -ENOMEM;
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * i40e_free_virt_mem_d - OS specific memory free for shared code
 166  * @hw:   pointer to the HW structure
 167  * @mem:  ptr to mem struct to free
 168  **/
 169 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 170 {
 171         /* it's ok to kfree a NULL pointer */
 172         kfree(mem->va);
 173         mem->va = NULL;
 174         mem->size = 0;
 175
 176         return 0;
 177 }
 178
 179 /**
 180  * i40e_get_lump - find a lump of free generic resource
 181  * @pf: board private structure
 182  * @pile: the pile of resource to search
 183  * @needed: the number of items needed
 184  * @id: an owner id to stick on the items assigned
 185  *
 186  * Returns the base item index of the lump, or negative for error
 187  *
 188  * The search_hint trick and lack of advanced fit-finding only work
 189  * because we're highly likely to have all the same size lump requests.
 190  * Linear search time and any fragmentation should be minimal.
 191  **/
 192 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 193                          u16 needed, u16 id)
 194 {
 195         int ret = -ENOMEM;
 196         int i, j;
 197
 198         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 199                 dev_info(&pf->pdev->dev,
 200                          "param err: pile=%s needed=%d id=0x%04x\n",
 201                          pile ? "<valid>" : "<null>", needed, id);
 202                 return -EINVAL;
 203         }
 204
 205         /* start the linear search with an imperfect hint */
 206         i = pile->search_hint;
 207         while (i < pile->num_entries) {
 208                 /* skip already allocated entries */
 209                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 210                         i++;
 211                         continue;
 212                 }
 213
 214                 /* do we have enough in this lump? */
 215                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 216                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 217                                 break;
 218                 }
 219
 220                 if (j == needed) {
 221                         /* there was enough, so assign it to the requestor */
 222                         for (j = 0; j < needed; j++)
 223                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 224                         ret = i;
 225                         pile->search_hint = i + j;
 226                         break;
 227                 }
 228
 229                 /* not enough, so skip over it and continue looking */
 230                 i += j;
 231         }
 232
 233         return ret;
 234 }
 235
 236 /**
 237  * i40e_put_lump - return a lump of generic resource
 238  * @pile: the pile of resource to search
 239  * @index: the base item index
 240  * @id: the owner id of the items assigned
 241  *
 242  * Returns the count of items in the lump
 243  **/
 244 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 245 {
 246         int valid_id = (id | I40E_PILE_VALID_BIT);
 247         int count = 0;
 248         int i;
 249
 250         if (!pile || index >= pile->num_entries)
 251                 return -EINVAL;
 252
 253         for (i = index;
 254              i < pile->num_entries && pile->list[i] == valid_id;
 255              i++) {
 256                 pile->list[i] = 0;
 257                 count++;
 258         }
 259
 260         if (count && index < pile->search_hint)
 261                 pile->search_hint = index;
 262
 263         return count;
 264 }
 265
 266 /**
 267  * i40e_find_vsi_from_id - searches for the vsi with the given id
 268  * @pf: the pf structure to search for the vsi
 269  * @id: id of the vsi it is searching for
 270  **/
 271 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 272 {
 273         int i;
 274
 275         for (i = 0; i < pf->num_alloc_vsi; i++)
 276                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 277                         return pf->vsi[i];
 278
 279         return NULL;
 280 }
 281
 282 /**
 283  * i40e_service_event_schedule - Schedule the service task to wake up
 284  * @pf: board private structure
 285  *
 286  * If not already scheduled, this puts the task into the work queue
 287  **/
 288 void i40e_service_event_schedule(struct i40e_pf *pf)
 289 {
 290         if ((!test_bit(__I40E_DOWN, pf->state) &&
 291              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
 292               test_bit(__I40E_RECOVERY_MODE, pf->state))
 293                 queue_work(i40e_wq, &pf->service_task);
 294 }
 295
 296 /**
 297  * i40e_tx_timeout - Respond to a Tx Hang
 298  * @netdev: network interface device structure
 299  *
 300  * If any port has noticed a Tx timeout, it is likely that the whole
 301  * device is munged, not just the one netdev port, so go for the full
 302  * reset.
 303  **/
 304 static void i40e_tx_timeout(struct net_device *netdev)
 305 {
 306         struct i40e_netdev_priv *np = netdev_priv(netdev);
 307         struct i40e_vsi *vsi = np->vsi;
 308         struct i40e_pf *pf = vsi->back;
 309         struct i40e_ring *tx_ring = NULL;
 310         unsigned int i, hung_queue = 0;
 311         u32 head, val;
 312
 313         pf->tx_timeout_count++;
 314
 315         /* find the stopped queue the same way the stack does */
 316         for (i = 0; i < netdev->num_tx_queues; i++) {
 317                 struct netdev_queue *q;
 318                 unsigned long trans_start;
 319
 320                 q = netdev_get_tx_queue(netdev, i);
 321                 trans_start = q->trans_start;
 322                 if (netif_xmit_stopped(q) &&
 323                     time_after(jiffies,
 324                                (trans_start + netdev->watchdog_timeo))) {
 325                         hung_queue = i;
 326                         break;
 327                 }
 328         }
 329
 330         if (i == netdev->num_tx_queues) {
 331                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 332         } else {
 333                 /* now that we have an index, find the tx_ring struct */
 334                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 335                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 336                                 if (hung_queue ==
 337                                     vsi->tx_rings[i]->queue_index) {
 338                                         tx_ring = vsi->tx_rings[i];
 339                                         break;
 340                                 }
 341                         }
 342                 }
 343         }
 344
 345         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 346                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 347         else if (time_before(jiffies,
 348                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 349                 return;   /* don't do any new action before the next timeout */
 350
 351         /* don't kick off another recovery if one is already pending */
 352         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 353                 return;
 354
 355         if (tx_ring) {
 356                 head = i40e_get_head(tx_ring);
 357                 /* Read interrupt register */
 358                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 359                         val = rd32(&pf->hw,
 360                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 361                                                 tx_ring->vsi->base_vector - 1));
 362                 else
 363                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 364
 365                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 366                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 367                             head, tx_ring->next_to_use,
 368                             readl(tx_ring->tail), val);
 369         }
 370
 371         pf->tx_timeout_last_recovery = jiffies;
 372         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 373                     pf->tx_timeout_recovery_level, hung_queue);
 374
 375         switch (pf->tx_timeout_recovery_level) {
 376         case 1:
 377                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 378                 break;
 379         case 2:
 380                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 381                 break;
 382         case 3:
 383                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 384                 break;
 385         default:
 386                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 387                 break;
 388         }
 389
 390         i40e_service_event_schedule(pf);
 391         pf->tx_timeout_recovery_level++;
 392 }
 393
 394 /**
 395  * i40e_get_vsi_stats_struct - Get System Network Statistics
 396  * @vsi: the VSI we care about
 397  *
 398  * Returns the address of the device statistics structure.
 399  * The statistics are actually updated from the service task.
 400  **/
 401 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 402 {
 403         return &vsi->net_stats;
 404 }
 405
 406 /**
 407  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 408  * @ring: Tx ring to get statistics from
 409  * @stats: statistics entry to be updated
 410  **/
 411 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 412                                             struct rtnl_link_stats64 *stats)
 413 {
 414         u64 bytes, packets;
 415         unsigned int start;
 416
 417         do {
 418                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 419                 packets = ring->stats.packets;
 420                 bytes   = ring->stats.bytes;
 421         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 422
 423         stats->tx_packets += packets;
 424         stats->tx_bytes   += bytes;
 425 }
 426
 427 /**
 428  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 429  * @netdev: network interface device structure
 430  * @stats: data structure to store statistics
 431  *
 432  * Returns the address of the device statistics structure.
 433  * The statistics are actually updated from the service task.
 434  **/
 435 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 436                                   struct rtnl_link_stats64 *stats)
 437 {
 438         struct i40e_netdev_priv *np = netdev_priv(netdev);
 439         struct i40e_vsi *vsi = np->vsi;
 440         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 441         struct i40e_ring *ring;
 442         int i;
 443
 444         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 445                 return;
 446
 447         if (!vsi->tx_rings)
 448                 return;
 449
 450         rcu_read_lock();
 451         for (i = 0; i < vsi->num_queue_pairs; i++) {
 452                 u64 bytes, packets;
 453                 unsigned int start;
 454
 455                 ring = READ_ONCE(vsi->tx_rings[i]);
 456                 if (!ring)
 457                         continue;
 458                 i40e_get_netdev_stats_struct_tx(ring, stats);
 459
 460                 if (i40e_enabled_xdp_vsi(vsi)) {
 461                         ring++;
 462                         i40e_get_netdev_stats_struct_tx(ring, stats);
 463                 }
 464
 465                 ring++;
 466                 do {
 467                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
 468                         packets = ring->stats.packets;
 469                         bytes   = ring->stats.bytes;
 470                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 471
 472                 stats->rx_packets += packets;
 473                 stats->rx_bytes   += bytes;
 474
 475         }
 476         rcu_read_unlock();
 477
 478         /* following stats updated by i40e_watchdog_subtask() */
 479         stats->multicast        = vsi_stats->multicast;
 480         stats->tx_errors        = vsi_stats->tx_errors;
 481         stats->tx_dropped       = vsi_stats->tx_dropped;
 482         stats->rx_errors        = vsi_stats->rx_errors;
 483         stats->rx_dropped       = vsi_stats->rx_dropped;
 484         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 485         stats->rx_length_errors = vsi_stats->rx_length_errors;
 486 }
 487
 488 /**
 489  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 490  * @vsi: the VSI to have its stats reset
 491  **/
 492 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 493 {
 494         struct rtnl_link_stats64 *ns;
 495         int i;
 496
 497         if (!vsi)
 498                 return;
 499
 500         ns = i40e_get_vsi_stats_struct(vsi);
 501         memset(ns, 0, sizeof(*ns));
 502         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 503         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 504         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 505         if (vsi->rx_rings && vsi->rx_rings[0]) {
 506                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 507                         memset(&vsi->rx_rings[i]->stats, 0,
 508                                sizeof(vsi->rx_rings[i]->stats));
 509                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 510                                sizeof(vsi->rx_rings[i]->rx_stats));
 511                         memset(&vsi->tx_rings[i]->stats, 0,
 512                                sizeof(vsi->tx_rings[i]->stats));
 513                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 514                                sizeof(vsi->tx_rings[i]->tx_stats));
 515                 }
 516         }
 517         vsi->stat_offsets_loaded = false;
 518 }
 519
 520 /**
 521  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 522  * @pf: the PF to be reset
 523  **/
 524 void i40e_pf_reset_stats(struct i40e_pf *pf)
 525 {
 526         int i;
 527
 528         memset(&pf->stats, 0, sizeof(pf->stats));
 529         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 530         pf->stat_offsets_loaded = false;
 531
 532         for (i = 0; i < I40E_MAX_VEB; i++) {
 533                 if (pf->veb[i]) {
 534                         memset(&pf->veb[i]->stats, 0,
 535                                sizeof(pf->veb[i]->stats));
 536                         memset(&pf->veb[i]->stats_offsets, 0,
 537                                sizeof(pf->veb[i]->stats_offsets));
 538                         memset(&pf->veb[i]->tc_stats, 0,
 539                                sizeof(pf->veb[i]->tc_stats));
 540                         memset(&pf->veb[i]->tc_stats_offsets, 0,
 541                                sizeof(pf->veb[i]->tc_stats_offsets));
 542                         pf->veb[i]->stat_offsets_loaded = false;
 543                 }
 544         }
 545         pf->hw_csum_rx_error = 0;
 546 }
 547
 548 /**
 549  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 550  * @hw: ptr to the hardware info
 551  * @hireg: the high 32 bit reg to read
 552  * @loreg: the low 32 bit reg to read
 553  * @offset_loaded: has the initial offset been loaded yet
 554  * @offset: ptr to current offset value
 555  * @stat: ptr to the stat
 556  *
 557  * Since the device stats are not reset at PFReset, they likely will not
 558  * be zeroed when the driver starts.  We'll save the first values read
 559  * and use them as offsets to be subtracted from the raw values in order
 560  * to report stats that count from zero.  In the process, we also manage
 561  * the potential roll-over.
 562  **/
 563 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 564                                bool offset_loaded, u64 *offset, u64 *stat)
 565 {
 566         u64 new_data;
 567
 568         if (hw->device_id == I40E_DEV_ID_QEMU) {
 569                 new_data = rd32(hw, loreg);
 570                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 571         } else {
 572                 new_data = rd64(hw, loreg);
 573         }
 574         if (!offset_loaded)
 575                 *offset = new_data;
 576         if (likely(new_data >= *offset))
 577                 *stat = new_data - *offset;
 578         else
 579                 *stat = (new_data + BIT_ULL(48)) - *offset;
 580         *stat &= 0xFFFFFFFFFFFFULL;
 581 }
 582
 583 /**
 584  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 585  * @hw: ptr to the hardware info
 586  * @reg: the hw reg to read
 587  * @offset_loaded: has the initial offset been loaded yet
 588  * @offset: ptr to current offset value
 589  * @stat: ptr to the stat
 590  **/
 591 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 592                                bool offset_loaded, u64 *offset, u64 *stat)
 593 {
 594         u32 new_data;
 595
 596         new_data = rd32(hw, reg);
 597         if (!offset_loaded)
 598                 *offset = new_data;
 599         if (likely(new_data >= *offset))
 600                 *stat = (u32)(new_data - *offset);
 601         else
 602                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 603 }
 604
 605 /**
 606  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 607  * @hw: ptr to the hardware info
 608  * @reg: the hw reg to read and clear
 609  * @stat: ptr to the stat
 610  **/
 611 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 612 {
 613         u32 new_data = rd32(hw, reg);
 614
 615         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 616         *stat += new_data;
 617 }
 618
 619 /**
 620  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 621  * @vsi: the VSI to be updated
 622  **/
 623 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 624 {
 625         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 626         struct i40e_pf *pf = vsi->back;
 627         struct i40e_hw *hw = &pf->hw;
 628         struct i40e_eth_stats *oes;
 629         struct i40e_eth_stats *es;     /* device's eth stats */
 630
 631         es = &vsi->eth_stats;
 632         oes = &vsi->eth_stats_offsets;
 633
 634         /* Gather up the stats that the hw collects */
 635         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 636                            vsi->stat_offsets_loaded,
 637                            &oes->tx_errors, &es->tx_errors);
 638         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 639                            vsi->stat_offsets_loaded,
 640                            &oes->rx_discards, &es->rx_discards);
 641         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 642                            vsi->stat_offsets_loaded,
 643                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 644
 645         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 646                            I40E_GLV_GORCL(stat_idx),
 647                            vsi->stat_offsets_loaded,
 648                            &oes->rx_bytes, &es->rx_bytes);
 649         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 650                            I40E_GLV_UPRCL(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->rx_unicast, &es->rx_unicast);
 653         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 654                            I40E_GLV_MPRCL(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->rx_multicast, &es->rx_multicast);
 657         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 658                            I40E_GLV_BPRCL(stat_idx),
 659                            vsi->stat_offsets_loaded,
 660                            &oes->rx_broadcast, &es->rx_broadcast);
 661
 662         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 663                            I40E_GLV_GOTCL(stat_idx),
 664                            vsi->stat_offsets_loaded,
 665                            &oes->tx_bytes, &es->tx_bytes);
 666         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 667                            I40E_GLV_UPTCL(stat_idx),
 668                            vsi->stat_offsets_loaded,
 669                            &oes->tx_unicast, &es->tx_unicast);
 670         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 671                            I40E_GLV_MPTCL(stat_idx),
 672                            vsi->stat_offsets_loaded,
 673                            &oes->tx_multicast, &es->tx_multicast);
 674         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 675                            I40E_GLV_BPTCL(stat_idx),
 676                            vsi->stat_offsets_loaded,
 677                            &oes->tx_broadcast, &es->tx_broadcast);
 678         vsi->stat_offsets_loaded = true;
 679 }
 680
 681 /**
 682  * i40e_update_veb_stats - Update Switch component statistics
 683  * @veb: the VEB being updated
 684  **/
 685 void i40e_update_veb_stats(struct i40e_veb *veb)
 686 {
 687         struct i40e_pf *pf = veb->pf;
 688         struct i40e_hw *hw = &pf->hw;
 689         struct i40e_eth_stats *oes;
 690         struct i40e_eth_stats *es;     /* device's eth stats */
 691         struct i40e_veb_tc_stats *veb_oes;
 692         struct i40e_veb_tc_stats *veb_es;
 693         int i, idx = 0;
 694
 695         idx = veb->stats_idx;
 696         es = &veb->stats;
 697         oes = &veb->stats_offsets;
 698         veb_es = &veb->tc_stats;
 699         veb_oes = &veb->tc_stats_offsets;
 700
 701         /* Gather up the stats that the hw collects */
 702         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 703                            veb->stat_offsets_loaded,
 704                            &oes->tx_discards, &es->tx_discards);
 705         if (hw->revision_id > 0)
 706                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 707                                    veb->stat_offsets_loaded,
 708                                    &oes->rx_unknown_protocol,
 709                                    &es->rx_unknown_protocol);
 710         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 711                            veb->stat_offsets_loaded,
 712                            &oes->rx_bytes, &es->rx_bytes);
 713         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 714                            veb->stat_offsets_loaded,
 715                            &oes->rx_unicast, &es->rx_unicast);
 716         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 717                            veb->stat_offsets_loaded,
 718                            &oes->rx_multicast, &es->rx_multicast);
 719         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 720                            veb->stat_offsets_loaded,
 721                            &oes->rx_broadcast, &es->rx_broadcast);
 722
 723         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 724                            veb->stat_offsets_loaded,
 725                            &oes->tx_bytes, &es->tx_bytes);
 726         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 727                            veb->stat_offsets_loaded,
 728                            &oes->tx_unicast, &es->tx_unicast);
 729         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 730                            veb->stat_offsets_loaded,
 731                            &oes->tx_multicast, &es->tx_multicast);
 732         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 733                            veb->stat_offsets_loaded,
 734                            &oes->tx_broadcast, &es->tx_broadcast);
 735         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 736                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 737                                    I40E_GLVEBTC_RPCL(i, idx),
 738                                    veb->stat_offsets_loaded,
 739                                    &veb_oes->tc_rx_packets[i],
 740                                    &veb_es->tc_rx_packets[i]);
 741                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 742                                    I40E_GLVEBTC_RBCL(i, idx),
 743                                    veb->stat_offsets_loaded,
 744                                    &veb_oes->tc_rx_bytes[i],
 745                                    &veb_es->tc_rx_bytes[i]);
 746                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 747                                    I40E_GLVEBTC_TPCL(i, idx),
 748                                    veb->stat_offsets_loaded,
 749                                    &veb_oes->tc_tx_packets[i],
 750                                    &veb_es->tc_tx_packets[i]);
 751                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 752                                    I40E_GLVEBTC_TBCL(i, idx),
 753                                    veb->stat_offsets_loaded,
 754                                    &veb_oes->tc_tx_bytes[i],
 755                                    &veb_es->tc_tx_bytes[i]);
 756         }
 757         veb->stat_offsets_loaded = true;
 758 }
 759
 760 /**
 761  * i40e_update_vsi_stats - Update the vsi statistics counters.
 762  * @vsi: the VSI to be updated
 763  *
 764  * There are a few instances where we store the same stat in a
 765  * couple of different structs.  This is partly because we have
 766  * the netdev stats that need to be filled out, which is slightly
 767  * different from the "eth_stats" defined by the chip and used in
 768  * VF communications.  We sort it out here.
 769  **/
 770 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 771 {
 772         struct i40e_pf *pf = vsi->back;
 773         struct rtnl_link_stats64 *ons;
 774         struct rtnl_link_stats64 *ns;   /* netdev stats */
 775         struct i40e_eth_stats *oes;
 776         struct i40e_eth_stats *es;     /* device's eth stats */
 777         u32 tx_restart, tx_busy;
 778         struct i40e_ring *p;
 779         u32 rx_page, rx_buf;
 780         u64 bytes, packets;
 781         unsigned int start;
 782         u64 tx_linearize;
 783         u64 tx_force_wb;
 784         u64 rx_p, rx_b;
 785         u64 tx_p, tx_b;
 786         u16 q;
 787
 788         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 789             test_bit(__I40E_CONFIG_BUSY, pf->state))
 790                 return;
 791
 792         ns = i40e_get_vsi_stats_struct(vsi);
 793         ons = &vsi->net_stats_offsets;
 794         es = &vsi->eth_stats;
 795         oes = &vsi->eth_stats_offsets;
 796
 797         /* Gather up the netdev and vsi stats that the driver collects
 798          * on the fly during packet processing
 799          */
 800         rx_b = rx_p = 0;
 801         tx_b = tx_p = 0;
 802         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 803         rx_page = 0;
 804         rx_buf = 0;
 805         rcu_read_lock();
 806         for (q = 0; q < vsi->num_queue_pairs; q++) {
 807                 /* locate Tx ring */
 808                 p = READ_ONCE(vsi->tx_rings[q]);
 809
 810                 do {
 811                         start = u64_stats_fetch_begin_irq(&p->syncp);
 812                         packets = p->stats.packets;
 813                         bytes = p->stats.bytes;
 814                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 815                 tx_b += bytes;
 816                 tx_p += packets;
 817                 tx_restart += p->tx_stats.restart_queue;
 818                 tx_busy += p->tx_stats.tx_busy;
 819                 tx_linearize += p->tx_stats.tx_linearize;
 820                 tx_force_wb += p->tx_stats.tx_force_wb;
 821
 822                 /* Rx queue is part of the same block as Tx queue */
 823                 p = &p[1];
 824                 do {
 825                         start = u64_stats_fetch_begin_irq(&p->syncp);
 826                         packets = p->stats.packets;
 827                         bytes = p->stats.bytes;
 828                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 829                 rx_b += bytes;
 830                 rx_p += packets;
 831                 rx_buf += p->rx_stats.alloc_buff_failed;
 832                 rx_page += p->rx_stats.alloc_page_failed;
 833         }
 834         rcu_read_unlock();
 835         vsi->tx_restart = tx_restart;
 836         vsi->tx_busy = tx_busy;
 837         vsi->tx_linearize = tx_linearize;
 838         vsi->tx_force_wb = tx_force_wb;
 839         vsi->rx_page_failed = rx_page;
 840         vsi->rx_buf_failed = rx_buf;
 841
 842         ns->rx_packets = rx_p;
 843         ns->rx_bytes = rx_b;
 844         ns->tx_packets = tx_p;
 845         ns->tx_bytes = tx_b;
 846
 847         /* update netdev stats from eth stats */
 848         i40e_update_eth_stats(vsi);
 849         ons->tx_errors = oes->tx_errors;
 850         ns->tx_errors = es->tx_errors;
 851         ons->multicast = oes->rx_multicast;
 852         ns->multicast = es->rx_multicast;
 853         ons->rx_dropped = oes->rx_discards;
 854         ns->rx_dropped = es->rx_discards;
 855         ons->tx_dropped = oes->tx_discards;
 856         ns->tx_dropped = es->tx_discards;
 857
 858         /* pull in a couple PF stats if this is the main vsi */
 859         if (vsi == pf->vsi[pf->lan_vsi]) {
 860                 ns->rx_crc_errors = pf->stats.crc_errors;
 861                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 862                 ns->rx_length_errors = pf->stats.rx_length_errors;
 863         }
 864 }
 865
 866 /**
 867  * i40e_update_pf_stats - Update the PF statistics counters.
 868  * @pf: the PF to be updated
 869  **/
 870 static void i40e_update_pf_stats(struct i40e_pf *pf)
 871 {
 872         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 873         struct i40e_hw_port_stats *nsd = &pf->stats;
 874         struct i40e_hw *hw = &pf->hw;
 875         u32 val;
 876         int i;
 877
 878         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 879                            I40E_GLPRT_GORCL(hw->port),
 880                            pf->stat_offsets_loaded,
 881                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 882         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 883                            I40E_GLPRT_GOTCL(hw->port),
 884                            pf->stat_offsets_loaded,
 885                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 886         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 887                            pf->stat_offsets_loaded,
 888                            &osd->eth.rx_discards,
 889                            &nsd->eth.rx_discards);
 890         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 891                            I40E_GLPRT_UPRCL(hw->port),
 892                            pf->stat_offsets_loaded,
 893                            &osd->eth.rx_unicast,
 894                            &nsd->eth.rx_unicast);
 895         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 896                            I40E_GLPRT_MPRCL(hw->port),
 897                            pf->stat_offsets_loaded,
 898                            &osd->eth.rx_multicast,
 899                            &nsd->eth.rx_multicast);
 900         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 901                            I40E_GLPRT_BPRCL(hw->port),
 902                            pf->stat_offsets_loaded,
 903                            &osd->eth.rx_broadcast,
 904                            &nsd->eth.rx_broadcast);
 905         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 906                            I40E_GLPRT_UPTCL(hw->port),
 907                            pf->stat_offsets_loaded,
 908                            &osd->eth.tx_unicast,
 909                            &nsd->eth.tx_unicast);
 910         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 911                            I40E_GLPRT_MPTCL(hw->port),
 912                            pf->stat_offsets_loaded,
 913                            &osd->eth.tx_multicast,
 914                            &nsd->eth.tx_multicast);
 915         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 916                            I40E_GLPRT_BPTCL(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->eth.tx_broadcast,
 919                            &nsd->eth.tx_broadcast);
 920
 921         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 922                            pf->stat_offsets_loaded,
 923                            &osd->tx_dropped_link_down,
 924                            &nsd->tx_dropped_link_down);
 925
 926         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 927                            pf->stat_offsets_loaded,
 928                            &osd->crc_errors, &nsd->crc_errors);
 929
 930         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->illegal_bytes, &nsd->illegal_bytes);
 933
 934         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->mac_local_faults,
 937                            &nsd->mac_local_faults);
 938         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 939                            pf->stat_offsets_loaded,
 940                            &osd->mac_remote_faults,
 941                            &nsd->mac_remote_faults);
 942
 943         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->rx_length_errors,
 946                            &nsd->rx_length_errors);
 947
 948         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 949                            pf->stat_offsets_loaded,
 950                            &osd->link_xon_rx, &nsd->link_xon_rx);
 951         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 952                            pf->stat_offsets_loaded,
 953                            &osd->link_xon_tx, &nsd->link_xon_tx);
 954         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 957         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 958                            pf->stat_offsets_loaded,
 959                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 960
 961         for (i = 0; i < 8; i++) {
 962                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 963                                    pf->stat_offsets_loaded,
 964                                    &osd->priority_xoff_rx[i],
 965                                    &nsd->priority_xoff_rx[i]);
 966                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 967                                    pf->stat_offsets_loaded,
 968                                    &osd->priority_xon_rx[i],
 969                                    &nsd->priority_xon_rx[i]);
 970                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->priority_xon_tx[i],
 973                                    &nsd->priority_xon_tx[i]);
 974                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 975                                    pf->stat_offsets_loaded,
 976                                    &osd->priority_xoff_tx[i],
 977                                    &nsd->priority_xoff_tx[i]);
 978                 i40e_stat_update32(hw,
 979                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 980                                    pf->stat_offsets_loaded,
 981                                    &osd->priority_xon_2_xoff[i],
 982                                    &nsd->priority_xon_2_xoff[i]);
 983         }
 984
 985         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 986                            I40E_GLPRT_PRC64L(hw->port),
 987                            pf->stat_offsets_loaded,
 988                            &osd->rx_size_64, &nsd->rx_size_64);
 989         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 990                            I40E_GLPRT_PRC127L(hw->port),
 991                            pf->stat_offsets_loaded,
 992                            &osd->rx_size_127, &nsd->rx_size_127);
 993         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 994                            I40E_GLPRT_PRC255L(hw->port),
 995                            pf->stat_offsets_loaded,
 996                            &osd->rx_size_255, &nsd->rx_size_255);
 997         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 998                            I40E_GLPRT_PRC511L(hw->port),
 999                            pf->stat_offsets_loaded,
1000                            &osd->rx_size_511, &nsd->rx_size_511);
1001         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1002                            I40E_GLPRT_PRC1023L(hw->port),
1003                            pf->stat_offsets_loaded,
1004                            &osd->rx_size_1023, &nsd->rx_size_1023);
1005         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1006                            I40E_GLPRT_PRC1522L(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_size_1522, &nsd->rx_size_1522);
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1010                            I40E_GLPRT_PRC9522L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_big, &nsd->rx_size_big);
1013
1014         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1015                            I40E_GLPRT_PTC64L(hw->port),
1016                            pf->stat_offsets_loaded,
1017                            &osd->tx_size_64, &nsd->tx_size_64);
1018         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1019                            I40E_GLPRT_PTC127L(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->tx_size_127, &nsd->tx_size_127);
1022         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1023                            I40E_GLPRT_PTC255L(hw->port),
1024                            pf->stat_offsets_loaded,
1025                            &osd->tx_size_255, &nsd->tx_size_255);
1026         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1027                            I40E_GLPRT_PTC511L(hw->port),
1028                            pf->stat_offsets_loaded,
1029                            &osd->tx_size_511, &nsd->tx_size_511);
1030         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1031                            I40E_GLPRT_PTC1023L(hw->port),
1032                            pf->stat_offsets_loaded,
1033                            &osd->tx_size_1023, &nsd->tx_size_1023);
1034         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1035                            I40E_GLPRT_PTC1522L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->tx_size_1522, &nsd->tx_size_1522);
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1039                            I40E_GLPRT_PTC9522L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_big, &nsd->tx_size_big);
1042
1043         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->rx_undersize, &nsd->rx_undersize);
1046         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1047                            pf->stat_offsets_loaded,
1048                            &osd->rx_fragments, &nsd->rx_fragments);
1049         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->rx_oversize, &nsd->rx_oversize);
1052         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_jabber, &nsd->rx_jabber);
1055
1056         /* FDIR stats */
1057         i40e_stat_update_and_clear32(hw,
1058                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1059                         &nsd->fd_atr_match);
1060         i40e_stat_update_and_clear32(hw,
1061                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1062                         &nsd->fd_sb_match);
1063         i40e_stat_update_and_clear32(hw,
1064                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1065                         &nsd->fd_atr_tunnel_match);
1066
1067         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1068         nsd->tx_lpi_status =
1069                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1070                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1071         nsd->rx_lpi_status =
1072                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1073                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1074         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1075                            pf->stat_offsets_loaded,
1076                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1077         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1078                            pf->stat_offsets_loaded,
1079                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1080
1081         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1082             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1083                 nsd->fd_sb_status = true;
1084         else
1085                 nsd->fd_sb_status = false;
1086
1087         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1088             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1089                 nsd->fd_atr_status = true;
1090         else
1091                 nsd->fd_atr_status = false;
1092
1093         pf->stat_offsets_loaded = true;
1094 }
1095
1096 /**
1097  * i40e_update_stats - Update the various statistics counters.
1098  * @vsi: the VSI to be updated
1099  *
1100  * Update the various stats for this VSI and its related entities.
1101  **/
1102 void i40e_update_stats(struct i40e_vsi *vsi)
1103 {
1104         struct i40e_pf *pf = vsi->back;
1105
1106         if (vsi == pf->vsi[pf->lan_vsi])
1107                 i40e_update_pf_stats(pf);
1108
1109         i40e_update_vsi_stats(vsi);
1110 }
1111
1112 /**
1113  * i40e_count_filters - counts VSI mac filters
1114  * @vsi: the VSI to be searched
1115  *
1116  * Returns count of mac filters
1117  **/
1118 int i40e_count_filters(struct i40e_vsi *vsi)
1119 {
1120         struct i40e_mac_filter *f;
1121         struct hlist_node *h;
1122         int bkt;
1123         int cnt = 0;
1124
1125         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1126                 ++cnt;
1127
1128         return cnt;
1129 }
1130
1131 /**
1132  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1133  * @vsi: the VSI to be searched
1134  * @macaddr: the MAC address
1135  * @vlan: the vlan
1136  *
1137  * Returns ptr to the filter object or NULL
1138  **/
1139 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1140                                                 const u8 *macaddr, s16 vlan)
1141 {
1142         struct i40e_mac_filter *f;
1143         u64 key;
1144
1145         if (!vsi || !macaddr)
1146                 return NULL;
1147
1148         key = i40e_addr_to_hkey(macaddr);
1149         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1150                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1151                     (vlan == f->vlan))
1152                         return f;
1153         }
1154         return NULL;
1155 }
1156
1157 /**
1158  * i40e_find_mac - Find a mac addr in the macvlan filters list
1159  * @vsi: the VSI to be searched
1160  * @macaddr: the MAC address we are searching for
1161  *
1162  * Returns the first filter with the provided MAC address or NULL if
1163  * MAC address was not found
1164  **/
1165 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1166 {
1167         struct i40e_mac_filter *f;
1168         u64 key;
1169
1170         if (!vsi || !macaddr)
1171                 return NULL;
1172
1173         key = i40e_addr_to_hkey(macaddr);
1174         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1175                 if ((ether_addr_equal(macaddr, f->macaddr)))
1176                         return f;
1177         }
1178         return NULL;
1179 }
1180
1181 /**
1182  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1183  * @vsi: the VSI to be searched
1184  *
1185  * Returns true if VSI is in vlan mode or false otherwise
1186  **/
1187 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1188 {
1189         /* If we have a PVID, always operate in VLAN mode */
1190         if (vsi->info.pvid)
1191                 return true;
1192
1193         /* We need to operate in VLAN mode whenever we have any filters with
1194          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1195          * time, incurring search cost repeatedly. However, we can notice two
1196          * things:
1197          *
1198          * 1) the only place where we can gain a VLAN filter is in
1199          *    i40e_add_filter.
1200          *
1201          * 2) the only place where filters are actually removed is in
1202          *    i40e_sync_filters_subtask.
1203          *
1204          * Thus, we can simply use a boolean value, has_vlan_filters which we
1205          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1206          * we have to perform the full search after deleting filters in
1207          * i40e_sync_filters_subtask, but we already have to search
1208          * filters here and can perform the check at the same time. This
1209          * results in avoiding embedding a loop for VLAN mode inside another
1210          * loop over all the filters, and should maintain correctness as noted
1211          * above.
1212          */
1213         return vsi->has_vlan_filter;
1214 }
1215
1216 /**
1217  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1218  * @vsi: the VSI to configure
1219  * @tmp_add_list: list of filters ready to be added
1220  * @tmp_del_list: list of filters ready to be deleted
1221  * @vlan_filters: the number of active VLAN filters
1222  *
1223  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1224  * behave as expected. If we have any active VLAN filters remaining or about
1225  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1226  * so that they only match against untagged traffic. If we no longer have any
1227  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1228  * so that they match against both tagged and untagged traffic. In this way,
1229  * we ensure that we correctly receive the desired traffic. This ensures that
1230  * when we have an active VLAN we will receive only untagged traffic and
1231  * traffic matching active VLANs. If we have no active VLANs then we will
1232  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1233  *
1234  * Finally, in a similar fashion, this function also corrects filters when
1235  * there is an active PVID assigned to this VSI.
1236  *
1237  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1238  *
1239  * This function is only expected to be called from within
1240  * i40e_sync_vsi_filters.
1241  *
1242  * NOTE: This function expects to be called while under the
1243  * mac_filter_hash_lock
1244  */
1245 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1246                                          struct hlist_head *tmp_add_list,
1247                                          struct hlist_head *tmp_del_list,
1248                                          int vlan_filters)
1249 {
1250         s16 pvid = le16_to_cpu(vsi->info.pvid);
1251         struct i40e_mac_filter *f, *add_head;
1252         struct i40e_new_mac_filter *new;
1253         struct hlist_node *h;
1254         int bkt, new_vlan;
1255
1256         /* To determine if a particular filter needs to be replaced we
1257          * have the three following conditions:
1258          *
1259          * a) if we have a PVID assigned, then all filters which are
1260          *    not marked as VLAN=PVID must be replaced with filters that
1261          *    are.
1262          * b) otherwise, if we have any active VLANS, all filters
1263          *    which are marked as VLAN=-1 must be replaced with
1264          *    filters marked as VLAN=0
1265          * c) finally, if we do not have any active VLANS, all filters
1266          *    which are marked as VLAN=0 must be replaced with filters
1267          *    marked as VLAN=-1
1268          */
1269
1270         /* Update the filters about to be added in place */
1271         hlist_for_each_entry(new, tmp_add_list, hlist) {
1272                 if (pvid && new->f->vlan != pvid)
1273                         new->f->vlan = pvid;
1274                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1275                         new->f->vlan = 0;
1276                 else if (!vlan_filters && new->f->vlan == 0)
1277                         new->f->vlan = I40E_VLAN_ANY;
1278         }
1279
1280         /* Update the remaining active filters */
1281         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1282                 /* Combine the checks for whether a filter needs to be changed
1283                  * and then determine the new VLAN inside the if block, in
1284                  * order to avoid duplicating code for adding the new filter
1285                  * then deleting the old filter.
1286                  */
1287                 if ((pvid && f->vlan != pvid) ||
1288                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1289                     (!vlan_filters && f->vlan == 0)) {
1290                         /* Determine the new vlan we will be adding */
1291                         if (pvid)
1292                                 new_vlan = pvid;
1293                         else if (vlan_filters)
1294                                 new_vlan = 0;
1295                         else
1296                                 new_vlan = I40E_VLAN_ANY;
1297
1298                         /* Create the new filter */
1299                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1300                         if (!add_head)
1301                                 return -ENOMEM;
1302
1303                         /* Create a temporary i40e_new_mac_filter */
1304                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1305                         if (!new)
1306                                 return -ENOMEM;
1307
1308                         new->f = add_head;
1309                         new->state = add_head->state;
1310
1311                         /* Add the new filter to the tmp list */
1312                         hlist_add_head(&new->hlist, tmp_add_list);
1313
1314                         /* Put the original filter into the delete list */
1315                         f->state = I40E_FILTER_REMOVE;
1316                         hash_del(&f->hlist);
1317                         hlist_add_head(&f->hlist, tmp_del_list);
1318                 }
1319         }
1320
1321         vsi->has_vlan_filter = !!vlan_filters;
1322
1323         return 0;
1324 }
1325
1326 /**
1327  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1328  * @vsi: the PF Main VSI - inappropriate for any other VSI
1329  * @macaddr: the MAC address
1330  *
1331  * Remove whatever filter the firmware set up so the driver can manage
1332  * its own filtering intelligently.
1333  **/
1334 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1335 {
1336         struct i40e_aqc_remove_macvlan_element_data element;
1337         struct i40e_pf *pf = vsi->back;
1338
1339         /* Only appropriate for the PF main VSI */
1340         if (vsi->type != I40E_VSI_MAIN)
1341                 return;
1342
1343         memset(&element, 0, sizeof(element));
1344         ether_addr_copy(element.mac_addr, macaddr);
1345         element.vlan_tag = 0;
1346         /* Ignore error returns, some firmware does it this way... */
1347         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1348         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1349
1350         memset(&element, 0, sizeof(element));
1351         ether_addr_copy(element.mac_addr, macaddr);
1352         element.vlan_tag = 0;
1353         /* ...and some firmware does it this way. */
1354         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1355                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1356         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1357 }
1358
1359 /**
1360  * i40e_add_filter - Add a mac/vlan filter to the VSI
1361  * @vsi: the VSI to be searched
1362  * @macaddr: the MAC address
1363  * @vlan: the vlan
1364  *
1365  * Returns ptr to the filter object or NULL when no memory available.
1366  *
1367  * NOTE: This function is expected to be called with mac_filter_hash_lock
1368  * being held.
1369  **/
1370 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1371                                         const u8 *macaddr, s16 vlan)
1372 {
1373         struct i40e_mac_filter *f;
1374         u64 key;
1375
1376         if (!vsi || !macaddr)
1377                 return NULL;
1378
1379         f = i40e_find_filter(vsi, macaddr, vlan);
1380         if (!f) {
1381                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1382                 if (!f)
1383                         return NULL;
1384
1385                 /* Update the boolean indicating if we need to function in
1386                  * VLAN mode.
1387                  */
1388                 if (vlan >= 0)
1389                         vsi->has_vlan_filter = true;
1390
1391                 ether_addr_copy(f->macaddr, macaddr);
1392                 f->vlan = vlan;
1393                 f->state = I40E_FILTER_NEW;
1394                 INIT_HLIST_NODE(&f->hlist);
1395
1396                 key = i40e_addr_to_hkey(macaddr);
1397                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1398
1399                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1400                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1401         }
1402
1403         /* If we're asked to add a filter that has been marked for removal, it
1404          * is safe to simply restore it to active state. __i40e_del_filter
1405          * will have simply deleted any filters which were previously marked
1406          * NEW or FAILED, so if it is currently marked REMOVE it must have
1407          * previously been ACTIVE. Since we haven't yet run the sync filters
1408          * task, just restore this filter to the ACTIVE state so that the
1409          * sync task leaves it in place
1410          */
1411         if (f->state == I40E_FILTER_REMOVE)
1412                 f->state = I40E_FILTER_ACTIVE;
1413
1414         return f;
1415 }
1416
1417 /**
1418  * __i40e_del_filter - Remove a specific filter from the VSI
1419  * @vsi: VSI to remove from
1420  * @f: the filter to remove from the list
1421  *
1422  * This function should be called instead of i40e_del_filter only if you know
1423  * the exact filter you will remove already, such as via i40e_find_filter or
1424  * i40e_find_mac.
1425  *
1426  * NOTE: This function is expected to be called with mac_filter_hash_lock
1427  * being held.
1428  * ANOTHER NOTE: This function MUST be called from within the context of
1429  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1430  * instead of list_for_each_entry().
1431  **/
1432 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1433 {
1434         if (!f)
1435                 return;
1436
1437         /* If the filter was never added to firmware then we can just delete it
1438          * directly and we don't want to set the status to remove or else an
1439          * admin queue command will unnecessarily fire.
1440          */
1441         if ((f->state == I40E_FILTER_FAILED) ||
1442             (f->state == I40E_FILTER_NEW)) {
1443                 hash_del(&f->hlist);
1444                 kfree(f);
1445         } else {
1446                 f->state = I40E_FILTER_REMOVE;
1447         }
1448
1449         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1450         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1451 }
1452
1453 /**
1454  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1455  * @vsi: the VSI to be searched
1456  * @macaddr: the MAC address
1457  * @vlan: the VLAN
1458  *
1459  * NOTE: This function is expected to be called with mac_filter_hash_lock
1460  * being held.
1461  * ANOTHER NOTE: This function MUST be called from within the context of
1462  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1463  * instead of list_for_each_entry().
1464  **/
1465 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1466 {
1467         struct i40e_mac_filter *f;
1468
1469         if (!vsi || !macaddr)
1470                 return;
1471
1472         f = i40e_find_filter(vsi, macaddr, vlan);
1473         __i40e_del_filter(vsi, f);
1474 }
1475
1476 /**
1477  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1478  * @vsi: the VSI to be searched
1479  * @macaddr: the mac address to be filtered
1480  *
1481  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1482  * go through all the macvlan filters and add a macvlan filter for each
1483  * unique vlan that already exists. If a PVID has been assigned, instead only
1484  * add the macaddr to that VLAN.
1485  *
1486  * Returns last filter added on success, else NULL
1487  **/
1488 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1489                                             const u8 *macaddr)
1490 {
1491         struct i40e_mac_filter *f, *add = NULL;
1492         struct hlist_node *h;
1493         int bkt;
1494
1495         if (vsi->info.pvid)
1496                 return i40e_add_filter(vsi, macaddr,
1497                                        le16_to_cpu(vsi->info.pvid));
1498
1499         if (!i40e_is_vsi_in_vlan(vsi))
1500                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1501
1502         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1503                 if (f->state == I40E_FILTER_REMOVE)
1504                         continue;
1505                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1506                 if (!add)
1507                         return NULL;
1508         }
1509
1510         return add;
1511 }
1512
1513 /**
1514  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1515  * @vsi: the VSI to be searched
1516  * @macaddr: the mac address to be removed
1517  *
1518  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1519  * associated with.
1520  *
1521  * Returns 0 for success, or error
1522  **/
1523 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1524 {
1525         struct i40e_mac_filter *f;
1526         struct hlist_node *h;
1527         bool found = false;
1528         int bkt;
1529
1530         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1531         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1532                 if (ether_addr_equal(macaddr, f->macaddr)) {
1533                         __i40e_del_filter(vsi, f);
1534                         found = true;
1535                 }
1536         }
1537
1538         if (found)
1539                 return 0;
1540         else
1541                 return -ENOENT;
1542 }
1543
1544 /**
1545  * i40e_set_mac - NDO callback to set mac address
1546  * @netdev: network interface device structure
1547  * @p: pointer to an address structure
1548  *
1549  * Returns 0 on success, negative on failure
1550  **/
1551 static int i40e_set_mac(struct net_device *netdev, void *p)
1552 {
1553         struct i40e_netdev_priv *np = netdev_priv(netdev);
1554         struct i40e_vsi *vsi = np->vsi;
1555         struct i40e_pf *pf = vsi->back;
1556         struct i40e_hw *hw = &pf->hw;
1557         struct sockaddr *addr = p;
1558
1559         if (!is_valid_ether_addr(addr->sa_data))
1560                 return -EADDRNOTAVAIL;
1561
1562         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1563                 netdev_info(netdev, "already using mac address %pM\n",
1564                             addr->sa_data);
1565                 return 0;
1566         }
1567
1568         if (test_bit(__I40E_DOWN, pf->state) ||
1569             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1570                 return -EADDRNOTAVAIL;
1571
1572         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1573                 netdev_info(netdev, "returning to hw mac address %pM\n",
1574                             hw->mac.addr);
1575         else
1576                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1577
1578         /* Copy the address first, so that we avoid a possible race with
1579          * .set_rx_mode().
1580          * - Remove old address from MAC filter
1581          * - Copy new address
1582          * - Add new address to MAC filter
1583          */
1584         spin_lock_bh(&vsi->mac_filter_hash_lock);
1585         i40e_del_mac_filter(vsi, netdev->dev_addr);
1586         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1587         i40e_add_mac_filter(vsi, netdev->dev_addr);
1588         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1589
1590         if (vsi->type == I40E_VSI_MAIN) {
1591                 i40e_status ret;
1592
1593                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1594                                                 addr->sa_data, NULL);
1595                 if (ret)
1596                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1597                                     i40e_stat_str(hw, ret),
1598                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1599         }
1600
1601         /* schedule our worker thread which will take care of
1602          * applying the new filter changes
1603          */
1604         i40e_service_event_schedule(pf);
1605         return 0;
1606 }
1607
1608 /**
1609  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1610  * @vsi: vsi structure
1611  * @seed: RSS hash seed
1612  **/
1613 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1614                               u8 *lut, u16 lut_size)
1615 {
1616         struct i40e_pf *pf = vsi->back;
1617         struct i40e_hw *hw = &pf->hw;
1618         int ret = 0;
1619
1620         if (seed) {
1621                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1622                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1623                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1624                 if (ret) {
1625                         dev_info(&pf->pdev->dev,
1626                                  "Cannot set RSS key, err %s aq_err %s\n",
1627                                  i40e_stat_str(hw, ret),
1628                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1629                         return ret;
1630                 }
1631         }
1632         if (lut) {
1633                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
1634
1635                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1636                 if (ret) {
1637                         dev_info(&pf->pdev->dev,
1638                                  "Cannot set RSS lut, err %s aq_err %s\n",
1639                                  i40e_stat_str(hw, ret),
1640                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1641                         return ret;
1642                 }
1643         }
1644         return ret;
1645 }
1646
1647 /**
1648  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1649  * @vsi: VSI structure
1650  **/
1651 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1652 {
1653         struct i40e_pf *pf = vsi->back;
1654         u8 seed[I40E_HKEY_ARRAY_SIZE];
1655         u8 *lut;
1656         int ret;
1657
1658         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1659                 return 0;
1660         if (!vsi->rss_size)
1661                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1662                                       vsi->num_queue_pairs);
1663         if (!vsi->rss_size)
1664                 return -EINVAL;
1665         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1666         if (!lut)
1667                 return -ENOMEM;
1668
1669         /* Use the user configured hash keys and lookup table if there is one,
1670          * otherwise use default
1671          */
1672         if (vsi->rss_lut_user)
1673                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1674         else
1675                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1676         if (vsi->rss_hkey_user)
1677                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1678         else
1679                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1680         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1681         kfree(lut);
1682         return ret;
1683 }
1684
1685 /**
1686  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1687  * @vsi: the VSI being configured,
1688  * @ctxt: VSI context structure
1689  * @enabled_tc: number of traffic classes to enable
1690  *
1691  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1692  **/
1693 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1694                                            struct i40e_vsi_context *ctxt,
1695                                            u8 enabled_tc)
1696 {
1697         u16 qcount = 0, max_qcount, qmap, sections = 0;
1698         int i, override_q, pow, num_qps, ret;
1699         u8 netdev_tc = 0, offset = 0;
1700
1701         if (vsi->type != I40E_VSI_MAIN)
1702                 return -EINVAL;
1703         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1704         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1705         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1706         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1707         num_qps = vsi->mqprio_qopt.qopt.count[0];
1708
1709         /* find the next higher power-of-2 of num queue pairs */
1710         pow = ilog2(num_qps);
1711         if (!is_power_of_2(num_qps))
1712                 pow++;
1713         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1714                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1715
1716         /* Setup queue offset/count for all TCs for given VSI */
1717         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1718         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1719                 /* See if the given TC is enabled for the given VSI */
1720                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1721                         offset = vsi->mqprio_qopt.qopt.offset[i];
1722                         qcount = vsi->mqprio_qopt.qopt.count[i];
1723                         if (qcount > max_qcount)
1724                                 max_qcount = qcount;
1725                         vsi->tc_config.tc_info[i].qoffset = offset;
1726                         vsi->tc_config.tc_info[i].qcount = qcount;
1727                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1728                 } else {
1729                         /* TC is not enabled so set the offset to
1730                          * default queue and allocate one queue
1731                          * for the given TC.
1732                          */
1733                         vsi->tc_config.tc_info[i].qoffset = 0;
1734                         vsi->tc_config.tc_info[i].qcount = 1;
1735                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1736                 }
1737         }
1738
1739         /* Set actual Tx/Rx queue pairs */
1740         vsi->num_queue_pairs = offset + qcount;
1741
1742         /* Setup queue TC[0].qmap for given VSI context */
1743         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1744         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1745         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1746         ctxt->info.valid_sections |= cpu_to_le16(sections);
1747
1748         /* Reconfigure RSS for main VSI with max queue count */
1749         vsi->rss_size = max_qcount;
1750         ret = i40e_vsi_config_rss(vsi);
1751         if (ret) {
1752                 dev_info(&vsi->back->pdev->dev,
1753                          "Failed to reconfig rss for num_queues (%u)\n",
1754                          max_qcount);
1755                 return ret;
1756         }
1757         vsi->reconfig_rss = true;
1758         dev_dbg(&vsi->back->pdev->dev,
1759                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1760
1761         /* Find queue count available for channel VSIs and starting offset
1762          * for channel VSIs
1763          */
1764         override_q = vsi->mqprio_qopt.qopt.count[0];
1765         if (override_q && override_q < vsi->num_queue_pairs) {
1766                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1767                 vsi->next_base_queue = override_q;
1768         }
1769         return 0;
1770 }
1771
1772 /**
1773  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1774  * @vsi: the VSI being setup
1775  * @ctxt: VSI context structure
1776  * @enabled_tc: Enabled TCs bitmap
1777  * @is_add: True if called before Add VSI
1778  *
1779  * Setup VSI queue mapping for enabled traffic classes.
1780  **/
1781 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1782                                      struct i40e_vsi_context *ctxt,
1783                                      u8 enabled_tc,
1784                                      bool is_add)
1785 {
1786         struct i40e_pf *pf = vsi->back;
1787         u16 sections = 0;
1788         u8 netdev_tc = 0;
1789         u16 numtc = 1;
1790         u16 qcount;
1791         u8 offset;
1792         u16 qmap;
1793         int i;
1794         u16 num_tc_qps = 0;
1795
1796         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1797         offset = 0;
1798
1799         /* Number of queues per enabled TC */
1800         num_tc_qps = vsi->alloc_queue_pairs;
1801         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1802                 /* Find numtc from enabled TC bitmap */
1803                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1804                         if (enabled_tc & BIT(i)) /* TC is enabled */
1805                                 numtc++;
1806                 }
1807                 if (!numtc) {
1808                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1809                         numtc = 1;
1810                 }
1811                 num_tc_qps = num_tc_qps / numtc;
1812                 num_tc_qps = min_t(int, num_tc_qps,
1813                                    i40e_pf_get_max_q_per_tc(pf));
1814         }
1815
1816         vsi->tc_config.numtc = numtc;
1817         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1818
1819         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1820         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1821                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1822
1823         /* Setup queue offset/count for all TCs for given VSI */
1824         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1825                 /* See if the given TC is enabled for the given VSI */
1826                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1827                         /* TC is enabled */
1828                         int pow, num_qps;
1829
1830                         switch (vsi->type) {
1831                         case I40E_VSI_MAIN:
1832                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1833                                     I40E_FLAG_FD_ATR_ENABLED)) ||
1834                                     vsi->tc_config.enabled_tc != 1) {
1835                                         qcount = min_t(int, pf->alloc_rss_size,
1836                                                        num_tc_qps);
1837                                         break;
1838                                 }
1839                                 /* fall through */
1840                         case I40E_VSI_FDIR:
1841                         case I40E_VSI_SRIOV:
1842                         case I40E_VSI_VMDQ2:
1843                         default:
1844                                 qcount = num_tc_qps;
1845                                 WARN_ON(i != 0);
1846                                 break;
1847                         }
1848                         vsi->tc_config.tc_info[i].qoffset = offset;
1849                         vsi->tc_config.tc_info[i].qcount = qcount;
1850
1851                         /* find the next higher power-of-2 of num queue pairs */
1852                         num_qps = qcount;
1853                         pow = 0;
1854                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1855                                 pow++;
1856                                 num_qps >>= 1;
1857                         }
1858
1859                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1860                         qmap =
1861                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1862                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1863
1864                         offset += qcount;
1865                 } else {
1866                         /* TC is not enabled so set the offset to
1867                          * default queue and allocate one queue
1868                          * for the given TC.
1869                          */
1870                         vsi->tc_config.tc_info[i].qoffset = 0;
1871                         vsi->tc_config.tc_info[i].qcount = 1;
1872                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1873
1874                         qmap = 0;
1875                 }
1876                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1877         }
1878
1879         /* Set actual Tx/Rx queue pairs */
1880         vsi->num_queue_pairs = offset;
1881         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1882                 if (vsi->req_queue_pairs > 0)
1883                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1884                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1885                         vsi->num_queue_pairs = pf->num_lan_msix;
1886         }
1887
1888         /* Scheduler section valid can only be set for ADD VSI */
1889         if (is_add) {
1890                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1891
1892                 ctxt->info.up_enable_bits = enabled_tc;
1893         }
1894         if (vsi->type == I40E_VSI_SRIOV) {
1895                 ctxt->info.mapping_flags |=
1896                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1897                 for (i = 0; i < vsi->num_queue_pairs; i++)
1898                         ctxt->info.queue_mapping[i] =
1899                                                cpu_to_le16(vsi->base_queue + i);
1900         } else {
1901                 ctxt->info.mapping_flags |=
1902                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1903                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1904         }
1905         ctxt->info.valid_sections |= cpu_to_le16(sections);
1906 }
1907
1908 /**
1909  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1910  * @netdev: the netdevice
1911  * @addr: address to add
1912  *
1913  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1914  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1915  */
1916 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1917 {
1918         struct i40e_netdev_priv *np = netdev_priv(netdev);
1919         struct i40e_vsi *vsi = np->vsi;
1920
1921         if (i40e_add_mac_filter(vsi, addr))
1922                 return 0;
1923         else
1924                 return -ENOMEM;
1925 }
1926
1927 /**
1928  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1929  * @netdev: the netdevice
1930  * @addr: address to add
1931  *
1932  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1933  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1934  */
1935 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1936 {
1937         struct i40e_netdev_priv *np = netdev_priv(netdev);
1938         struct i40e_vsi *vsi = np->vsi;
1939
1940         /* Under some circumstances, we might receive a request to delete
1941          * our own device address from our uc list. Because we store the
1942          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1943          * such requests and not delete our device address from this list.
1944          */
1945         if (ether_addr_equal(addr, netdev->dev_addr))
1946                 return 0;
1947
1948         i40e_del_mac_filter(vsi, addr);
1949
1950         return 0;
1951 }
1952
1953 /**
1954  * i40e_set_rx_mode - NDO callback to set the netdev filters
1955  * @netdev: network interface device structure
1956  **/
1957 static void i40e_set_rx_mode(struct net_device *netdev)
1958 {
1959         struct i40e_netdev_priv *np = netdev_priv(netdev);
1960         struct i40e_vsi *vsi = np->vsi;
1961
1962         spin_lock_bh(&vsi->mac_filter_hash_lock);
1963
1964         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1965         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1966
1967         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1968
1969         /* check for other flag changes */
1970         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1971                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1972                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1973         }
1974 }
1975
1976 /**
1977  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1978  * @vsi: Pointer to VSI struct
1979  * @from: Pointer to list which contains MAC filter entries - changes to
1980  *        those entries needs to be undone.
1981  *
1982  * MAC filter entries from this list were slated for deletion.
1983  **/
1984 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1985                                          struct hlist_head *from)
1986 {
1987         struct i40e_mac_filter *f;
1988         struct hlist_node *h;
1989
1990         hlist_for_each_entry_safe(f, h, from, hlist) {
1991                 u64 key = i40e_addr_to_hkey(f->macaddr);
1992
1993                 /* Move the element back into MAC filter list*/
1994                 hlist_del(&f->hlist);
1995                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1996         }
1997 }
1998
1999 /**
2000  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2001  * @vsi: Pointer to vsi struct
2002  * @from: Pointer to list which contains MAC filter entries - changes to
2003  *        those entries needs to be undone.
2004  *
2005  * MAC filter entries from this list were slated for addition.
2006  **/
2007 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2008                                          struct hlist_head *from)
2009 {
2010         struct i40e_new_mac_filter *new;
2011         struct hlist_node *h;
2012
2013         hlist_for_each_entry_safe(new, h, from, hlist) {
2014                 /* We can simply free the wrapper structure */
2015                 hlist_del(&new->hlist);
2016                 kfree(new);
2017         }
2018 }
2019
2020 /**
2021  * i40e_next_entry - Get the next non-broadcast filter from a list
2022  * @next: pointer to filter in list
2023  *
2024  * Returns the next non-broadcast filter in the list. Required so that we
2025  * ignore broadcast filters within the list, since these are not handled via
2026  * the normal firmware update path.
2027  */
2028 static
2029 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2030 {
2031         hlist_for_each_entry_continue(next, hlist) {
2032                 if (!is_broadcast_ether_addr(next->f->macaddr))
2033                         return next;
2034         }
2035
2036         return NULL;
2037 }
2038
2039 /**
2040  * i40e_update_filter_state - Update filter state based on return data
2041  * from firmware
2042  * @count: Number of filters added
2043  * @add_list: return data from fw
2044  * @add_head: pointer to first filter in current batch
2045  *
2046  * MAC filter entries from list were slated to be added to device. Returns
2047  * number of successful filters. Note that 0 does NOT mean success!
2048  **/
2049 static int
2050 i40e_update_filter_state(int count,
2051                          struct i40e_aqc_add_macvlan_element_data *add_list,
2052                          struct i40e_new_mac_filter *add_head)
2053 {
2054         int retval = 0;
2055         int i;
2056
2057         for (i = 0; i < count; i++) {
2058                 /* Always check status of each filter. We don't need to check
2059                  * the firmware return status because we pre-set the filter
2060                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2061                  * request to the adminq. Thus, if it no longer matches then
2062                  * we know the filter is active.
2063                  */
2064                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2065                         add_head->state = I40E_FILTER_FAILED;
2066                 } else {
2067                         add_head->state = I40E_FILTER_ACTIVE;
2068                         retval++;
2069                 }
2070
2071                 add_head = i40e_next_filter(add_head);
2072                 if (!add_head)
2073                         break;
2074         }
2075
2076         return retval;
2077 }
2078
2079 /**
2080  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2081  * @vsi: ptr to the VSI
2082  * @vsi_name: name to display in messages
2083  * @list: the list of filters to send to firmware
2084  * @num_del: the number of filters to delete
2085  * @retval: Set to -EIO on failure to delete
2086  *
2087  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2088  * *retval instead of a return value so that success does not force ret_val to
2089  * be set to 0. This ensures that a sequence of calls to this function
2090  * preserve the previous value of *retval on successful delete.
2091  */
2092 static
2093 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2094                           struct i40e_aqc_remove_macvlan_element_data *list,
2095                           int num_del, int *retval)
2096 {
2097         struct i40e_hw *hw = &vsi->back->hw;
2098         i40e_status aq_ret;
2099         int aq_err;
2100
2101         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2102         aq_err = hw->aq.asq_last_status;
2103
2104         /* Explicitly ignore and do not report when firmware returns ENOENT */
2105         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2106                 *retval = -EIO;
2107                 dev_info(&vsi->back->pdev->dev,
2108                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2109                          vsi_name, i40e_stat_str(hw, aq_ret),
2110                          i40e_aq_str(hw, aq_err));
2111         }
2112 }
2113
2114 /**
2115  * i40e_aqc_add_filters - Request firmware to add a set of filters
2116  * @vsi: ptr to the VSI
2117  * @vsi_name: name to display in messages
2118  * @list: the list of filters to send to firmware
2119  * @add_head: Position in the add hlist
2120  * @num_add: the number of filters to add
2121  *
2122  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2123  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2124  * space for more filters.
2125  */
2126 static
2127 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2128                           struct i40e_aqc_add_macvlan_element_data *list,
2129                           struct i40e_new_mac_filter *add_head,
2130                           int num_add)
2131 {
2132         struct i40e_hw *hw = &vsi->back->hw;
2133         int aq_err, fcnt;
2134
2135         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2136         aq_err = hw->aq.asq_last_status;
2137         fcnt = i40e_update_filter_state(num_add, list, add_head);
2138
2139         if (fcnt != num_add) {
2140                 if (vsi->type == I40E_VSI_MAIN) {
2141                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2142                         dev_warn(&vsi->back->pdev->dev,
2143                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2144                                  i40e_aq_str(hw, aq_err), vsi_name);
2145                 } else if (vsi->type == I40E_VSI_SRIOV ||
2146                            vsi->type == I40E_VSI_VMDQ1 ||
2147                            vsi->type == I40E_VSI_VMDQ2) {
2148                         dev_warn(&vsi->back->pdev->dev,
2149                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2150                                  i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2151                 } else {
2152                         dev_warn(&vsi->back->pdev->dev,
2153                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2154                                  i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2155                 }
2156         }
2157 }
2158
2159 /**
2160  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2161  * @vsi: pointer to the VSI
2162  * @vsi_name: the VSI name
2163  * @f: filter data
2164  *
2165  * This function sets or clears the promiscuous broadcast flags for VLAN
2166  * filters in order to properly receive broadcast frames. Assumes that only
2167  * broadcast filters are passed.
2168  *
2169  * Returns status indicating success or failure;
2170  **/
2171 static i40e_status
2172 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2173                           struct i40e_mac_filter *f)
2174 {
2175         bool enable = f->state == I40E_FILTER_NEW;
2176         struct i40e_hw *hw = &vsi->back->hw;
2177         i40e_status aq_ret;
2178
2179         if (f->vlan == I40E_VLAN_ANY) {
2180                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2181                                                    vsi->seid,
2182                                                    enable,
2183                                                    NULL);
2184         } else {
2185                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2186                                                             vsi->seid,
2187                                                             enable,
2188                                                             f->vlan,
2189                                                             NULL);
2190         }
2191
2192         if (aq_ret) {
2193                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2194                 dev_warn(&vsi->back->pdev->dev,
2195                          "Error %s, forcing overflow promiscuous on %s\n",
2196                          i40e_aq_str(hw, hw->aq.asq_last_status),
2197                          vsi_name);
2198         }
2199
2200         return aq_ret;
2201 }
2202
2203 /**
2204  * i40e_set_promiscuous - set promiscuous mode
2205  * @pf: board private structure
2206  * @promisc: promisc on or off
2207  *
2208  * There are different ways of setting promiscuous mode on a PF depending on
2209  * what state/environment we're in.  This identifies and sets it appropriately.
2210  * Returns 0 on success.
2211  **/
2212 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2213 {
2214         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2215         struct i40e_hw *hw = &pf->hw;
2216         i40e_status aq_ret;
2217
2218         if (vsi->type == I40E_VSI_MAIN &&
2219             pf->lan_veb != I40E_NO_VEB &&
2220             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2221                 /* set defport ON for Main VSI instead of true promisc
2222                  * this way we will get all unicast/multicast and VLAN
2223                  * promisc behavior but will not get VF or VMDq traffic
2224                  * replicated on the Main VSI.
2225                  */
2226                 if (promisc)
2227                         aq_ret = i40e_aq_set_default_vsi(hw,
2228                                                          vsi->seid,
2229                                                          NULL);
2230                 else
2231                         aq_ret = i40e_aq_clear_default_vsi(hw,
2232                                                            vsi->seid,
2233                                                            NULL);
2234                 if (aq_ret) {
2235                         dev_info(&pf->pdev->dev,
2236                                  "Set default VSI failed, err %s, aq_err %s\n",
2237                                  i40e_stat_str(hw, aq_ret),
2238                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2239                 }
2240         } else {
2241                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2242                                                   hw,
2243                                                   vsi->seid,
2244                                                   promisc, NULL,
2245                                                   true);
2246                 if (aq_ret) {
2247                         dev_info(&pf->pdev->dev,
2248                                  "set unicast promisc failed, err %s, aq_err %s\n",
2249                                  i40e_stat_str(hw, aq_ret),
2250                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2251                 }
2252                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2253                                                   hw,
2254                                                   vsi->seid,
2255                                                   promisc, NULL);
2256                 if (aq_ret) {
2257                         dev_info(&pf->pdev->dev,
2258                                  "set multicast promisc failed, err %s, aq_err %s\n",
2259                                  i40e_stat_str(hw, aq_ret),
2260                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2261                 }
2262         }
2263
2264         if (!aq_ret)
2265                 pf->cur_promisc = promisc;
2266
2267         return aq_ret;
2268 }
2269
2270 /**
2271  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2272  * @vsi: ptr to the VSI
2273  *
2274  * Push any outstanding VSI filter changes through the AdminQ.
2275  *
2276  * Returns 0 or error value
2277  **/
2278 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2279 {
2280         struct hlist_head tmp_add_list, tmp_del_list;
2281         struct i40e_mac_filter *f;
2282         struct i40e_new_mac_filter *new, *add_head = NULL;
2283         struct i40e_hw *hw = &vsi->back->hw;
2284         bool old_overflow, new_overflow;
2285         unsigned int failed_filters = 0;
2286         unsigned int vlan_filters = 0;
2287         char vsi_name[16] = "PF";
2288         int filter_list_len = 0;
2289         i40e_status aq_ret = 0;
2290         u32 changed_flags = 0;
2291         struct hlist_node *h;
2292         struct i40e_pf *pf;
2293         int num_add = 0;
2294         int num_del = 0;
2295         int retval = 0;
2296         u16 cmd_flags;
2297         int list_size;
2298         int bkt;
2299
2300         /* empty array typed pointers, kcalloc later */
2301         struct i40e_aqc_add_macvlan_element_data *add_list;
2302         struct i40e_aqc_remove_macvlan_element_data *del_list;
2303
2304         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2305                 usleep_range(1000, 2000);
2306         pf = vsi->back;
2307
2308         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2309
2310         if (vsi->netdev) {
2311                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2312                 vsi->current_netdev_flags = vsi->netdev->flags;
2313         }
2314
2315         INIT_HLIST_HEAD(&tmp_add_list);
2316         INIT_HLIST_HEAD(&tmp_del_list);
2317
2318         if (vsi->type == I40E_VSI_SRIOV)
2319                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2320         else if (vsi->type != I40E_VSI_MAIN)
2321                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2322
2323         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2324                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2325
2326                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2327                 /* Create a list of filters to delete. */
2328                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2329                         if (f->state == I40E_FILTER_REMOVE) {
2330                                 /* Move the element into temporary del_list */
2331                                 hash_del(&f->hlist);
2332                                 hlist_add_head(&f->hlist, &tmp_del_list);
2333
2334                                 /* Avoid counting removed filters */
2335                                 continue;
2336                         }
2337                         if (f->state == I40E_FILTER_NEW) {
2338                                 /* Create a temporary i40e_new_mac_filter */
2339                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2340                                 if (!new)
2341                                         goto err_no_memory_locked;
2342
2343                                 /* Store pointer to the real filter */
2344                                 new->f = f;
2345                                 new->state = f->state;
2346
2347                                 /* Add it to the hash list */
2348                                 hlist_add_head(&new->hlist, &tmp_add_list);
2349                         }
2350
2351                         /* Count the number of active (current and new) VLAN
2352                          * filters we have now. Does not count filters which
2353                          * are marked for deletion.
2354                          */
2355                         if (f->vlan > 0)
2356                                 vlan_filters++;
2357                 }
2358
2359                 retval = i40e_correct_mac_vlan_filters(vsi,
2360                                                        &tmp_add_list,
2361                                                        &tmp_del_list,
2362                                                        vlan_filters);
2363                 if (retval)
2364                         goto err_no_memory_locked;
2365
2366                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2367         }
2368
2369         /* Now process 'del_list' outside the lock */
2370         if (!hlist_empty(&tmp_del_list)) {
2371                 filter_list_len = hw->aq.asq_buf_size /
2372                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2373                 list_size = filter_list_len *
2374                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2375                 del_list = kzalloc(list_size, GFP_ATOMIC);
2376                 if (!del_list)
2377                         goto err_no_memory;
2378
2379                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2380                         cmd_flags = 0;
2381
2382                         /* handle broadcast filters by updating the broadcast
2383                          * promiscuous flag and release filter list.
2384                          */
2385                         if (is_broadcast_ether_addr(f->macaddr)) {
2386                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2387
2388                                 hlist_del(&f->hlist);
2389                                 kfree(f);
2390                                 continue;
2391                         }
2392
2393                         /* add to delete list */
2394                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2395                         if (f->vlan == I40E_VLAN_ANY) {
2396                                 del_list[num_del].vlan_tag = 0;
2397                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2398                         } else {
2399                                 del_list[num_del].vlan_tag =
2400                                         cpu_to_le16((u16)(f->vlan));
2401                         }
2402
2403                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2404                         del_list[num_del].flags = cmd_flags;
2405                         num_del++;
2406
2407                         /* flush a full buffer */
2408                         if (num_del == filter_list_len) {
2409                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2410                                                      num_del, &retval);
2411                                 memset(del_list, 0, list_size);
2412                                 num_del = 0;
2413                         }
2414                         /* Release memory for MAC filter entries which were
2415                          * synced up with HW.
2416                          */
2417                         hlist_del(&f->hlist);
2418                         kfree(f);
2419                 }
2420
2421                 if (num_del) {
2422                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2423                                              num_del, &retval);
2424                 }
2425
2426                 kfree(del_list);
2427                 del_list = NULL;
2428         }
2429
2430         if (!hlist_empty(&tmp_add_list)) {
2431                 /* Do all the adds now. */
2432                 filter_list_len = hw->aq.asq_buf_size /
2433                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2434                 list_size = filter_list_len *
2435                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2436                 add_list = kzalloc(list_size, GFP_ATOMIC);
2437                 if (!add_list)
2438                         goto err_no_memory;
2439
2440                 num_add = 0;
2441                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2442                         /* handle broadcast filters by updating the broadcast
2443                          * promiscuous flag instead of adding a MAC filter.
2444                          */
2445                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2446                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2447                                                               new->f))
2448                                         new->state = I40E_FILTER_FAILED;
2449                                 else
2450                                         new->state = I40E_FILTER_ACTIVE;
2451                                 continue;
2452                         }
2453
2454                         /* add to add array */
2455                         if (num_add == 0)
2456                                 add_head = new;
2457                         cmd_flags = 0;
2458                         ether_addr_copy(add_list[num_add].mac_addr,
2459                                         new->f->macaddr);
2460                         if (new->f->vlan == I40E_VLAN_ANY) {
2461                                 add_list[num_add].vlan_tag = 0;
2462                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2463                         } else {
2464                                 add_list[num_add].vlan_tag =
2465                                         cpu_to_le16((u16)(new->f->vlan));
2466                         }
2467                         add_list[num_add].queue_number = 0;
2468                         /* set invalid match method for later detection */
2469                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2470                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2471                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2472                         num_add++;
2473
2474                         /* flush a full buffer */
2475                         if (num_add == filter_list_len) {
2476                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2477                                                      add_head, num_add);
2478                                 memset(add_list, 0, list_size);
2479                                 num_add = 0;
2480                         }
2481                 }
2482                 if (num_add) {
2483                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2484                                              num_add);
2485                 }
2486                 /* Now move all of the filters from the temp add list back to
2487                  * the VSI's list.
2488                  */
2489                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2490                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2491                         /* Only update the state if we're still NEW */
2492                         if (new->f->state == I40E_FILTER_NEW)
2493                                 new->f->state = new->state;
2494                         hlist_del(&new->hlist);
2495                         kfree(new);
2496                 }
2497                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2498                 kfree(add_list);
2499                 add_list = NULL;
2500         }
2501
2502         /* Determine the number of active and failed filters. */
2503         spin_lock_bh(&vsi->mac_filter_hash_lock);
2504         vsi->active_filters = 0;
2505         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2506                 if (f->state == I40E_FILTER_ACTIVE)
2507                         vsi->active_filters++;
2508                 else if (f->state == I40E_FILTER_FAILED)
2509                         failed_filters++;
2510         }
2511         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2512
2513         /* Check if we are able to exit overflow promiscuous mode. We can
2514          * safely exit if we didn't just enter, we no longer have any failed
2515          * filters, and we have reduced filters below the threshold value.
2516          */
2517         if (old_overflow && !failed_filters &&
2518             vsi->active_filters < vsi->promisc_threshold) {
2519                 dev_info(&pf->pdev->dev,
2520                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2521                          vsi_name);
2522                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2523                 vsi->promisc_threshold = 0;
2524         }
2525
2526         /* if the VF is not trusted do not do promisc */
2527         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2528                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2529                 goto out;
2530         }
2531
2532         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2533
2534         /* If we are entering overflow promiscuous, we need to calculate a new
2535          * threshold for when we are safe to exit
2536          */
2537         if (!old_overflow && new_overflow)
2538                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2539
2540         /* check for changes in promiscuous modes */
2541         if (changed_flags & IFF_ALLMULTI) {
2542                 bool cur_multipromisc;
2543
2544                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2545                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2546                                                                vsi->seid,
2547                                                                cur_multipromisc,
2548                                                                NULL);
2549                 if (aq_ret) {
2550                         retval = i40e_aq_rc_to_posix(aq_ret,
2551                                                      hw->aq.asq_last_status);
2552                         dev_info(&pf->pdev->dev,
2553                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2554                                  vsi_name,
2555                                  i40e_stat_str(hw, aq_ret),
2556                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2557                 } else {
2558                         dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n",
2559                                  vsi->netdev->name,
2560                                  cur_multipromisc ? "entering" : "leaving");
2561                 }
2562         }
2563
2564         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2565                 bool cur_promisc;
2566
2567                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2568                                new_overflow);
2569                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2570                 if (aq_ret) {
2571                         retval = i40e_aq_rc_to_posix(aq_ret,
2572                                                      hw->aq.asq_last_status);
2573                         dev_info(&pf->pdev->dev,
2574                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2575                                  cur_promisc ? "on" : "off",
2576                                  vsi_name,
2577                                  i40e_stat_str(hw, aq_ret),
2578                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2579                 }
2580         }
2581 out:
2582         /* if something went wrong then set the changed flag so we try again */
2583         if (retval)
2584                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2585
2586         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2587         return retval;
2588
2589 err_no_memory:
2590         /* Restore elements on the temporary add and delete lists */
2591         spin_lock_bh(&vsi->mac_filter_hash_lock);
2592 err_no_memory_locked:
2593         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2594         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2595         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2596
2597         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2598         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2599         return -ENOMEM;
2600 }
2601
2602 /**
2603  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2604  * @pf: board private structure
2605  **/
2606 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2607 {
2608         int v;
2609
2610         if (!pf)
2611                 return;
2612         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2613                 return;
2614         if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) {
2615                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2616                 return;
2617         }
2618
2619         for (v = 0; v < pf->num_alloc_vsi; v++) {
2620                 if (pf->vsi[v] &&
2621                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2622                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2623
2624                         if (ret) {
2625                                 /* come back and try again later */
2626                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2627                                         pf->state);
2628                                 break;
2629                         }
2630                 }
2631         }
2632         clear_bit(__I40E_VF_DISABLE, pf->state);
2633 }
2634
2635 /**
2636  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2637  * @vsi: the vsi
2638  **/
2639 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2640 {
2641         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2642                 return I40E_RXBUFFER_2048;
2643         else
2644                 return I40E_RXBUFFER_3072;
2645 }
2646
2647 /**
2648  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2649  * @netdev: network interface device structure
2650  * @new_mtu: new value for maximum frame size
2651  *
2652  * Returns 0 on success, negative on failure
2653  **/
2654 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2655 {
2656         struct i40e_netdev_priv *np = netdev_priv(netdev);
2657         struct i40e_vsi *vsi = np->vsi;
2658         struct i40e_pf *pf = vsi->back;
2659
2660         if (i40e_enabled_xdp_vsi(vsi)) {
2661                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2662
2663                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2664                         return -EINVAL;
2665         }
2666
2667         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2668                    netdev->mtu, new_mtu);
2669         netdev->mtu = new_mtu;
2670         if (netif_running(netdev))
2671                 i40e_vsi_reinit_locked(vsi);
2672         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2673         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2674         return 0;
2675 }
2676
2677 /**
2678  * i40e_ioctl - Access the hwtstamp interface
2679  * @netdev: network interface device structure
2680  * @ifr: interface request data
2681  * @cmd: ioctl command
2682  **/
2683 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2684 {
2685         struct i40e_netdev_priv *np = netdev_priv(netdev);
2686         struct i40e_pf *pf = np->vsi->back;
2687
2688         switch (cmd) {
2689         case SIOCGHWTSTAMP:
2690                 return i40e_ptp_get_ts_config(pf, ifr);
2691         case SIOCSHWTSTAMP:
2692                 return i40e_ptp_set_ts_config(pf, ifr);
2693         default:
2694                 return -EOPNOTSUPP;
2695         }
2696 }
2697
2698 /**
2699  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2700  * @vsi: the vsi being adjusted
2701  **/
2702 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2703 {
2704         struct i40e_vsi_context ctxt;
2705         i40e_status ret;
2706
2707         /* Don't modify stripping options if a port VLAN is active */
2708         if (vsi->info.pvid)
2709                 return;
2710
2711         if ((vsi->info.valid_sections &
2712              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2713             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2714                 return;  /* already enabled */
2715
2716         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2717         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2718                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2719
2720         ctxt.seid = vsi->seid;
2721         ctxt.info = vsi->info;
2722         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2723         if (ret) {
2724                 dev_info(&vsi->back->pdev->dev,
2725                          "update vlan stripping failed, err %s aq_err %s\n",
2726                          i40e_stat_str(&vsi->back->hw, ret),
2727                          i40e_aq_str(&vsi->back->hw,
2728                                      vsi->back->hw.aq.asq_last_status));
2729         }
2730 }
2731
2732 /**
2733  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2734  * @vsi: the vsi being adjusted
2735  **/
2736 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2737 {
2738         struct i40e_vsi_context ctxt;
2739         i40e_status ret;
2740
2741         /* Don't modify stripping options if a port VLAN is active */
2742         if (vsi->info.pvid)
2743                 return;
2744
2745         if ((vsi->info.valid_sections &
2746              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2747             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2748              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2749                 return;  /* already disabled */
2750
2751         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2752         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2753                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2754
2755         ctxt.seid = vsi->seid;
2756         ctxt.info = vsi->info;
2757         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2758         if (ret) {
2759                 dev_info(&vsi->back->pdev->dev,
2760                          "update vlan stripping failed, err %s aq_err %s\n",
2761                          i40e_stat_str(&vsi->back->hw, ret),
2762                          i40e_aq_str(&vsi->back->hw,
2763                                      vsi->back->hw.aq.asq_last_status));
2764         }
2765 }
2766
2767 /**
2768  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2769  * @vsi: the vsi being configured
2770  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2771  *
2772  * This is a helper function for adding a new MAC/VLAN filter with the
2773  * specified VLAN for each existing MAC address already in the hash table.
2774  * This function does *not* perform any accounting to update filters based on
2775  * VLAN mode.
2776  *
2777  * NOTE: this function expects to be called while under the
2778  * mac_filter_hash_lock
2779  **/
2780 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2781 {
2782         struct i40e_mac_filter *f, *add_f;
2783         struct hlist_node *h;
2784         int bkt;
2785
2786         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2787                 if (f->state == I40E_FILTER_REMOVE)
2788                         continue;
2789                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2790                 if (!add_f) {
2791                         dev_info(&vsi->back->pdev->dev,
2792                                  "Could not add vlan filter %d for %pM\n",
2793                                  vid, f->macaddr);
2794                         return -ENOMEM;
2795                 }
2796         }
2797
2798         return 0;
2799 }
2800
2801 /**
2802  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2803  * @vsi: the VSI being configured
2804  * @vid: VLAN id to be added
2805  **/
2806 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2807 {
2808         int err;
2809
2810         if (vsi->info.pvid)
2811                 return -EINVAL;
2812
2813         /* The network stack will attempt to add VID=0, with the intention to
2814          * receive priority tagged packets with a VLAN of 0. Our HW receives
2815          * these packets by default when configured to receive untagged
2816          * packets, so we don't need to add a filter for this case.
2817          * Additionally, HW interprets adding a VID=0 filter as meaning to
2818          * receive *only* tagged traffic and stops receiving untagged traffic.
2819          * Thus, we do not want to actually add a filter for VID=0
2820          */
2821         if (!vid)
2822                 return 0;
2823
2824         /* Locked once because all functions invoked below iterates list*/
2825         spin_lock_bh(&vsi->mac_filter_hash_lock);
2826         err = i40e_add_vlan_all_mac(vsi, vid);
2827         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2828         if (err)
2829                 return err;
2830
2831         /* schedule our worker thread which will take care of
2832          * applying the new filter changes
2833          */
2834         i40e_service_event_schedule(vsi->back);
2835         return 0;
2836 }
2837
2838 /**
2839  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2840  * @vsi: the vsi being configured
2841  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2842  *
2843  * This function should be used to remove all VLAN filters which match the
2844  * given VID. It does not schedule the service event and does not take the
2845  * mac_filter_hash_lock so it may be combined with other operations under
2846  * a single invocation of the mac_filter_hash_lock.
2847  *
2848  * NOTE: this function expects to be called while under the
2849  * mac_filter_hash_lock
2850  */
2851 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2852 {
2853         struct i40e_mac_filter *f;
2854         struct hlist_node *h;
2855         int bkt;
2856
2857         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2858                 if (f->vlan == vid)
2859                         __i40e_del_filter(vsi, f);
2860         }
2861 }
2862
2863 /**
2864  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2865  * @vsi: the VSI being configured
2866  * @vid: VLAN id to be removed
2867  **/
2868 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2869 {
2870         if (!vid || vsi->info.pvid)
2871                 return;
2872
2873         spin_lock_bh(&vsi->mac_filter_hash_lock);
2874         i40e_rm_vlan_all_mac(vsi, vid);
2875         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2876
2877         /* schedule our worker thread which will take care of
2878          * applying the new filter changes
2879          */
2880         i40e_service_event_schedule(vsi->back);
2881 }
2882
2883 /**
2884  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2885  * @netdev: network interface to be adjusted
2886  * @proto: unused protocol value
2887  * @vid: vlan id to be added
2888  *
2889  * net_device_ops implementation for adding vlan ids
2890  **/
2891 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2892                                 __always_unused __be16 proto, u16 vid)
2893 {
2894         struct i40e_netdev_priv *np = netdev_priv(netdev);
2895         struct i40e_vsi *vsi = np->vsi;
2896         int ret = 0;
2897
2898         if (vid >= VLAN_N_VID)
2899                 return -EINVAL;
2900
2901         ret = i40e_vsi_add_vlan(vsi, vid);
2902         if (!ret)
2903                 set_bit(vid, vsi->active_vlans);
2904
2905         return ret;
2906 }
2907
2908 /**
2909  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
2910  * @netdev: network interface to be adjusted
2911  * @proto: unused protocol value
2912  * @vid: vlan id to be added
2913  **/
2914 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2915                                     __always_unused __be16 proto, u16 vid)
2916 {
2917         struct i40e_netdev_priv *np = netdev_priv(netdev);
2918         struct i40e_vsi *vsi = np->vsi;
2919
2920         if (vid >= VLAN_N_VID)
2921                 return;
2922         set_bit(vid, vsi->active_vlans);
2923 }
2924
2925 /**
2926  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2927  * @netdev: network interface to be adjusted
2928  * @proto: unused protocol value
2929  * @vid: vlan id to be removed
2930  *
2931  * net_device_ops implementation for removing vlan ids
2932  **/
2933 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2934                                  __always_unused __be16 proto, u16 vid)
2935 {
2936         struct i40e_netdev_priv *np = netdev_priv(netdev);
2937         struct i40e_vsi *vsi = np->vsi;
2938
2939         /* return code is ignored as there is nothing a user
2940          * can do about failure to remove and a log message was
2941          * already printed from the other function
2942          */
2943         i40e_vsi_kill_vlan(vsi, vid);
2944
2945         clear_bit(vid, vsi->active_vlans);
2946
2947         return 0;
2948 }
2949
2950 /**
2951  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2952  * @vsi: the vsi being brought back up
2953  **/
2954 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2955 {
2956         u16 vid;
2957
2958         if (!vsi->netdev)
2959                 return;
2960
2961         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2962                 i40e_vlan_stripping_enable(vsi);
2963         else
2964                 i40e_vlan_stripping_disable(vsi);
2965
2966         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2967                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
2968                                         vid);
2969 }
2970
2971 /**
2972  * i40e_vsi_add_pvid - Add pvid for the VSI
2973  * @vsi: the vsi being adjusted
2974  * @vid: the vlan id to set as a PVID
2975  **/
2976 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2977 {
2978         struct i40e_vsi_context ctxt;
2979         i40e_status ret;
2980
2981         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2982         vsi->info.pvid = cpu_to_le16(vid);
2983         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2984                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2985                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2986
2987         ctxt.seid = vsi->seid;
2988         ctxt.info = vsi->info;
2989         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2990         if (ret) {
2991                 dev_info(&vsi->back->pdev->dev,
2992                          "add pvid failed, err %s aq_err %s\n",
2993                          i40e_stat_str(&vsi->back->hw, ret),
2994                          i40e_aq_str(&vsi->back->hw,
2995                                      vsi->back->hw.aq.asq_last_status));
2996                 return -ENOENT;
2997         }
2998
2999         return 0;
3000 }
3001
3002 /**
3003  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3004  * @vsi: the vsi being adjusted
3005  *
3006  * Just use the vlan_rx_register() service to put it back to normal
3007  **/
3008 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3009 {
3010         vsi->info.pvid = 0;
3011
3012         i40e_vlan_stripping_disable(vsi);
3013 }
3014
3015 /**
3016  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3017  * @vsi: ptr to the VSI
3018  *
3019  * If this function returns with an error, then it's possible one or
3020  * more of the rings is populated (while the rest are not).  It is the
3021  * callers duty to clean those orphaned rings.
3022  *
3023  * Return 0 on success, negative on failure
3024  **/
3025 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3026 {
3027         int i, err = 0;
3028
3029         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3030                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3031
3032         if (!i40e_enabled_xdp_vsi(vsi))
3033                 return err;
3034
3035         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3036                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3037
3038         return err;
3039 }
3040
3041 /**
3042  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3043  * @vsi: ptr to the VSI
3044  *
3045  * Free VSI's transmit software resources
3046  **/
3047 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3048 {
3049         int i;
3050
3051         if (vsi->tx_rings) {
3052                 for (i = 0; i < vsi->num_queue_pairs; i++)
3053                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3054                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3055         }
3056
3057         if (vsi->xdp_rings) {
3058                 for (i = 0; i < vsi->num_queue_pairs; i++)
3059                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3060                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3061         }
3062 }
3063
3064 /**
3065  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3066  * @vsi: ptr to the VSI
3067  *
3068  * If this function returns with an error, then it's possible one or
3069  * more of the rings is populated (while the rest are not).  It is the
3070  * callers duty to clean those orphaned rings.
3071  *
3072  * Return 0 on success, negative on failure
3073  **/
3074 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3075 {
3076         int i, err = 0;
3077
3078         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3079                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3080         return err;
3081 }
3082
3083 /**
3084  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3085  * @vsi: ptr to the VSI
3086  *
3087  * Free all receive software resources
3088  **/
3089 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3090 {
3091         int i;
3092
3093         if (!vsi->rx_rings)
3094                 return;
3095
3096         for (i = 0; i < vsi->num_queue_pairs; i++)
3097                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3098                         i40e_free_rx_resources(vsi->rx_rings[i]);
3099 }
3100
3101 /**
3102  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3103  * @ring: The Tx ring to configure
3104  *
3105  * This enables/disables XPS for a given Tx descriptor ring
3106  * based on the TCs enabled for the VSI that ring belongs to.
3107  **/
3108 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3109 {
3110         int cpu;
3111
3112         if (!ring->q_vector || !ring->netdev || ring->ch)
3113                 return;
3114
3115         /* We only initialize XPS once, so as not to overwrite user settings */
3116         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3117                 return;
3118
3119         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3120         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3121                             ring->queue_index);
3122 }
3123
3124 /**
3125  * i40e_xsk_umem - Retrieve the AF_XDP ZC if XDP and ZC is enabled
3126  * @ring: The Tx or Rx ring
3127  *
3128  * Returns the UMEM or NULL.
3129  **/
3130 static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
3131 {
3132         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3133         int qid = ring->queue_index;
3134
3135         if (ring_is_xdp(ring))
3136                 qid -= ring->vsi->alloc_queue_pairs;
3137
3138         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3139                 return NULL;
3140
3141         return xdp_get_umem_from_qid(ring->vsi->netdev, qid);
3142 }
3143
3144 /**
3145  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3146  * @ring: The Tx ring to configure
3147  *
3148  * Configure the Tx descriptor ring in the HMC context.
3149  **/
3150 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3151 {
3152         struct i40e_vsi *vsi = ring->vsi;
3153         u16 pf_q = vsi->base_queue + ring->queue_index;
3154         struct i40e_hw *hw = &vsi->back->hw;
3155         struct i40e_hmc_obj_txq tx_ctx;
3156         i40e_status err = 0;
3157         u32 qtx_ctl = 0;
3158
3159         if (ring_is_xdp(ring))
3160                 ring->xsk_umem = i40e_xsk_umem(ring);
3161
3162         /* some ATR related tx ring init */
3163         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3164                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3165                 ring->atr_count = 0;
3166         } else {
3167                 ring->atr_sample_rate = 0;
3168         }
3169
3170         /* configure XPS */
3171         i40e_config_xps_tx_ring(ring);
3172
3173         /* clear the context structure first */
3174         memset(&tx_ctx, 0, sizeof(tx_ctx));
3175
3176         tx_ctx.new_context = 1;
3177         tx_ctx.base = (ring->dma / 128);
3178         tx_ctx.qlen = ring->count;
3179         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3180                                                I40E_FLAG_FD_ATR_ENABLED));
3181         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3182         /* FDIR VSI tx ring can still use RS bit and writebacks */
3183         if (vsi->type != I40E_VSI_FDIR)
3184                 tx_ctx.head_wb_ena = 1;
3185         tx_ctx.head_wb_addr = ring->dma +
3186                               (ring->count * sizeof(struct i40e_tx_desc));
3187
3188         /* As part of VSI creation/update, FW allocates certain
3189          * Tx arbitration queue sets for each TC enabled for
3190          * the VSI. The FW returns the handles to these queue
3191          * sets as part of the response buffer to Add VSI,
3192          * Update VSI, etc. AQ commands. It is expected that
3193          * these queue set handles be associated with the Tx
3194          * queues by the driver as part of the TX queue context
3195          * initialization. This has to be done regardless of
3196          * DCB as by default everything is mapped to TC0.
3197          */
3198
3199         if (ring->ch)
3200                 tx_ctx.rdylist =
3201                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3202
3203         else
3204                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3205
3206         tx_ctx.rdylist_act = 0;
3207
3208         /* clear the context in the HMC */
3209         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3210         if (err) {
3211                 dev_info(&vsi->back->pdev->dev,
3212                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3213                          ring->queue_index, pf_q, err);
3214                 return -ENOMEM;
3215         }
3216
3217         /* set the context in the HMC */
3218         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3219         if (err) {
3220                 dev_info(&vsi->back->pdev->dev,
3221                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3222                          ring->queue_index, pf_q, err);
3223                 return -ENOMEM;
3224         }
3225
3226         /* Now associate this queue with this PCI function */
3227         if (ring->ch) {
3228                 if (ring->ch->type == I40E_VSI_VMDQ2)
3229                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3230                 else
3231                         return -EINVAL;
3232
3233                 qtx_ctl |= (ring->ch->vsi_number <<
3234                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3235                             I40E_QTX_CTL_VFVM_INDX_MASK;
3236         } else {
3237                 if (vsi->type == I40E_VSI_VMDQ2) {
3238                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3239                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3240                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3241                 } else {
3242                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3243                 }
3244         }
3245
3246         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3247                     I40E_QTX_CTL_PF_INDX_MASK);
3248         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3249         i40e_flush(hw);
3250
3251         /* cache tail off for easier writes later */
3252         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3253
3254         return 0;
3255 }
3256
3257 /**
3258  * i40e_configure_rx_ring - Configure a receive ring context
3259  * @ring: The Rx ring to configure
3260  *
3261  * Configure the Rx descriptor ring in the HMC context.
3262  **/
3263 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3264 {
3265         struct i40e_vsi *vsi = ring->vsi;
3266         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3267         u16 pf_q = vsi->base_queue + ring->queue_index;
3268         struct i40e_hw *hw = &vsi->back->hw;
3269         struct i40e_hmc_obj_rxq rx_ctx;
3270         i40e_status err = 0;
3271         bool ok;
3272         int ret;
3273
3274         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3275
3276         /* clear the context structure first */
3277         memset(&rx_ctx, 0, sizeof(rx_ctx));
3278
3279         if (ring->vsi->type == I40E_VSI_MAIN)
3280                 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3281
3282         ring->xsk_umem = i40e_xsk_umem(ring);
3283         if (ring->xsk_umem) {
3284                 ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
3285                                    XDP_PACKET_HEADROOM;
3286                 /* For AF_XDP ZC, we disallow packets to span on
3287                  * multiple buffers, thus letting us skip that
3288                  * handling in the fast-path.
3289                  */
3290                 chain_len = 1;
3291                 ring->zca.free = i40e_zca_free;
3292                 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3293                                                  MEM_TYPE_ZERO_COPY,
3294                                                  &ring->zca);
3295                 if (ret)
3296                         return ret;
3297                 dev_info(&vsi->back->pdev->dev,
3298                          "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
3299                          ring->queue_index);
3300
3301         } else {
3302                 ring->rx_buf_len = vsi->rx_buf_len;
3303                 if (ring->vsi->type == I40E_VSI_MAIN) {
3304                         ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3305                                                          MEM_TYPE_PAGE_SHARED,
3306                                                          NULL);
3307                         if (ret)
3308                                 return ret;
3309                 }
3310         }
3311
3312         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3313                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3314
3315         rx_ctx.base = (ring->dma / 128);
3316         rx_ctx.qlen = ring->count;
3317
3318         /* use 32 byte descriptors */
3319         rx_ctx.dsize = 1;
3320
3321         /* descriptor type is always zero
3322          * rx_ctx.dtype = 0;
3323          */
3324         rx_ctx.hsplit_0 = 0;
3325
3326         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3327         if (hw->revision_id == 0)
3328                 rx_ctx.lrxqthresh = 0;
3329         else
3330                 rx_ctx.lrxqthresh = 1;
3331         rx_ctx.crcstrip = 1;
3332         rx_ctx.l2tsel = 1;
3333         /* this controls whether VLAN is stripped from inner headers */
3334         rx_ctx.showiv = 0;
3335         /* set the prefena field to 1 because the manual says to */
3336         rx_ctx.prefena = 1;
3337
3338         /* clear the context in the HMC */
3339         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3340         if (err) {
3341                 dev_info(&vsi->back->pdev->dev,
3342                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3343                          ring->queue_index, pf_q, err);
3344                 return -ENOMEM;
3345         }
3346
3347         /* set the context in the HMC */
3348         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3349         if (err) {
3350                 dev_info(&vsi->back->pdev->dev,
3351                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3352                          ring->queue_index, pf_q, err);
3353                 return -ENOMEM;
3354         }
3355
3356         /* configure Rx buffer alignment */
3357         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3358                 clear_ring_build_skb_enabled(ring);
3359         else
3360                 set_ring_build_skb_enabled(ring);
3361
3362         /* cache tail for quicker writes, and clear the reg before use */
3363         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3364         writel(0, ring->tail);
3365
3366         ok = ring->xsk_umem ?
3367              i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) :
3368              !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3369         if (!ok) {
3370                 /* Log this in case the user has forgotten to give the kernel
3371                  * any buffers, even later in the application.
3372                  */
3373                 dev_info(&vsi->back->pdev->dev,
3374                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3375                          ring->xsk_umem ? "UMEM enabled " : "",
3376                          ring->queue_index, pf_q);
3377         }
3378
3379         return 0;
3380 }
3381
3382 /**
3383  * i40e_vsi_configure_tx - Configure the VSI for Tx
3384  * @vsi: VSI structure describing this set of rings and resources
3385  *
3386  * Configure the Tx VSI for operation.
3387  **/
3388 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3389 {
3390         int err = 0;
3391         u16 i;
3392
3393         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3394                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3395
3396         if (err || !i40e_enabled_xdp_vsi(vsi))
3397                 return err;
3398
3399         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3400                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3401
3402         return err;
3403 }
3404
3405 /**
3406  * i40e_vsi_configure_rx - Configure the VSI for Rx
3407  * @vsi: the VSI being configured
3408  *
3409  * Configure the Rx VSI for operation.
3410  **/
3411 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3412 {
3413         int err = 0;
3414         u16 i;
3415
3416         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3417                 vsi->max_frame = I40E_MAX_RXBUFFER;
3418                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3419 #if (PAGE_SIZE < 8192)
3420         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3421                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3422                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3423                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3424 #endif
3425         } else {
3426                 vsi->max_frame = I40E_MAX_RXBUFFER;
3427                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3428                                                        I40E_RXBUFFER_2048;
3429         }
3430
3431         /* set up individual rings */
3432         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3433                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3434
3435         return err;
3436 }
3437
3438 /**
3439  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3440  * @vsi: ptr to the VSI
3441  **/
3442 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3443 {
3444         struct i40e_ring *tx_ring, *rx_ring;
3445         u16 qoffset, qcount;
3446         int i, n;
3447
3448         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3449                 /* Reset the TC information */
3450                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3451                         rx_ring = vsi->rx_rings[i];
3452                         tx_ring = vsi->tx_rings[i];
3453                         rx_ring->dcb_tc = 0;
3454                         tx_ring->dcb_tc = 0;
3455                 }
3456                 return;
3457         }
3458
3459         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3460                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3461                         continue;
3462
3463                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3464                 qcount = vsi->tc_config.tc_info[n].qcount;
3465                 for (i = qoffset; i < (qoffset + qcount); i++) {
3466                         rx_ring = vsi->rx_rings[i];
3467                         tx_ring = vsi->tx_rings[i];
3468                         rx_ring->dcb_tc = n;
3469                         tx_ring->dcb_tc = n;
3470                 }
3471         }
3472 }
3473
3474 /**
3475  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3476  * @vsi: ptr to the VSI
3477  **/
3478 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3479 {
3480         if (vsi->netdev)
3481                 i40e_set_rx_mode(vsi->netdev);
3482 }
3483
3484 /**
3485  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3486  * @vsi: Pointer to the targeted VSI
3487  *
3488  * This function replays the hlist on the hw where all the SB Flow Director
3489  * filters were saved.
3490  **/
3491 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3492 {
3493         struct i40e_fdir_filter *filter;
3494         struct i40e_pf *pf = vsi->back;
3495         struct hlist_node *node;
3496
3497         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3498                 return;
3499
3500         /* Reset FDir counters as we're replaying all existing filters */
3501         pf->fd_tcp4_filter_cnt = 0;
3502         pf->fd_udp4_filter_cnt = 0;
3503         pf->fd_sctp4_filter_cnt = 0;
3504         pf->fd_ip4_filter_cnt = 0;
3505
3506         hlist_for_each_entry_safe(filter, node,
3507                                   &pf->fdir_filter_list, fdir_node) {
3508                 i40e_add_del_fdir(vsi, filter, true);
3509         }
3510 }
3511
3512 /**
3513  * i40e_vsi_configure - Set up the VSI for action
3514  * @vsi: the VSI being configured
3515  **/
3516 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3517 {
3518         int err;
3519
3520         i40e_set_vsi_rx_mode(vsi);
3521         i40e_restore_vlan(vsi);
3522         i40e_vsi_config_dcb_rings(vsi);
3523         err = i40e_vsi_configure_tx(vsi);
3524         if (!err)
3525                 err = i40e_vsi_configure_rx(vsi);
3526
3527         return err;
3528 }
3529
3530 /**
3531  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3532  * @vsi: the VSI being configured
3533  **/
3534 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3535 {
3536         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3537         struct i40e_pf *pf = vsi->back;
3538         struct i40e_hw *hw = &pf->hw;
3539         u16 vector;
3540         int i, q;
3541         u32 qp;
3542
3543         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3544          * and PFINT_LNKLSTn registers, e.g.:
3545          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3546          */
3547         qp = vsi->base_queue;
3548         vector = vsi->base_vector;
3549         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3550                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3551
3552                 q_vector->rx.next_update = jiffies + 1;
3553                 q_vector->rx.target_itr =
3554                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3555                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3556                      q_vector->rx.target_itr >> 1);
3557                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3558
3559                 q_vector->tx.next_update = jiffies + 1;
3560                 q_vector->tx.target_itr =
3561                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3562                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3563                      q_vector->tx.target_itr >> 1);
3564                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3565
3566                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3567                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3568
3569                 /* Linked list for the queuepairs assigned to this vector */
3570                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3571                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3572                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3573                         u32 val;
3574
3575                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3576                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3577                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3578                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3579                               (I40E_QUEUE_TYPE_TX <<
3580                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3581
3582                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3583
3584                         if (has_xdp) {
3585                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3586                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3587                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3588                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3589                                       (I40E_QUEUE_TYPE_TX <<
3590                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3591
3592                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3593                         }
3594
3595                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3596                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3597                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3598                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3599                               (I40E_QUEUE_TYPE_RX <<
3600                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3601
3602                         /* Terminate the linked list */
3603                         if (q == (q_vector->num_ringpairs - 1))
3604                                 val |= (I40E_QUEUE_END_OF_LIST <<
3605                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3606
3607                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3608                         qp++;
3609                 }
3610         }
3611
3612         i40e_flush(hw);
3613 }
3614
3615 /**
3616  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3617  * @pf: pointer to private device data structure
3618  **/
3619 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3620 {
3621         struct i40e_hw *hw = &pf->hw;
3622         u32 val;
3623
3624         /* clear things first */
3625         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3626         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3627
3628         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3629               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3630               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3631               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3632               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3633               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3634               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3635               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3636
3637         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3638                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3639
3640         if (pf->flags & I40E_FLAG_PTP)
3641                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3642
3643         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3644
3645         /* SW_ITR_IDX = 0, but don't change INTENA */
3646         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3647                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3648
3649         /* OTHER_ITR_IDX = 0 */
3650         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3651 }
3652
3653 /**
3654  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3655  * @vsi: the VSI being configured
3656  **/
3657 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3658 {
3659         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3660         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3661         struct i40e_pf *pf = vsi->back;
3662         struct i40e_hw *hw = &pf->hw;
3663         u32 val;
3664
3665         /* set the ITR configuration */
3666         q_vector->rx.next_update = jiffies + 1;
3667         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3668         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3669         q_vector->rx.current_itr = q_vector->rx.target_itr;
3670         q_vector->tx.next_update = jiffies + 1;
3671         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3672         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3673         q_vector->tx.current_itr = q_vector->tx.target_itr;
3674
3675         i40e_enable_misc_int_causes(pf);
3676
3677         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3678         wr32(hw, I40E_PFINT_LNKLST0, 0);
3679
3680         /* Associate the queue pair to the vector and enable the queue int */
3681         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3682               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3683               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3684               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3685
3686         wr32(hw, I40E_QINT_RQCTL(0), val);
3687
3688         if (i40e_enabled_xdp_vsi(vsi)) {
3689                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3690                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3691                       (I40E_QUEUE_TYPE_TX
3692                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3693
3694                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3695         }
3696
3697         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3698               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3699               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3700
3701         wr32(hw, I40E_QINT_TQCTL(0), val);
3702         i40e_flush(hw);
3703 }
3704
3705 /**
3706  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3707  * @pf: board private structure
3708  **/
3709 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3710 {
3711         struct i40e_hw *hw = &pf->hw;
3712
3713         wr32(hw, I40E_PFINT_DYN_CTL0,
3714              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3715         i40e_flush(hw);
3716 }
3717
3718 /**
3719  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3720  * @pf: board private structure
3721  **/
3722 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3723 {
3724         struct i40e_hw *hw = &pf->hw;
3725         u32 val;
3726
3727         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3728               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3729               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3730
3731         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3732         i40e_flush(hw);
3733 }
3734
3735 /**
3736  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3737  * @irq: interrupt number
3738  * @data: pointer to a q_vector
3739  **/
3740 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3741 {
3742         struct i40e_q_vector *q_vector = data;
3743
3744         if (!q_vector->tx.ring && !q_vector->rx.ring)
3745                 return IRQ_HANDLED;
3746
3747         napi_schedule_irqoff(&q_vector->napi);
3748
3749         return IRQ_HANDLED;
3750 }
3751
3752 /**
3753  * i40e_irq_affinity_notify - Callback for affinity changes
3754  * @notify: context as to what irq was changed
3755  * @mask: the new affinity mask
3756  *
3757  * This is a callback function used by the irq_set_affinity_notifier function
3758  * so that we may register to receive changes to the irq affinity masks.
3759  **/
3760 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3761                                      const cpumask_t *mask)
3762 {
3763         struct i40e_q_vector *q_vector =
3764                 container_of(notify, struct i40e_q_vector, affinity_notify);
3765
3766         cpumask_copy(&q_vector->affinity_mask, mask);
3767 }
3768
3769 /**
3770  * i40e_irq_affinity_release - Callback for affinity notifier release
3771  * @ref: internal core kernel usage
3772  *
3773  * This is a callback function used by the irq_set_affinity_notifier function
3774  * to inform the current notification subscriber that they will no longer
3775  * receive notifications.
3776  **/
3777 static void i40e_irq_affinity_release(struct kref *ref) {}
3778
3779 /**
3780  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3781  * @vsi: the VSI being configured
3782  * @basename: name for the vector
3783  *
3784  * Allocates MSI-X vectors and requests interrupts from the kernel.
3785  **/
3786 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3787 {
3788         int q_vectors = vsi->num_q_vectors;
3789         struct i40e_pf *pf = vsi->back;
3790         int base = vsi->base_vector;
3791         int rx_int_idx = 0;
3792         int tx_int_idx = 0;
3793         int vector, err;
3794         int irq_num;
3795         int cpu;
3796
3797         for (vector = 0; vector < q_vectors; vector++) {
3798                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3799
3800                 irq_num = pf->msix_entries[base + vector].vector;
3801
3802                 if (q_vector->tx.ring && q_vector->rx.ring) {
3803                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3804                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3805                         tx_int_idx++;
3806                 } else if (q_vector->rx.ring) {
3807                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3808                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3809                 } else if (q_vector->tx.ring) {
3810                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3811                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3812                 } else {
3813                         /* skip this unused q_vector */
3814                         continue;
3815                 }
3816                 err = request_irq(irq_num,
3817                                   vsi->irq_handler,
3818                                   0,
3819                                   q_vector->name,
3820                                   q_vector);
3821                 if (err) {
3822                         dev_info(&pf->pdev->dev,
3823                                  "MSIX request_irq failed, error: %d\n", err);
3824                         goto free_queue_irqs;
3825                 }
3826
3827                 /* register for affinity change notifications */
3828                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3829                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3830                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3831                 /* Spread affinity hints out across online CPUs.
3832                  *
3833                  * get_cpu_mask returns a static constant mask with
3834                  * a permanent lifetime so it's ok to pass to
3835                  * irq_set_affinity_hint without making a copy.
3836                  */
3837                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3838                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3839         }
3840
3841         vsi->irqs_ready = true;
3842         return 0;
3843
3844 free_queue_irqs:
3845         while (vector) {
3846                 vector--;
3847                 irq_num = pf->msix_entries[base + vector].vector;
3848                 irq_set_affinity_notifier(irq_num, NULL);
3849                 irq_set_affinity_hint(irq_num, NULL);
3850                 free_irq(irq_num, &vsi->q_vectors[vector]);
3851         }
3852         return err;
3853 }
3854
3855 /**
3856  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3857  * @vsi: the VSI being un-configured
3858  **/
3859 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3860 {
3861         struct i40e_pf *pf = vsi->back;
3862         struct i40e_hw *hw = &pf->hw;
3863         int base = vsi->base_vector;
3864         int i;
3865
3866         /* disable interrupt causation from each queue */
3867         for (i = 0; i < vsi->num_queue_pairs; i++) {
3868                 u32 val;
3869
3870                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3871                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3872                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3873
3874                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3875                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3876                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3877
3878                 if (!i40e_enabled_xdp_vsi(vsi))
3879                         continue;
3880                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3881         }
3882
3883         /* disable each interrupt */
3884         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3885                 for (i = vsi->base_vector;
3886                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3887                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3888
3889                 i40e_flush(hw);
3890                 for (i = 0; i < vsi->num_q_vectors; i++)
3891                         synchronize_irq(pf->msix_entries[i + base].vector);
3892         } else {
3893                 /* Legacy and MSI mode - this stops all interrupt handling */
3894                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3895                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3896                 i40e_flush(hw);
3897                 synchronize_irq(pf->pdev->irq);
3898         }
3899 }
3900
3901 /**
3902  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3903  * @vsi: the VSI being configured
3904  **/
3905 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3906 {
3907         struct i40e_pf *pf = vsi->back;
3908         int i;
3909
3910         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3911                 for (i = 0; i < vsi->num_q_vectors; i++)
3912                         i40e_irq_dynamic_enable(vsi, i);
3913         } else {
3914                 i40e_irq_dynamic_enable_icr0(pf);
3915         }
3916
3917         i40e_flush(&pf->hw);
3918         return 0;
3919 }
3920
3921 /**
3922  * i40e_free_misc_vector - Free the vector that handles non-queue events
3923  * @pf: board private structure
3924  **/
3925 static void i40e_free_misc_vector(struct i40e_pf *pf)
3926 {
3927         /* Disable ICR 0 */
3928         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3929         i40e_flush(&pf->hw);
3930
3931         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3932                 synchronize_irq(pf->msix_entries[0].vector);
3933                 free_irq(pf->msix_entries[0].vector, pf);
3934                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3935         }
3936 }
3937
3938 /**
3939  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3940  * @irq: interrupt number
3941  * @data: pointer to a q_vector
3942  *
3943  * This is the handler used for all MSI/Legacy interrupts, and deals
3944  * with both queue and non-queue interrupts.  This is also used in
3945  * MSIX mode to handle the non-queue interrupts.
3946  **/
3947 static irqreturn_t i40e_intr(int irq, void *data)
3948 {
3949         struct i40e_pf *pf = (struct i40e_pf *)data;
3950         struct i40e_hw *hw = &pf->hw;
3951         irqreturn_t ret = IRQ_NONE;
3952         u32 icr0, icr0_remaining;
3953         u32 val, ena_mask;
3954
3955         icr0 = rd32(hw, I40E_PFINT_ICR0);
3956         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3957
3958         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3959         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3960                 goto enable_intr;
3961
3962         /* if interrupt but no bits showing, must be SWINT */
3963         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3964             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3965                 pf->sw_int_count++;
3966
3967         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3968             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3969                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3970                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3971                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3972         }
3973
3974         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3975         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3976                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3977                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3978
3979                 /* We do not have a way to disarm Queue causes while leaving
3980                  * interrupt enabled for all other causes, ideally
3981                  * interrupt should be disabled while we are in NAPI but
3982                  * this is not a performance path and napi_schedule()
3983                  * can deal with rescheduling.
3984                  */
3985                 if (!test_bit(__I40E_DOWN, pf->state))
3986                         napi_schedule_irqoff(&q_vector->napi);
3987         }
3988
3989         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3990                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3991                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3992                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3993         }
3994
3995         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3996                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3997                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3998         }
3999
4000         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4001                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4002                 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4003         }
4004
4005         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4006                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4007                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4008                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4009                 val = rd32(hw, I40E_GLGEN_RSTAT);
4010                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4011                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4012                 if (val == I40E_RESET_CORER) {
4013                         pf->corer_count++;
4014                 } else if (val == I40E_RESET_GLOBR) {
4015                         pf->globr_count++;
4016                 } else if (val == I40E_RESET_EMPR) {
4017                         pf->empr_count++;
4018                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4019                 }
4020         }
4021
4022         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4023                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4024                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4025                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4026                          rd32(hw, I40E_PFHMC_ERRORINFO),
4027                          rd32(hw, I40E_PFHMC_ERRORDATA));
4028         }
4029
4030         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4031                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4032
4033                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
4034                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4035                         i40e_ptp_tx_hwtstamp(pf);
4036                 }
4037         }
4038
4039         /* If a critical error is pending we have no choice but to reset the
4040          * device.
4041          * Report and mask out any remaining unexpected interrupts.
4042          */
4043         icr0_remaining = icr0 & ena_mask;
4044         if (icr0_remaining) {
4045                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4046                          icr0_remaining);
4047                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4048                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4049                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4050                         dev_info(&pf->pdev->dev, "device will be reset\n");
4051                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4052                         i40e_service_event_schedule(pf);
4053                 }
4054                 ena_mask &= ~icr0_remaining;
4055         }
4056         ret = IRQ_HANDLED;
4057
4058 enable_intr:
4059         /* re-enable interrupt causes */
4060         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4061         if (!test_bit(__I40E_DOWN, pf->state) ||
4062             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4063                 i40e_service_event_schedule(pf);
4064                 i40e_irq_dynamic_enable_icr0(pf);
4065         }
4066
4067         return ret;
4068 }
4069
4070 /**
4071  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4072  * @tx_ring:  tx ring to clean
4073  * @budget:   how many cleans we're allowed
4074  *
4075  * Returns true if there's any budget left (e.g. the clean is finished)
4076  **/
4077 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4078 {
4079         struct i40e_vsi *vsi = tx_ring->vsi;
4080         u16 i = tx_ring->next_to_clean;
4081         struct i40e_tx_buffer *tx_buf;
4082         struct i40e_tx_desc *tx_desc;
4083
4084         tx_buf = &tx_ring->tx_bi[i];
4085         tx_desc = I40E_TX_DESC(tx_ring, i);
4086         i -= tx_ring->count;
4087
4088         do {
4089                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4090
4091                 /* if next_to_watch is not set then there is no work pending */
4092                 if (!eop_desc)
4093                         break;
4094
4095                 /* prevent any other reads prior to eop_desc */
4096                 smp_rmb();
4097
4098                 /* if the descriptor isn't done, no work yet to do */
4099                 if (!(eop_desc->cmd_type_offset_bsz &
4100                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4101                         break;
4102
4103                 /* clear next_to_watch to prevent false hangs */
4104                 tx_buf->next_to_watch = NULL;
4105
4106                 tx_desc->buffer_addr = 0;
4107                 tx_desc->cmd_type_offset_bsz = 0;
4108                 /* move past filter desc */
4109                 tx_buf++;
4110                 tx_desc++;
4111                 i++;
4112                 if (unlikely(!i)) {
4113                         i -= tx_ring->count;
4114                         tx_buf = tx_ring->tx_bi;
4115                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4116                 }
4117                 /* unmap skb header data */
4118                 dma_unmap_single(tx_ring->dev,
4119                                  dma_unmap_addr(tx_buf, dma),
4120                                  dma_unmap_len(tx_buf, len),
4121                                  DMA_TO_DEVICE);
4122                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4123                         kfree(tx_buf->raw_buf);
4124
4125                 tx_buf->raw_buf = NULL;
4126                 tx_buf->tx_flags = 0;
4127                 tx_buf->next_to_watch = NULL;
4128                 dma_unmap_len_set(tx_buf, len, 0);
4129                 tx_desc->buffer_addr = 0;
4130                 tx_desc->cmd_type_offset_bsz = 0;
4131
4132                 /* move us past the eop_desc for start of next FD desc */
4133                 tx_buf++;
4134                 tx_desc++;
4135                 i++;
4136                 if (unlikely(!i)) {
4137                         i -= tx_ring->count;
4138                         tx_buf = tx_ring->tx_bi;
4139                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4140                 }
4141
4142                 /* update budget accounting */
4143                 budget--;
4144         } while (likely(budget));
4145
4146         i += tx_ring->count;
4147         tx_ring->next_to_clean = i;
4148
4149         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4150                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4151
4152         return budget > 0;
4153 }
4154
4155 /**
4156  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4157  * @irq: interrupt number
4158  * @data: pointer to a q_vector
4159  **/
4160 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4161 {
4162         struct i40e_q_vector *q_vector = data;
4163         struct i40e_vsi *vsi;
4164
4165         if (!q_vector->tx.ring)
4166                 return IRQ_HANDLED;
4167
4168         vsi = q_vector->tx.ring->vsi;
4169         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4170
4171         return IRQ_HANDLED;
4172 }
4173
4174 /**
4175  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4176  * @vsi: the VSI being configured
4177  * @v_idx: vector index
4178  * @qp_idx: queue pair index
4179  **/
4180 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4181 {
4182         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4183         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4184         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4185
4186         tx_ring->q_vector = q_vector;
4187         tx_ring->next = q_vector->tx.ring;
4188         q_vector->tx.ring = tx_ring;
4189         q_vector->tx.count++;
4190
4191         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4192         if (i40e_enabled_xdp_vsi(vsi)) {
4193                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4194
4195                 xdp_ring->q_vector = q_vector;
4196                 xdp_ring->next = q_vector->tx.ring;
4197                 q_vector->tx.ring = xdp_ring;
4198                 q_vector->tx.count++;
4199         }
4200
4201         rx_ring->q_vector = q_vector;
4202         rx_ring->next = q_vector->rx.ring;
4203         q_vector->rx.ring = rx_ring;
4204         q_vector->rx.count++;
4205 }
4206
4207 /**
4208  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4209  * @vsi: the VSI being configured
4210  *
4211  * This function maps descriptor rings to the queue-specific vectors
4212  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4213  * one vector per queue pair, but on a constrained vector budget, we
4214  * group the queue pairs as "efficiently" as possible.
4215  **/
4216 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4217 {
4218         int qp_remaining = vsi->num_queue_pairs;
4219         int q_vectors = vsi->num_q_vectors;
4220         int num_ringpairs;
4221         int v_start = 0;
4222         int qp_idx = 0;
4223
4224         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4225          * group them so there are multiple queues per vector.
4226          * It is also important to go through all the vectors available to be
4227          * sure that if we don't use all the vectors, that the remaining vectors
4228          * are cleared. This is especially important when decreasing the
4229          * number of queues in use.
4230          */
4231         for (; v_start < q_vectors; v_start++) {
4232                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4233
4234                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4235
4236                 q_vector->num_ringpairs = num_ringpairs;
4237                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4238
4239                 q_vector->rx.count = 0;
4240                 q_vector->tx.count = 0;
4241                 q_vector->rx.ring = NULL;
4242                 q_vector->tx.ring = NULL;
4243
4244                 while (num_ringpairs--) {
4245                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4246                         qp_idx++;
4247                         qp_remaining--;
4248                 }
4249         }
4250 }
4251
4252 /**
4253  * i40e_vsi_request_irq - Request IRQ from the OS
4254  * @vsi: the VSI being configured
4255  * @basename: name for the vector
4256  **/
4257 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4258 {
4259         struct i40e_pf *pf = vsi->back;
4260         int err;
4261
4262         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4263                 err = i40e_vsi_request_irq_msix(vsi, basename);
4264         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4265                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4266                                   pf->int_name, pf);
4267         else
4268                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4269                                   pf->int_name, pf);
4270
4271         if (err)
4272                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4273
4274         return err;
4275 }
4276
4277 #ifdef CONFIG_NET_POLL_CONTROLLER
4278 /**
4279  * i40e_netpoll - A Polling 'interrupt' handler
4280  * @netdev: network interface device structure
4281  *
4282  * This is used by netconsole to send skbs without having to re-enable
4283  * interrupts.  It's not called while the normal interrupt routine is executing.
4284  **/
4285 static void i40e_netpoll(struct net_device *netdev)
4286 {
4287         struct i40e_netdev_priv *np = netdev_priv(netdev);
4288         struct i40e_vsi *vsi = np->vsi;
4289         struct i40e_pf *pf = vsi->back;
4290         int i;
4291
4292         /* if interface is down do nothing */
4293         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4294                 return;
4295
4296         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4297                 for (i = 0; i < vsi->num_q_vectors; i++)
4298                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4299         } else {
4300                 i40e_intr(pf->pdev->irq, netdev);
4301         }
4302 }
4303 #endif
4304
4305 #define I40E_QTX_ENA_WAIT_COUNT 50
4306
4307 /**
4308  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4309  * @pf: the PF being configured
4310  * @pf_q: the PF queue
4311  * @enable: enable or disable state of the queue
4312  *
4313  * This routine will wait for the given Tx queue of the PF to reach the
4314  * enabled or disabled state.
4315  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4316  * multiple retries; else will return 0 in case of success.
4317  **/
4318 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4319 {
4320         int i;
4321         u32 tx_reg;
4322
4323         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4324                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4325                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4326                         break;
4327
4328                 usleep_range(10, 20);
4329         }
4330         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4331                 return -ETIMEDOUT;
4332
4333         return 0;
4334 }
4335
4336 /**
4337  * i40e_control_tx_q - Start or stop a particular Tx queue
4338  * @pf: the PF structure
4339  * @pf_q: the PF queue to configure
4340  * @enable: start or stop the queue
4341  *
4342  * This function enables or disables a single queue. Note that any delay
4343  * required after the operation is expected to be handled by the caller of
4344  * this function.
4345  **/
4346 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4347 {
4348         struct i40e_hw *hw = &pf->hw;
4349         u32 tx_reg;
4350         int i;
4351
4352         /* warn the TX unit of coming changes */
4353         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4354         if (!enable)
4355                 usleep_range(10, 20);
4356
4357         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4358                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4359                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4360                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4361                         break;
4362                 usleep_range(1000, 2000);
4363         }
4364
4365         /* Skip if the queue is already in the requested state */
4366         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4367                 return;
4368
4369         /* turn on/off the queue */
4370         if (enable) {
4371                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4372                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4373         } else {
4374                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4375         }
4376
4377         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4378 }
4379
4380 /**
4381  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4382  * @seid: VSI SEID
4383  * @pf: the PF structure
4384  * @pf_q: the PF queue to configure
4385  * @is_xdp: true if the queue is used for XDP
4386  * @enable: start or stop the queue
4387  **/
4388 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4389                            bool is_xdp, bool enable)
4390 {
4391         int ret;
4392
4393         i40e_control_tx_q(pf, pf_q, enable);
4394
4395         /* wait for the change to finish */
4396         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4397         if (ret) {
4398                 dev_info(&pf->pdev->dev,
4399                          "VSI seid %d %sTx ring %d %sable timeout\n",
4400                          seid, (is_xdp ? "XDP " : ""), pf_q,
4401                          (enable ? "en" : "dis"));
4402         }
4403
4404         return ret;
4405 }
4406
4407 /**
4408  * i40e_vsi_control_tx - Start or stop a VSI's rings
4409  * @vsi: the VSI being configured
4410  * @enable: start or stop the rings
4411  **/
4412 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4413 {
4414         struct i40e_pf *pf = vsi->back;
4415         int i, pf_q, ret = 0;
4416
4417         pf_q = vsi->base_queue;
4418         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4419                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4420                                              pf_q,
4421                                              false /*is xdp*/, enable);
4422                 if (ret)
4423                         break;
4424
4425                 if (!i40e_enabled_xdp_vsi(vsi))
4426                         continue;
4427
4428                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4429                                              pf_q + vsi->alloc_queue_pairs,
4430                                              true /*is xdp*/, enable);
4431                 if (ret)
4432                         break;
4433         }
4434         return ret;
4435 }
4436
4437 /**
4438  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4439  * @pf: the PF being configured
4440  * @pf_q: the PF queue
4441  * @enable: enable or disable state of the queue
4442  *
4443  * This routine will wait for the given Rx queue of the PF to reach the
4444  * enabled or disabled state.
4445  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4446  * multiple retries; else will return 0 in case of success.
4447  **/
4448 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4449 {
4450         int i;
4451         u32 rx_reg;
4452
4453         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4454                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4455                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4456                         break;
4457
4458                 usleep_range(10, 20);
4459         }
4460         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4461                 return -ETIMEDOUT;
4462
4463         return 0;
4464 }
4465
4466 /**
4467  * i40e_control_rx_q - Start or stop a particular Rx queue
4468  * @pf: the PF structure
4469  * @pf_q: the PF queue to configure
4470  * @enable: start or stop the queue
4471  *
4472  * This function enables or disables a single queue. Note that
4473  * any delay required after the operation is expected to be
4474  * handled by the caller of this function.
4475  **/
4476 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4477 {
4478         struct i40e_hw *hw = &pf->hw;
4479         u32 rx_reg;
4480         int i;
4481
4482         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4483                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4484                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4485                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4486                         break;
4487                 usleep_range(1000, 2000);
4488         }
4489
4490         /* Skip if the queue is already in the requested state */
4491         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4492                 return;
4493
4494         /* turn on/off the queue */
4495         if (enable)
4496                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4497         else
4498                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4499
4500         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4501 }
4502
4503 /**
4504  * i40e_control_wait_rx_q
4505  * @pf: the PF structure
4506  * @pf_q: queue being configured
4507  * @enable: start or stop the rings
4508  *
4509  * This function enables or disables a single queue along with waiting
4510  * for the change to finish. The caller of this function should handle
4511  * the delays needed in the case of disabling queues.
4512  **/
4513 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4514 {
4515         int ret = 0;
4516
4517         i40e_control_rx_q(pf, pf_q, enable);
4518
4519         /* wait for the change to finish */
4520         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4521         if (ret)
4522                 return ret;
4523
4524         return ret;
4525 }
4526
4527 /**
4528  * i40e_vsi_control_rx - Start or stop a VSI's rings
4529  * @vsi: the VSI being configured
4530  * @enable: start or stop the rings
4531  **/
4532 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4533 {
4534         struct i40e_pf *pf = vsi->back;
4535         int i, pf_q, ret = 0;
4536
4537         pf_q = vsi->base_queue;
4538         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4539                 ret = i40e_control_wait_rx_q(pf, pf_q, enable);
4540                 if (ret) {
4541                         dev_info(&pf->pdev->dev,
4542                                  "VSI seid %d Rx ring %d %sable timeout\n",
4543                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4544                         break;
4545                 }
4546         }
4547
4548         /* Due to HW errata, on Rx disable only, the register can indicate done
4549          * before it really is. Needs 50ms to be sure
4550          */
4551         if (!enable)
4552                 mdelay(50);
4553
4554         return ret;
4555 }
4556
4557 /**
4558  * i40e_vsi_start_rings - Start a VSI's rings
4559  * @vsi: the VSI being configured
4560  **/
4561 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4562 {
4563         int ret = 0;
4564
4565         /* do rx first for enable and last for disable */
4566         ret = i40e_vsi_control_rx(vsi, true);
4567         if (ret)
4568                 return ret;
4569         ret = i40e_vsi_control_tx(vsi, true);
4570
4571         return ret;
4572 }
4573
4574 /**
4575  * i40e_vsi_stop_rings - Stop a VSI's rings
4576  * @vsi: the VSI being configured
4577  **/
4578 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4579 {
4580         /* When port TX is suspended, don't wait */
4581         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4582                 return i40e_vsi_stop_rings_no_wait(vsi);
4583
4584         /* do rx first for enable and last for disable
4585          * Ignore return value, we need to shutdown whatever we can
4586          */
4587         i40e_vsi_control_tx(vsi, false);
4588         i40e_vsi_control_rx(vsi, false);
4589 }
4590
4591 /**
4592  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4593  * @vsi: the VSI being shutdown
4594  *
4595  * This function stops all the rings for a VSI but does not delay to verify
4596  * that rings have been disabled. It is expected that the caller is shutting
4597  * down multiple VSIs at once and will delay together for all the VSIs after
4598  * initiating the shutdown. This is particularly useful for shutting down lots
4599  * of VFs together. Otherwise, a large delay can be incurred while configuring
4600  * each VSI in serial.
4601  **/
4602 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4603 {
4604         struct i40e_pf *pf = vsi->back;
4605         int i, pf_q;
4606
4607         pf_q = vsi->base_queue;
4608         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4609                 i40e_control_tx_q(pf, pf_q, false);
4610                 i40e_control_rx_q(pf, pf_q, false);
4611         }
4612 }
4613
4614 /**
4615  * i40e_vsi_free_irq - Free the irq association with the OS
4616  * @vsi: the VSI being configured
4617  **/
4618 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4619 {
4620         struct i40e_pf *pf = vsi->back;
4621         struct i40e_hw *hw = &pf->hw;
4622         int base = vsi->base_vector;
4623         u32 val, qp;
4624         int i;
4625
4626         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4627                 if (!vsi->q_vectors)
4628                         return;
4629
4630                 if (!vsi->irqs_ready)
4631                         return;
4632
4633                 vsi->irqs_ready = false;
4634                 for (i = 0; i < vsi->num_q_vectors; i++) {
4635                         int irq_num;
4636                         u16 vector;
4637
4638                         vector = i + base;
4639                         irq_num = pf->msix_entries[vector].vector;
4640
4641                         /* free only the irqs that were actually requested */
4642                         if (!vsi->q_vectors[i] ||
4643                             !vsi->q_vectors[i]->num_ringpairs)
4644                                 continue;
4645
4646                         /* clear the affinity notifier in the IRQ descriptor */
4647                         irq_set_affinity_notifier(irq_num, NULL);
4648                         /* remove our suggested affinity mask for this IRQ */
4649                         irq_set_affinity_hint(irq_num, NULL);
4650                         synchronize_irq(irq_num);
4651                         free_irq(irq_num, vsi->q_vectors[i]);
4652
4653                         /* Tear down the interrupt queue link list
4654                          *
4655                          * We know that they come in pairs and always
4656                          * the Rx first, then the Tx.  To clear the
4657                          * link list, stick the EOL value into the
4658                          * next_q field of the registers.
4659                          */
4660                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4661                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4662                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4663                         val |= I40E_QUEUE_END_OF_LIST
4664                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4665                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4666
4667                         while (qp != I40E_QUEUE_END_OF_LIST) {
4668                                 u32 next;
4669
4670                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4671
4672                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4673                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4674                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4675                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4676
4677                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4678                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4679
4680                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4681
4682                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4683
4684                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4685                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4686
4687                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4688                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4689                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4690                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4691
4692                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4693                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4694
4695                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4696                                 qp = next;
4697                         }
4698                 }
4699         } else {
4700                 free_irq(pf->pdev->irq, pf);
4701
4702                 val = rd32(hw, I40E_PFINT_LNKLST0);
4703                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4704                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4705                 val |= I40E_QUEUE_END_OF_LIST
4706                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4707                 wr32(hw, I40E_PFINT_LNKLST0, val);
4708
4709                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4710                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4711                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4712                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4713                          I40E_QINT_RQCTL_INTEVENT_MASK);
4714
4715                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4716                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4717
4718                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4719
4720                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4721
4722                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4723                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4724                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4725                          I40E_QINT_TQCTL_INTEVENT_MASK);
4726
4727                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4728                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4729
4730                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4731         }
4732 }
4733
4734 /**
4735  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4736  * @vsi: the VSI being configured
4737  * @v_idx: Index of vector to be freed
4738  *
4739  * This function frees the memory allocated to the q_vector.  In addition if
4740  * NAPI is enabled it will delete any references to the NAPI struct prior
4741  * to freeing the q_vector.
4742  **/
4743 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4744 {
4745         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4746         struct i40e_ring *ring;
4747
4748         if (!q_vector)
4749                 return;
4750
4751         /* disassociate q_vector from rings */
4752         i40e_for_each_ring(ring, q_vector->tx)
4753                 ring->q_vector = NULL;
4754
4755         i40e_for_each_ring(ring, q_vector->rx)
4756                 ring->q_vector = NULL;
4757
4758         /* only VSI w/ an associated netdev is set up w/ NAPI */
4759         if (vsi->netdev)
4760                 netif_napi_del(&q_vector->napi);
4761
4762         vsi->q_vectors[v_idx] = NULL;
4763
4764         kfree_rcu(q_vector, rcu);
4765 }
4766
4767 /**
4768  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4769  * @vsi: the VSI being un-configured
4770  *
4771  * This frees the memory allocated to the q_vectors and
4772  * deletes references to the NAPI struct.
4773  **/
4774 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4775 {
4776         int v_idx;
4777
4778         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4779                 i40e_free_q_vector(vsi, v_idx);
4780 }
4781
4782 /**
4783  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4784  * @pf: board private structure
4785  **/
4786 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4787 {
4788         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4789         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4790                 pci_disable_msix(pf->pdev);
4791                 kfree(pf->msix_entries);
4792                 pf->msix_entries = NULL;
4793                 kfree(pf->irq_pile);
4794                 pf->irq_pile = NULL;
4795         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4796                 pci_disable_msi(pf->pdev);
4797         }
4798         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4799 }
4800
4801 /**
4802  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4803  * @pf: board private structure
4804  *
4805  * We go through and clear interrupt specific resources and reset the structure
4806  * to pre-load conditions
4807  **/
4808 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4809 {
4810         int i;
4811
4812         i40e_free_misc_vector(pf);
4813
4814         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4815                       I40E_IWARP_IRQ_PILE_ID);
4816
4817         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4818         for (i = 0; i < pf->num_alloc_vsi; i++)
4819                 if (pf->vsi[i])
4820                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4821         i40e_reset_interrupt_capability(pf);
4822 }
4823
4824 /**
4825  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4826  * @vsi: the VSI being configured
4827  **/
4828 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4829 {
4830         int q_idx;
4831
4832         if (!vsi->netdev)
4833                 return;
4834
4835         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4836                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4837
4838                 if (q_vector->rx.ring || q_vector->tx.ring)
4839                         napi_enable(&q_vector->napi);
4840         }
4841 }
4842
4843 /**
4844  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4845  * @vsi: the VSI being configured
4846  **/
4847 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4848 {
4849         int q_idx;
4850
4851         if (!vsi->netdev)
4852                 return;
4853
4854         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4855                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4856
4857                 if (q_vector->rx.ring || q_vector->tx.ring)
4858                         napi_disable(&q_vector->napi);
4859         }
4860 }
4861
4862 /**
4863  * i40e_vsi_close - Shut down a VSI
4864  * @vsi: the vsi to be quelled
4865  **/
4866 static void i40e_vsi_close(struct i40e_vsi *vsi)
4867 {
4868         struct i40e_pf *pf = vsi->back;
4869         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4870                 i40e_down(vsi);
4871         i40e_vsi_free_irq(vsi);
4872         i40e_vsi_free_tx_resources(vsi);
4873         i40e_vsi_free_rx_resources(vsi);
4874         vsi->current_netdev_flags = 0;
4875         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4876         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4877                 set_bit(__I40E_CLIENT_RESET, pf->state);
4878 }
4879
4880 /**
4881  * i40e_quiesce_vsi - Pause a given VSI
4882  * @vsi: the VSI being paused
4883  **/
4884 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4885 {
4886         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4887                 return;
4888
4889         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4890         if (vsi->netdev && netif_running(vsi->netdev))
4891                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4892         else
4893                 i40e_vsi_close(vsi);
4894 }
4895
4896 /**
4897  * i40e_unquiesce_vsi - Resume a given VSI
4898  * @vsi: the VSI being resumed
4899  **/
4900 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4901 {
4902         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4903                 return;
4904
4905         if (vsi->netdev && netif_running(vsi->netdev))
4906                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4907         else
4908                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4909 }
4910
4911 /**
4912  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4913  * @pf: the PF
4914  **/
4915 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4916 {
4917         int v;
4918
4919         for (v = 0; v < pf->num_alloc_vsi; v++) {
4920                 if (pf->vsi[v])
4921                         i40e_quiesce_vsi(pf->vsi[v]);
4922         }
4923 }
4924
4925 /**
4926  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4927  * @pf: the PF
4928  **/
4929 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4930 {
4931         int v;
4932
4933         for (v = 0; v < pf->num_alloc_vsi; v++) {
4934                 if (pf->vsi[v])
4935                         i40e_unquiesce_vsi(pf->vsi[v]);
4936         }
4937 }
4938
4939 /**
4940  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4941  * @vsi: the VSI being configured
4942  *
4943  * Wait until all queues on a given VSI have been disabled.
4944  **/
4945 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4946 {
4947         struct i40e_pf *pf = vsi->back;
4948         int i, pf_q, ret;
4949
4950         pf_q = vsi->base_queue;
4951         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4952                 /* Check and wait for the Tx queue */
4953                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4954                 if (ret) {
4955                         dev_info(&pf->pdev->dev,
4956                                  "VSI seid %d Tx ring %d disable timeout\n",
4957                                  vsi->seid, pf_q);
4958                         return ret;
4959                 }
4960
4961                 if (!i40e_enabled_xdp_vsi(vsi))
4962                         goto wait_rx;
4963
4964                 /* Check and wait for the XDP Tx queue */
4965                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4966                                        false);
4967                 if (ret) {
4968                         dev_info(&pf->pdev->dev,
4969                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4970                                  vsi->seid, pf_q);
4971                         return ret;
4972                 }
4973 wait_rx:
4974                 /* Check and wait for the Rx queue */
4975                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4976                 if (ret) {
4977                         dev_info(&pf->pdev->dev,
4978                                  "VSI seid %d Rx ring %d disable timeout\n",
4979                                  vsi->seid, pf_q);
4980                         return ret;
4981                 }
4982         }
4983
4984         return 0;
4985 }
4986
4987 #ifdef CONFIG_I40E_DCB
4988 /**
4989  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4990  * @pf: the PF
4991  *
4992  * This function waits for the queues to be in disabled state for all the
4993  * VSIs that are managed by this PF.
4994  **/
4995 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4996 {
4997         int v, ret = 0;
4998
4999         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5000                 if (pf->vsi[v]) {
5001                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5002                         if (ret)
5003                                 break;
5004                 }
5005         }
5006
5007         return ret;
5008 }
5009
5010 #endif
5011
5012 /**
5013  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5014  * @pf: pointer to PF
5015  *
5016  * Get TC map for ISCSI PF type that will include iSCSI TC
5017  * and LAN TC.
5018  **/
5019 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5020 {
5021         struct i40e_dcb_app_priority_table app;
5022         struct i40e_hw *hw = &pf->hw;
5023         u8 enabled_tc = 1; /* TC0 is always enabled */
5024         u8 tc, i;
5025         /* Get the iSCSI APP TLV */
5026         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5027
5028         for (i = 0; i < dcbcfg->numapps; i++) {
5029                 app = dcbcfg->app[i];
5030                 if (app.selector == I40E_APP_SEL_TCPIP &&
5031                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5032                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5033                         enabled_tc |= BIT(tc);
5034                         break;
5035                 }
5036         }
5037
5038         return enabled_tc;
5039 }
5040
5041 /**
5042  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5043  * @dcbcfg: the corresponding DCBx configuration structure
5044  *
5045  * Return the number of TCs from given DCBx configuration
5046  **/
5047 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5048 {
5049         int i, tc_unused = 0;
5050         u8 num_tc = 0;
5051         u8 ret = 0;
5052
5053         /* Scan the ETS Config Priority Table to find
5054          * traffic class enabled for a given priority
5055          * and create a bitmask of enabled TCs
5056          */
5057         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5058                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5059
5060         /* Now scan the bitmask to check for
5061          * contiguous TCs starting with TC0
5062          */
5063         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5064                 if (num_tc & BIT(i)) {
5065                         if (!tc_unused) {
5066                                 ret++;
5067                         } else {
5068                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5069                                 return 1;
5070                         }
5071                 } else {
5072                         tc_unused = 1;
5073                 }
5074         }
5075
5076         /* There is always at least TC0 */
5077         if (!ret)
5078                 ret = 1;
5079
5080         return ret;
5081 }
5082
5083 /**
5084  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5085  * @dcbcfg: the corresponding DCBx configuration structure
5086  *
5087  * Query the current DCB configuration and return the number of
5088  * traffic classes enabled from the given DCBX config
5089  **/
5090 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5091 {
5092         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5093         u8 enabled_tc = 1;
5094         u8 i;
5095
5096         for (i = 0; i < num_tc; i++)
5097                 enabled_tc |= BIT(i);
5098
5099         return enabled_tc;
5100 }
5101
5102 /**
5103  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5104  * @pf: PF being queried
5105  *
5106  * Query the current MQPRIO configuration and return the number of
5107  * traffic classes enabled.
5108  **/
5109 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5110 {
5111         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5112         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5113         u8 enabled_tc = 1, i;
5114
5115         for (i = 1; i < num_tc; i++)
5116                 enabled_tc |= BIT(i);
5117         return enabled_tc;
5118 }
5119
5120 /**
5121  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5122  * @pf: PF being queried
5123  *
5124  * Return number of traffic classes enabled for the given PF
5125  **/
5126 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5127 {
5128         struct i40e_hw *hw = &pf->hw;
5129         u8 i, enabled_tc = 1;
5130         u8 num_tc = 0;
5131         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5132
5133         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5134                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5135
5136         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5137         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5138                 return 1;
5139
5140         /* SFP mode will be enabled for all TCs on port */
5141         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5142                 return i40e_dcb_get_num_tc(dcbcfg);
5143
5144         /* MFP mode return count of enabled TCs for this PF */
5145         if (pf->hw.func_caps.iscsi)
5146                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5147         else
5148                 return 1; /* Only TC0 */
5149
5150         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5151                 if (enabled_tc & BIT(i))
5152                         num_tc++;
5153         }
5154         return num_tc;
5155 }
5156
5157 /**
5158  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
5159  * @pf: PF being queried
5160  *
5161  * Return a bitmap for enabled traffic classes for this PF.
5162  **/
5163 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5164 {
5165         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5166                 return i40e_mqprio_get_enabled_tc(pf);
5167
5168         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5169          * default TC
5170          */
5171         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5172                 return I40E_DEFAULT_TRAFFIC_CLASS;
5173
5174         /* SFP mode we want PF to be enabled for all TCs */
5175         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5176                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5177
5178         /* MFP enabled and iSCSI PF type */
5179         if (pf->hw.func_caps.iscsi)
5180                 return i40e_get_iscsi_tc_map(pf);
5181         else
5182                 return I40E_DEFAULT_TRAFFIC_CLASS;
5183 }
5184
5185 /**
5186  * i40e_vsi_get_bw_info - Query VSI BW Information
5187  * @vsi: the VSI being queried
5188  *
5189  * Returns 0 on success, negative value on failure
5190  **/
5191 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5192 {
5193         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5194         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5195         struct i40e_pf *pf = vsi->back;
5196         struct i40e_hw *hw = &pf->hw;
5197         i40e_status ret;
5198         u32 tc_bw_max;
5199         int i;
5200
5201         /* Get the VSI level BW configuration */
5202         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5203         if (ret) {
5204                 dev_info(&pf->pdev->dev,
5205                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5206                          i40e_stat_str(&pf->hw, ret),
5207                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5208                 return -EINVAL;
5209         }
5210
5211         /* Get the VSI level BW configuration per TC */
5212         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5213                                                NULL);
5214         if (ret) {
5215                 dev_info(&pf->pdev->dev,
5216                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5217                          i40e_stat_str(&pf->hw, ret),
5218                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5219                 return -EINVAL;
5220         }
5221
5222         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5223                 dev_info(&pf->pdev->dev,
5224                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5225                          bw_config.tc_valid_bits,
5226                          bw_ets_config.tc_valid_bits);
5227                 /* Still continuing */
5228         }
5229
5230         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5231         vsi->bw_max_quanta = bw_config.max_bw;
5232         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5233                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5234         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5235                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5236                 vsi->bw_ets_limit_credits[i] =
5237                                         le16_to_cpu(bw_ets_config.credits[i]);
5238                 /* 3 bits out of 4 for each TC */
5239                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5240         }
5241
5242         return 0;
5243 }
5244
5245 /**
5246  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5247  * @vsi: the VSI being configured
5248  * @enabled_tc: TC bitmap
5249  * @bw_share: BW shared credits per TC
5250  *
5251  * Returns 0 on success, negative value on failure
5252  **/
5253 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5254                                        u8 *bw_share)
5255 {
5256         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5257         struct i40e_pf *pf = vsi->back;
5258         i40e_status ret;
5259         int i;
5260
5261         /* There is no need to reset BW when mqprio mode is on.  */
5262         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5263                 return 0;
5264         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5265                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5266                 if (ret)
5267                         dev_info(&pf->pdev->dev,
5268                                  "Failed to reset tx rate for vsi->seid %u\n",
5269                                  vsi->seid);
5270                 return ret;
5271         }
5272         bw_data.tc_valid_bits = enabled_tc;
5273         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5274                 bw_data.tc_bw_credits[i] = bw_share[i];
5275
5276         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5277         if (ret) {
5278                 dev_info(&pf->pdev->dev,
5279                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5280                          pf->hw.aq.asq_last_status);
5281                 return -EINVAL;
5282         }
5283
5284         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5285                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5286
5287         return 0;
5288 }
5289
5290 /**
5291  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5292  * @vsi: the VSI being configured
5293  * @enabled_tc: TC map to be enabled
5294  *
5295  **/
5296 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5297 {
5298         struct net_device *netdev = vsi->netdev;
5299         struct i40e_pf *pf = vsi->back;
5300         struct i40e_hw *hw = &pf->hw;
5301         u8 netdev_tc = 0;
5302         int i;
5303         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5304
5305         if (!netdev)
5306                 return;
5307
5308         if (!enabled_tc) {
5309                 netdev_reset_tc(netdev);
5310                 return;
5311         }
5312
5313         /* Set up actual enabled TCs on the VSI */
5314         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5315                 return;
5316
5317         /* set per TC queues for the VSI */
5318         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5319                 /* Only set TC queues for enabled tcs
5320                  *
5321                  * e.g. For a VSI that has TC0 and TC3 enabled the
5322                  * enabled_tc bitmap would be 0x00001001; the driver
5323                  * will set the numtc for netdev as 2 that will be
5324                  * referenced by the netdev layer as TC 0 and 1.
5325                  */
5326                 if (vsi->tc_config.enabled_tc & BIT(i))
5327                         netdev_set_tc_queue(netdev,
5328                                         vsi->tc_config.tc_info[i].netdev_tc,
5329                                         vsi->tc_config.tc_info[i].qcount,
5330                                         vsi->tc_config.tc_info[i].qoffset);
5331         }
5332
5333         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5334                 return;
5335
5336         /* Assign UP2TC map for the VSI */
5337         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5338                 /* Get the actual TC# for the UP */
5339                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5340                 /* Get the mapped netdev TC# for the UP */
5341                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5342                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5343         }
5344 }
5345
5346 /**
5347  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5348  * @vsi: the VSI being configured
5349  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5350  **/
5351 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5352                                       struct i40e_vsi_context *ctxt)
5353 {
5354         /* copy just the sections touched not the entire info
5355          * since not all sections are valid as returned by
5356          * update vsi params
5357          */
5358         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5359         memcpy(&vsi->info.queue_mapping,
5360                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5361         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5362                sizeof(vsi->info.tc_mapping));
5363 }
5364
5365 /**
5366  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5367  * @vsi: VSI to be configured
5368  * @enabled_tc: TC bitmap
5369  *
5370  * This configures a particular VSI for TCs that are mapped to the
5371  * given TC bitmap. It uses default bandwidth share for TCs across
5372  * VSIs to configure TC for a particular VSI.
5373  *
5374  * NOTE:
5375  * It is expected that the VSI queues have been quisced before calling
5376  * this function.
5377  **/
5378 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5379 {
5380         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5381         struct i40e_pf *pf = vsi->back;
5382         struct i40e_hw *hw = &pf->hw;
5383         struct i40e_vsi_context ctxt;
5384         int ret = 0;
5385         int i;
5386
5387         /* Check if enabled_tc is same as existing or new TCs */
5388         if (vsi->tc_config.enabled_tc == enabled_tc &&
5389             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5390                 return ret;
5391
5392         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5393         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5394                 if (enabled_tc & BIT(i))
5395                         bw_share[i] = 1;
5396         }
5397
5398         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5399         if (ret) {
5400                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5401
5402                 dev_info(&pf->pdev->dev,
5403                          "Failed configuring TC map %d for VSI %d\n",
5404                          enabled_tc, vsi->seid);
5405                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5406                                                   &bw_config, NULL);
5407                 if (ret) {
5408                         dev_info(&pf->pdev->dev,
5409                                  "Failed querying vsi bw info, err %s aq_err %s\n",
5410                                  i40e_stat_str(hw, ret),
5411                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5412                         goto out;
5413                 }
5414                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5415                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5416
5417                         if (!valid_tc)
5418                                 valid_tc = bw_config.tc_valid_bits;
5419                         /* Always enable TC0, no matter what */
5420                         valid_tc |= 1;
5421                         dev_info(&pf->pdev->dev,
5422                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5423                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5424                         enabled_tc = valid_tc;
5425                 }
5426
5427                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5428                 if (ret) {
5429                         dev_err(&pf->pdev->dev,
5430                                 "Unable to  configure TC map %d for VSI %d\n",
5431                                 enabled_tc, vsi->seid);
5432                         goto out;
5433                 }
5434         }
5435
5436         /* Update Queue Pairs Mapping for currently enabled UPs */
5437         ctxt.seid = vsi->seid;
5438         ctxt.pf_num = vsi->back->hw.pf_id;
5439         ctxt.vf_num = 0;
5440         ctxt.uplink_seid = vsi->uplink_seid;
5441         ctxt.info = vsi->info;
5442         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5443                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5444                 if (ret)
5445                         goto out;
5446         } else {
5447                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5448         }
5449
5450         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5451          * queues changed.
5452          */
5453         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5454                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5455                                       vsi->num_queue_pairs);
5456                 ret = i40e_vsi_config_rss(vsi);
5457                 if (ret) {
5458                         dev_info(&vsi->back->pdev->dev,
5459                                  "Failed to reconfig rss for num_queues\n");
5460                         return ret;
5461                 }
5462                 vsi->reconfig_rss = false;
5463         }
5464         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5465                 ctxt.info.valid_sections |=
5466                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5467                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5468         }
5469
5470         /* Update the VSI after updating the VSI queue-mapping
5471          * information
5472          */
5473         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5474         if (ret) {
5475                 dev_info(&pf->pdev->dev,
5476                          "Update vsi tc config failed, err %s aq_err %s\n",
5477                          i40e_stat_str(hw, ret),
5478                          i40e_aq_str(hw, hw->aq.asq_last_status));
5479                 goto out;
5480         }
5481         /* update the local VSI info with updated queue map */
5482         i40e_vsi_update_queue_map(vsi, &ctxt);
5483         vsi->info.valid_sections = 0;
5484
5485         /* Update current VSI BW information */
5486         ret = i40e_vsi_get_bw_info(vsi);
5487         if (ret) {
5488                 dev_info(&pf->pdev->dev,
5489                          "Failed updating vsi bw info, err %s aq_err %s\n",
5490                          i40e_stat_str(hw, ret),
5491                          i40e_aq_str(hw, hw->aq.asq_last_status));
5492                 goto out;
5493         }
5494
5495         /* Update the netdev TC setup */
5496         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5497 out:
5498         return ret;
5499 }
5500
5501 /**
5502  * i40e_get_link_speed - Returns link speed for the interface
5503  * @vsi: VSI to be configured
5504  *
5505  **/
5506 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5507 {
5508         struct i40e_pf *pf = vsi->back;
5509
5510         switch (pf->hw.phy.link_info.link_speed) {
5511         case I40E_LINK_SPEED_40GB:
5512                 return 40000;
5513         case I40E_LINK_SPEED_25GB:
5514                 return 25000;
5515         case I40E_LINK_SPEED_20GB:
5516                 return 20000;
5517         case I40E_LINK_SPEED_10GB:
5518                 return 10000;
5519         case I40E_LINK_SPEED_1GB:
5520                 return 1000;
5521         default:
5522                 return -EINVAL;
5523         }
5524 }
5525
5526 /**
5527  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5528  * @vsi: VSI to be configured
5529  * @seid: seid of the channel/VSI
5530  * @max_tx_rate: max TX rate to be configured as BW limit
5531  *
5532  * Helper function to set BW limit for a given VSI
5533  **/
5534 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5535 {
5536         struct i40e_pf *pf = vsi->back;
5537         u64 credits = 0;
5538         int speed = 0;
5539         int ret = 0;
5540
5541         speed = i40e_get_link_speed(vsi);
5542         if (max_tx_rate > speed) {
5543                 dev_err(&pf->pdev->dev,
5544                         "Invalid max tx rate %llu specified for VSI seid %d.",
5545                         max_tx_rate, seid);
5546                 return -EINVAL;
5547         }
5548         if (max_tx_rate && max_tx_rate < 50) {
5549                 dev_warn(&pf->pdev->dev,
5550                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5551                 max_tx_rate = 50;
5552         }
5553
5554         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5555         credits = max_tx_rate;
5556         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5557         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5558                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5559         if (ret)
5560                 dev_err(&pf->pdev->dev,
5561                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5562                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5563                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5564         return ret;
5565 }
5566
5567 /**
5568  * i40e_remove_queue_channels - Remove queue channels for the TCs
5569  * @vsi: VSI to be configured
5570  *
5571  * Remove queue channels for the TCs
5572  **/
5573 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5574 {
5575         enum i40e_admin_queue_err last_aq_status;
5576         struct i40e_cloud_filter *cfilter;
5577         struct i40e_channel *ch, *ch_tmp;
5578         struct i40e_pf *pf = vsi->back;
5579         struct hlist_node *node;
5580         int ret, i;
5581
5582         /* Reset rss size that was stored when reconfiguring rss for
5583          * channel VSIs with non-power-of-2 queue count.
5584          */
5585         vsi->current_rss_size = 0;
5586
5587         /* perform cleanup for channels if they exist */
5588         if (list_empty(&vsi->ch_list))
5589                 return;
5590
5591         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5592                 struct i40e_vsi *p_vsi;
5593
5594                 list_del(&ch->list);
5595                 p_vsi = ch->parent_vsi;
5596                 if (!p_vsi || !ch->initialized) {
5597                         kfree(ch);
5598                         continue;
5599                 }
5600                 /* Reset queue contexts */
5601                 for (i = 0; i < ch->num_queue_pairs; i++) {
5602                         struct i40e_ring *tx_ring, *rx_ring;
5603                         u16 pf_q;
5604
5605                         pf_q = ch->base_queue + i;
5606                         tx_ring = vsi->tx_rings[pf_q];
5607                         tx_ring->ch = NULL;
5608
5609                         rx_ring = vsi->rx_rings[pf_q];
5610                         rx_ring->ch = NULL;
5611                 }
5612
5613                 /* Reset BW configured for this VSI via mqprio */
5614                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5615                 if (ret)
5616                         dev_info(&vsi->back->pdev->dev,
5617                                  "Failed to reset tx rate for ch->seid %u\n",
5618                                  ch->seid);
5619
5620                 /* delete cloud filters associated with this channel */
5621                 hlist_for_each_entry_safe(cfilter, node,
5622                                           &pf->cloud_filter_list, cloud_node) {
5623                         if (cfilter->seid != ch->seid)
5624                                 continue;
5625
5626                         hash_del(&cfilter->cloud_node);
5627                         if (cfilter->dst_port)
5628                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5629                                                                         cfilter,
5630                                                                         false);
5631                         else
5632                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5633                                                                 false);
5634                         last_aq_status = pf->hw.aq.asq_last_status;
5635                         if (ret)
5636                                 dev_info(&pf->pdev->dev,
5637                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5638                                          i40e_stat_str(&pf->hw, ret),
5639                                          i40e_aq_str(&pf->hw, last_aq_status));
5640                         kfree(cfilter);
5641                 }
5642
5643                 /* delete VSI from FW */
5644                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5645                                              NULL);
5646                 if (ret)
5647                         dev_err(&vsi->back->pdev->dev,
5648                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5649                                 ch->seid, p_vsi->seid);
5650                 kfree(ch);
5651         }
5652         INIT_LIST_HEAD(&vsi->ch_list);
5653 }
5654
5655 /**
5656  * i40e_is_any_channel - channel exist or not
5657  * @vsi: ptr to VSI to which channels are associated with
5658  *
5659  * Returns true or false if channel(s) exist for associated VSI or not
5660  **/
5661 static bool i40e_is_any_channel(struct i40e_vsi *vsi)
5662 {
5663         struct i40e_channel *ch, *ch_tmp;
5664
5665         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5666                 if (ch->initialized)
5667                         return true;
5668         }
5669
5670         return false;
5671 }
5672
5673 /**
5674  * i40e_get_max_queues_for_channel
5675  * @vsi: ptr to VSI to which channels are associated with
5676  *
5677  * Helper function which returns max value among the queue counts set on the
5678  * channels/TCs created.
5679  **/
5680 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5681 {
5682         struct i40e_channel *ch, *ch_tmp;
5683         int max = 0;
5684
5685         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5686                 if (!ch->initialized)
5687                         continue;
5688                 if (ch->num_queue_pairs > max)
5689                         max = ch->num_queue_pairs;
5690         }
5691
5692         return max;
5693 }
5694
5695 /**
5696  * i40e_validate_num_queues - validate num_queues w.r.t channel
5697  * @pf: ptr to PF device
5698  * @num_queues: number of queues
5699  * @vsi: the parent VSI
5700  * @reconfig_rss: indicates should the RSS be reconfigured or not
5701  *
5702  * This function validates number of queues in the context of new channel
5703  * which is being established and determines if RSS should be reconfigured
5704  * or not for parent VSI.
5705  **/
5706 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5707                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5708 {
5709         int max_ch_queues;
5710
5711         if (!reconfig_rss)
5712                 return -EINVAL;
5713
5714         *reconfig_rss = false;
5715         if (vsi->current_rss_size) {
5716                 if (num_queues > vsi->current_rss_size) {
5717                         dev_dbg(&pf->pdev->dev,
5718                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5719                                 num_queues, vsi->current_rss_size);
5720                         return -EINVAL;
5721                 } else if ((num_queues < vsi->current_rss_size) &&
5722                            (!is_power_of_2(num_queues))) {
5723                         dev_dbg(&pf->pdev->dev,
5724                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5725                                 num_queues, vsi->current_rss_size);
5726                         return -EINVAL;
5727                 }
5728         }
5729
5730         if (!is_power_of_2(num_queues)) {
5731                 /* Find the max num_queues configured for channel if channel
5732                  * exist.
5733                  * if channel exist, then enforce 'num_queues' to be more than
5734                  * max ever queues configured for channel.
5735                  */
5736                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5737                 if (num_queues < max_ch_queues) {
5738                         dev_dbg(&pf->pdev->dev,
5739                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5740                                 num_queues, max_ch_queues);
5741                         return -EINVAL;
5742                 }
5743                 *reconfig_rss = true;
5744         }
5745
5746         return 0;
5747 }
5748
5749 /**
5750  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5751  * @vsi: the VSI being setup
5752  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5753  *
5754  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5755  **/
5756 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5757 {
5758         struct i40e_pf *pf = vsi->back;
5759         u8 seed[I40E_HKEY_ARRAY_SIZE];
5760         struct i40e_hw *hw = &pf->hw;
5761         int local_rss_size;
5762         u8 *lut;
5763         int ret;
5764
5765         if (!vsi->rss_size)
5766                 return -EINVAL;
5767
5768         if (rss_size > vsi->rss_size)
5769                 return -EINVAL;
5770
5771         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5772         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5773         if (!lut)
5774                 return -ENOMEM;
5775
5776         /* Ignoring user configured lut if there is one */
5777         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5778
5779         /* Use user configured hash key if there is one, otherwise
5780          * use default.
5781          */
5782         if (vsi->rss_hkey_user)
5783                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5784         else
5785                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5786
5787         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5788         if (ret) {
5789                 dev_info(&pf->pdev->dev,
5790                          "Cannot set RSS lut, err %s aq_err %s\n",
5791                          i40e_stat_str(hw, ret),
5792                          i40e_aq_str(hw, hw->aq.asq_last_status));
5793                 kfree(lut);
5794                 return ret;
5795         }
5796         kfree(lut);
5797
5798         /* Do the update w.r.t. storing rss_size */
5799         if (!vsi->orig_rss_size)
5800                 vsi->orig_rss_size = vsi->rss_size;
5801         vsi->current_rss_size = local_rss_size;
5802
5803         return ret;
5804 }
5805
5806 /**
5807  * i40e_channel_setup_queue_map - Setup a channel queue map
5808  * @pf: ptr to PF device
5809  * @vsi: the VSI being setup
5810  * @ctxt: VSI context structure
5811  * @ch: ptr to channel structure
5812  *
5813  * Setup queue map for a specific channel
5814  **/
5815 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5816                                          struct i40e_vsi_context *ctxt,
5817                                          struct i40e_channel *ch)
5818 {
5819         u16 qcount, qmap, sections = 0;
5820         u8 offset = 0;
5821         int pow;
5822
5823         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5824         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5825
5826         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5827         ch->num_queue_pairs = qcount;
5828
5829         /* find the next higher power-of-2 of num queue pairs */
5830         pow = ilog2(qcount);
5831         if (!is_power_of_2(qcount))
5832                 pow++;
5833
5834         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5835                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5836
5837         /* Setup queue TC[0].qmap for given VSI context */
5838         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5839
5840         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5841         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5842         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5843         ctxt->info.valid_sections |= cpu_to_le16(sections);
5844 }
5845
5846 /**
5847  * i40e_add_channel - add a channel by adding VSI
5848  * @pf: ptr to PF device
5849  * @uplink_seid: underlying HW switching element (VEB) ID
5850  * @ch: ptr to channel structure
5851  *
5852  * Add a channel (VSI) using add_vsi and queue_map
5853  **/
5854 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5855                             struct i40e_channel *ch)
5856 {
5857         struct i40e_hw *hw = &pf->hw;
5858         struct i40e_vsi_context ctxt;
5859         u8 enabled_tc = 0x1; /* TC0 enabled */
5860         int ret;
5861
5862         if (ch->type != I40E_VSI_VMDQ2) {
5863                 dev_info(&pf->pdev->dev,
5864                          "add new vsi failed, ch->type %d\n", ch->type);
5865                 return -EINVAL;
5866         }
5867
5868         memset(&ctxt, 0, sizeof(ctxt));
5869         ctxt.pf_num = hw->pf_id;
5870         ctxt.vf_num = 0;
5871         ctxt.uplink_seid = uplink_seid;
5872         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5873         if (ch->type == I40E_VSI_VMDQ2)
5874                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5875
5876         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5877                 ctxt.info.valid_sections |=
5878                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5879                 ctxt.info.switch_id =
5880                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5881         }
5882
5883         /* Set queue map for a given VSI context */
5884         i40e_channel_setup_queue_map(pf, &ctxt, ch);
5885
5886         /* Now time to create VSI */
5887         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5888         if (ret) {
5889                 dev_info(&pf->pdev->dev,
5890                          "add new vsi failed, err %s aq_err %s\n",
5891                          i40e_stat_str(&pf->hw, ret),
5892                          i40e_aq_str(&pf->hw,
5893                                      pf->hw.aq.asq_last_status));
5894                 return -ENOENT;
5895         }
5896
5897         /* Success, update channel, set enabled_tc only if the channel
5898          * is not a macvlan
5899          */
5900         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
5901         ch->seid = ctxt.seid;
5902         ch->vsi_number = ctxt.vsi_number;
5903         ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
5904
5905         /* copy just the sections touched not the entire info
5906          * since not all sections are valid as returned by
5907          * update vsi params
5908          */
5909         ch->info.mapping_flags = ctxt.info.mapping_flags;
5910         memcpy(&ch->info.queue_mapping,
5911                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
5912         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
5913                sizeof(ctxt.info.tc_mapping));
5914
5915         return 0;
5916 }
5917
5918 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
5919                                   u8 *bw_share)
5920 {
5921         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5922         i40e_status ret;
5923         int i;
5924
5925         bw_data.tc_valid_bits = ch->enabled_tc;
5926         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5927                 bw_data.tc_bw_credits[i] = bw_share[i];
5928
5929         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
5930                                        &bw_data, NULL);
5931         if (ret) {
5932                 dev_info(&vsi->back->pdev->dev,
5933                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
5934                          vsi->back->hw.aq.asq_last_status, ch->seid);
5935                 return -EINVAL;
5936         }
5937
5938         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5939                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
5940
5941         return 0;
5942 }
5943
5944 /**
5945  * i40e_channel_config_tx_ring - config TX ring associated with new channel
5946  * @pf: ptr to PF device
5947  * @vsi: the VSI being setup
5948  * @ch: ptr to channel structure
5949  *
5950  * Configure TX rings associated with channel (VSI) since queues are being
5951  * from parent VSI.
5952  **/
5953 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
5954                                        struct i40e_vsi *vsi,
5955                                        struct i40e_channel *ch)
5956 {
5957         i40e_status ret;
5958         int i;
5959         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5960
5961         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5962         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5963                 if (ch->enabled_tc & BIT(i))
5964                         bw_share[i] = 1;
5965         }
5966
5967         /* configure BW for new VSI */
5968         ret = i40e_channel_config_bw(vsi, ch, bw_share);
5969         if (ret) {
5970                 dev_info(&vsi->back->pdev->dev,
5971                          "Failed configuring TC map %d for channel (seid %u)\n",
5972                          ch->enabled_tc, ch->seid);
5973                 return ret;
5974         }
5975
5976         for (i = 0; i < ch->num_queue_pairs; i++) {
5977                 struct i40e_ring *tx_ring, *rx_ring;
5978                 u16 pf_q;
5979
5980                 pf_q = ch->base_queue + i;
5981
5982                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
5983                  * context
5984                  */
5985                 tx_ring = vsi->tx_rings[pf_q];
5986                 tx_ring->ch = ch;
5987
5988                 /* Get the RX ring ptr */
5989                 rx_ring = vsi->rx_rings[pf_q];
5990                 rx_ring->ch = ch;
5991         }
5992
5993         return 0;
5994 }
5995
5996 /**
5997  * i40e_setup_hw_channel - setup new channel
5998  * @pf: ptr to PF device
5999  * @vsi: the VSI being setup
6000  * @ch: ptr to channel structure
6001  * @uplink_seid: underlying HW switching element (VEB) ID
6002  * @type: type of channel to be created (VMDq2/VF)
6003  *
6004  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6005  * and configures TX rings accordingly
6006  **/
6007 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6008                                         struct i40e_vsi *vsi,
6009                                         struct i40e_channel *ch,
6010                                         u16 uplink_seid, u8 type)
6011 {
6012         int ret;
6013
6014         ch->initialized = false;
6015         ch->base_queue = vsi->next_base_queue;
6016         ch->type = type;
6017
6018         /* Proceed with creation of channel (VMDq2) VSI */
6019         ret = i40e_add_channel(pf, uplink_seid, ch);
6020         if (ret) {
6021                 dev_info(&pf->pdev->dev,
6022                          "failed to add_channel using uplink_seid %u\n",
6023                          uplink_seid);
6024                 return ret;
6025         }
6026
6027         /* Mark the successful creation of channel */
6028         ch->initialized = true;
6029
6030         /* Reconfigure TX queues using QTX_CTL register */
6031         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6032         if (ret) {
6033                 dev_info(&pf->pdev->dev,
6034                          "failed to configure TX rings for channel %u\n",
6035                          ch->seid);
6036                 return ret;
6037         }
6038
6039         /* update 'next_base_queue' */
6040         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6041         dev_dbg(&pf->pdev->dev,
6042                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6043                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6044                 ch->num_queue_pairs,
6045                 vsi->next_base_queue);
6046         return ret;
6047 }
6048
6049 /**
6050  * i40e_setup_channel - setup new channel using uplink element
6051  * @pf: ptr to PF device
6052  * @type: type of channel to be created (VMDq2/VF)
6053  * @uplink_seid: underlying HW switching element (VEB) ID
6054  * @ch: ptr to channel structure
6055  *
6056  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6057  * and uplink switching element (uplink_seid)
6058  **/
6059 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6060                                struct i40e_channel *ch)
6061 {
6062         u8 vsi_type;
6063         u16 seid;
6064         int ret;
6065
6066         if (vsi->type == I40E_VSI_MAIN) {
6067                 vsi_type = I40E_VSI_VMDQ2;
6068         } else {
6069                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6070                         vsi->type);
6071                 return false;
6072         }
6073
6074         /* underlying switching element */
6075         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6076
6077         /* create channel (VSI), configure TX rings */
6078         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6079         if (ret) {
6080                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6081                 return false;
6082         }
6083
6084         return ch->initialized ? true : false;
6085 }
6086
6087 /**
6088  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6089  * @vsi: ptr to VSI which has PF backing
6090  *
6091  * Sets up switch mode correctly if it needs to be changed and perform
6092  * what are allowed modes.
6093  **/
6094 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6095 {
6096         u8 mode;
6097         struct i40e_pf *pf = vsi->back;
6098         struct i40e_hw *hw = &pf->hw;
6099         int ret;
6100
6101         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6102         if (ret)
6103                 return -EINVAL;
6104
6105         if (hw->dev_caps.switch_mode) {
6106                 /* if switch mode is set, support mode2 (non-tunneled for
6107                  * cloud filter) for now
6108                  */
6109                 u32 switch_mode = hw->dev_caps.switch_mode &
6110                                   I40E_SWITCH_MODE_MASK;
6111                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6112                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6113                                 return 0;
6114                         dev_err(&pf->pdev->dev,
6115                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6116                                 hw->dev_caps.switch_mode);
6117                         return -EINVAL;
6118                 }
6119         }
6120
6121         /* Set Bit 7 to be valid */
6122         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6123
6124         /* Set L4type for TCP support */
6125         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6126
6127         /* Set cloud filter mode */
6128         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6129
6130         /* Prep mode field for set_switch_config */
6131         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6132                                         pf->last_sw_conf_valid_flags,
6133                                         mode, NULL);
6134         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6135                 dev_err(&pf->pdev->dev,
6136                         "couldn't set switch config bits, err %s aq_err %s\n",
6137                         i40e_stat_str(hw, ret),
6138                         i40e_aq_str(hw,
6139                                     hw->aq.asq_last_status));
6140
6141         return ret;
6142 }
6143
6144 /**
6145  * i40e_create_queue_channel - function to create channel
6146  * @vsi: VSI to be configured
6147  * @ch: ptr to channel (it contains channel specific params)
6148  *
6149  * This function creates channel (VSI) using num_queues specified by user,
6150  * reconfigs RSS if needed.
6151  **/
6152 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6153                               struct i40e_channel *ch)
6154 {
6155         struct i40e_pf *pf = vsi->back;
6156         bool reconfig_rss;
6157         int err;
6158
6159         if (!ch)
6160                 return -EINVAL;
6161
6162         if (!ch->num_queue_pairs) {
6163                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6164                         ch->num_queue_pairs);
6165                 return -EINVAL;
6166         }
6167
6168         /* validate user requested num_queues for channel */
6169         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6170                                        &reconfig_rss);
6171         if (err) {
6172                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6173                          ch->num_queue_pairs);
6174                 return -EINVAL;
6175         }
6176
6177         /* By default we are in VEPA mode, if this is the first VF/VMDq
6178          * VSI to be added switch to VEB mode.
6179          */
6180         if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
6181             (!i40e_is_any_channel(vsi))) {
6182                 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
6183                         dev_dbg(&pf->pdev->dev,
6184                                 "Failed to create channel. Override queues (%u) not power of 2\n",
6185                                 vsi->tc_config.tc_info[0].qcount);
6186                         return -EINVAL;
6187                 }
6188
6189                 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6190                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6191
6192                         if (vsi->type == I40E_VSI_MAIN) {
6193                                 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6194                                         i40e_do_reset(pf, I40E_PF_RESET_FLAG,
6195                                                       true);
6196                                 else
6197                                         i40e_do_reset_safe(pf,
6198                                                            I40E_PF_RESET_FLAG);
6199                         }
6200                 }
6201                 /* now onwards for main VSI, number of queues will be value
6202                  * of TC0's queue count
6203                  */
6204         }
6205
6206         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6207          * it should be more than num_queues
6208          */
6209         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6210                 dev_dbg(&pf->pdev->dev,
6211                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6212                         vsi->cnt_q_avail, ch->num_queue_pairs);
6213                 return -EINVAL;
6214         }
6215
6216         /* reconfig_rss only if vsi type is MAIN_VSI */
6217         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6218                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6219                 if (err) {
6220                         dev_info(&pf->pdev->dev,
6221                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6222                                  ch->num_queue_pairs);
6223                         return -EINVAL;
6224                 }
6225         }
6226
6227         if (!i40e_setup_channel(pf, vsi, ch)) {
6228                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6229                 return -EINVAL;
6230         }
6231
6232         dev_info(&pf->pdev->dev,
6233                  "Setup channel (id:%u) utilizing num_queues %d\n",
6234                  ch->seid, ch->num_queue_pairs);
6235
6236         /* configure VSI for BW limit */
6237         if (ch->max_tx_rate) {
6238                 u64 credits = ch->max_tx_rate;
6239
6240                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6241                         return -EINVAL;
6242
6243                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6244                 dev_dbg(&pf->pdev->dev,
6245                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6246                         ch->max_tx_rate,
6247                         credits,
6248                         ch->seid);
6249         }
6250
6251         /* in case of VF, this will be main SRIOV VSI */
6252         ch->parent_vsi = vsi;
6253
6254         /* and update main_vsi's count for queue_available to use */
6255         vsi->cnt_q_avail -= ch->num_queue_pairs;
6256
6257         return 0;
6258 }
6259
6260 /**
6261  * i40e_configure_queue_channels - Add queue channel for the given TCs
6262  * @vsi: VSI to be configured
6263  *
6264  * Configures queue channel mapping to the given TCs
6265  **/
6266 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6267 {
6268         struct i40e_channel *ch;
6269         u64 max_rate = 0;
6270         int ret = 0, i;
6271
6272         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6273         vsi->tc_seid_map[0] = vsi->seid;
6274         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6275                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6276                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6277                         if (!ch) {
6278                                 ret = -ENOMEM;
6279                                 goto err_free;
6280                         }
6281
6282                         INIT_LIST_HEAD(&ch->list);
6283                         ch->num_queue_pairs =
6284                                 vsi->tc_config.tc_info[i].qcount;
6285                         ch->base_queue =
6286                                 vsi->tc_config.tc_info[i].qoffset;
6287
6288                         /* Bandwidth limit through tc interface is in bytes/s,
6289                          * change to Mbit/s
6290                          */
6291                         max_rate = vsi->mqprio_qopt.max_rate[i];
6292                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6293                         ch->max_tx_rate = max_rate;
6294
6295                         list_add_tail(&ch->list, &vsi->ch_list);
6296
6297                         ret = i40e_create_queue_channel(vsi, ch);
6298                         if (ret) {
6299                                 dev_err(&vsi->back->pdev->dev,
6300                                         "Failed creating queue channel with TC%d: queues %d\n",
6301                                         i, ch->num_queue_pairs);
6302                                 goto err_free;
6303                         }
6304                         vsi->tc_seid_map[i] = ch->seid;
6305                 }
6306         }
6307         return ret;
6308
6309 err_free:
6310         i40e_remove_queue_channels(vsi);
6311         return ret;
6312 }
6313
6314 /**
6315  * i40e_veb_config_tc - Configure TCs for given VEB
6316  * @veb: given VEB
6317  * @enabled_tc: TC bitmap
6318  *
6319  * Configures given TC bitmap for VEB (switching) element
6320  **/
6321 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6322 {
6323         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6324         struct i40e_pf *pf = veb->pf;
6325         int ret = 0;
6326         int i;
6327
6328         /* No TCs or already enabled TCs just return */
6329         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6330                 return ret;
6331
6332         bw_data.tc_valid_bits = enabled_tc;
6333         /* bw_data.absolute_credits is not set (relative) */
6334
6335         /* Enable ETS TCs with equal BW Share for now */
6336         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6337                 if (enabled_tc & BIT(i))
6338                         bw_data.tc_bw_share_credits[i] = 1;
6339         }
6340
6341         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6342                                                    &bw_data, NULL);
6343         if (ret) {
6344                 dev_info(&pf->pdev->dev,
6345                          "VEB bw config failed, err %s aq_err %s\n",
6346                          i40e_stat_str(&pf->hw, ret),
6347                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6348                 goto out;
6349         }
6350
6351         /* Update the BW information */
6352         ret = i40e_veb_get_bw_info(veb);
6353         if (ret) {
6354                 dev_info(&pf->pdev->dev,
6355                          "Failed getting veb bw config, err %s aq_err %s\n",
6356                          i40e_stat_str(&pf->hw, ret),
6357                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6358         }
6359
6360 out:
6361         return ret;
6362 }
6363
6364 #ifdef CONFIG_I40E_DCB
6365 /**
6366  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6367  * @pf: PF struct
6368  *
6369  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6370  * the caller would've quiesce all the VSIs before calling
6371  * this function
6372  **/
6373 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6374 {
6375         u8 tc_map = 0;
6376         int ret;
6377         u8 v;
6378
6379         /* Enable the TCs available on PF to all VEBs */
6380         tc_map = i40e_pf_get_tc_map(pf);
6381         for (v = 0; v < I40E_MAX_VEB; v++) {
6382                 if (!pf->veb[v])
6383                         continue;
6384                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6385                 if (ret) {
6386                         dev_info(&pf->pdev->dev,
6387                                  "Failed configuring TC for VEB seid=%d\n",
6388                                  pf->veb[v]->seid);
6389                         /* Will try to configure as many components */
6390                 }
6391         }
6392
6393         /* Update each VSI */
6394         for (v = 0; v < pf->num_alloc_vsi; v++) {
6395                 if (!pf->vsi[v])
6396                         continue;
6397
6398                 /* - Enable all TCs for the LAN VSI
6399                  * - For all others keep them at TC0 for now
6400                  */
6401                 if (v == pf->lan_vsi)
6402                         tc_map = i40e_pf_get_tc_map(pf);
6403                 else
6404                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6405
6406                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6407                 if (ret) {
6408                         dev_info(&pf->pdev->dev,
6409                                  "Failed configuring TC for VSI seid=%d\n",
6410                                  pf->vsi[v]->seid);
6411                         /* Will try to configure as many components */
6412                 } else {
6413                         /* Re-configure VSI vectors based on updated TC map */
6414                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6415                         if (pf->vsi[v]->netdev)
6416                                 i40e_dcbnl_set_all(pf->vsi[v]);
6417                 }
6418         }
6419 }
6420
6421 /**
6422  * i40e_resume_port_tx - Resume port Tx
6423  * @pf: PF struct
6424  *
6425  * Resume a port's Tx and issue a PF reset in case of failure to
6426  * resume.
6427  **/
6428 static int i40e_resume_port_tx(struct i40e_pf *pf)
6429 {
6430         struct i40e_hw *hw = &pf->hw;
6431         int ret;
6432
6433         ret = i40e_aq_resume_port_tx(hw, NULL);
6434         if (ret) {
6435                 dev_info(&pf->pdev->dev,
6436                          "Resume Port Tx failed, err %s aq_err %s\n",
6437                           i40e_stat_str(&pf->hw, ret),
6438                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6439                 /* Schedule PF reset to recover */
6440                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6441                 i40e_service_event_schedule(pf);
6442         }
6443
6444         return ret;
6445 }
6446
6447 /**
6448  * i40e_init_pf_dcb - Initialize DCB configuration
6449  * @pf: PF being configured
6450  *
6451  * Query the current DCB configuration and cache it
6452  * in the hardware structure
6453  **/
6454 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6455 {
6456         struct i40e_hw *hw = &pf->hw;
6457         int err = 0;
6458
6459         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6460          * Also do not enable DCBx if FW LLDP agent is disabled
6461          */
6462         if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
6463             (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) {
6464                 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n");
6465                 err = I40E_NOT_SUPPORTED;
6466                 goto out;
6467         }
6468
6469         err = i40e_init_dcb(hw, true);
6470         if (!err) {
6471                 /* Device/Function is not DCBX capable */
6472                 if ((!hw->func_caps.dcb) ||
6473                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6474                         dev_info(&pf->pdev->dev,
6475                                  "DCBX offload is not supported or is disabled for this PF.\n");
6476                 } else {
6477                         /* When status is not DISABLED then DCBX in FW */
6478                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6479                                        DCB_CAP_DCBX_VER_IEEE;
6480
6481                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6482                         /* Enable DCB tagging only when more than one TC
6483                          * or explicitly disable if only one TC
6484                          */
6485                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6486                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6487                         else
6488                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6489                         dev_dbg(&pf->pdev->dev,
6490                                 "DCBX offload is supported for this PF.\n");
6491                 }
6492         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6493                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6494                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6495         } else {
6496                 dev_info(&pf->pdev->dev,
6497                          "Query for DCB configuration failed, err %s aq_err %s\n",
6498                          i40e_stat_str(&pf->hw, err),
6499                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6500         }
6501
6502 out:
6503         return err;
6504 }
6505 #endif /* CONFIG_I40E_DCB */
6506 #define SPEED_SIZE 14
6507 #define FC_SIZE 8
6508 /**
6509  * i40e_print_link_message - print link up or down
6510  * @vsi: the VSI for which link needs a message
6511  * @isup: true of link is up, false otherwise
6512  */
6513 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6514 {
6515         enum i40e_aq_link_speed new_speed;
6516         struct i40e_pf *pf = vsi->back;
6517         char *speed = "Unknown";
6518         char *fc = "Unknown";
6519         char *fec = "";
6520         char *req_fec = "";
6521         char *an = "";
6522
6523         if (isup)
6524                 new_speed = pf->hw.phy.link_info.link_speed;
6525         else
6526                 new_speed = I40E_LINK_SPEED_UNKNOWN;
6527
6528         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6529                 return;
6530         vsi->current_isup = isup;
6531         vsi->current_speed = new_speed;
6532         if (!isup) {
6533                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6534                 return;
6535         }
6536
6537         /* Warn user if link speed on NPAR enabled partition is not at
6538          * least 10GB
6539          */
6540         if (pf->hw.func_caps.npar_enable &&
6541             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6542              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6543                 netdev_warn(vsi->netdev,
6544                             "The partition detected link speed that is less than 10Gbps\n");
6545
6546         switch (pf->hw.phy.link_info.link_speed) {
6547         case I40E_LINK_SPEED_40GB:
6548                 speed = "40 G";
6549                 break;
6550         case I40E_LINK_SPEED_20GB:
6551                 speed = "20 G";
6552                 break;
6553         case I40E_LINK_SPEED_25GB:
6554                 speed = "25 G";
6555                 break;
6556         case I40E_LINK_SPEED_10GB:
6557                 speed = "10 G";
6558                 break;
6559         case I40E_LINK_SPEED_5GB:
6560                 speed = "5 G";
6561                 break;
6562         case I40E_LINK_SPEED_2_5GB:
6563                 speed = "2.5 G";
6564                 break;
6565         case I40E_LINK_SPEED_1GB:
6566                 speed = "1000 M";
6567                 break;
6568         case I40E_LINK_SPEED_100MB:
6569                 speed = "100 M";
6570                 break;
6571         default:
6572                 break;
6573         }
6574
6575         switch (pf->hw.fc.current_mode) {
6576         case I40E_FC_FULL:
6577                 fc = "RX/TX";
6578                 break;
6579         case I40E_FC_TX_PAUSE:
6580                 fc = "TX";
6581                 break;
6582         case I40E_FC_RX_PAUSE:
6583                 fc = "RX";
6584                 break;
6585         default:
6586                 fc = "None";
6587                 break;
6588         }
6589
6590         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6591                 req_fec = "None";
6592                 fec = "None";
6593                 an = "False";
6594
6595                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6596                         an = "True";
6597
6598                 if (pf->hw.phy.link_info.fec_info &
6599                     I40E_AQ_CONFIG_FEC_KR_ENA)
6600                         fec = "CL74 FC-FEC/BASE-R";
6601                 else if (pf->hw.phy.link_info.fec_info &
6602                          I40E_AQ_CONFIG_FEC_RS_ENA)
6603                         fec = "CL108 RS-FEC";
6604
6605                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
6606                  * both RS and FC are requested
6607                  */
6608                 if (vsi->back->hw.phy.link_info.req_fec_info &
6609                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
6610                         if (vsi->back->hw.phy.link_info.req_fec_info &
6611                             I40E_AQ_REQUEST_FEC_RS)
6612                                 req_fec = "CL108 RS-FEC";
6613                         else
6614                                 req_fec = "CL74 FC-FEC/BASE-R";
6615                 }
6616                 netdev_info(vsi->netdev,
6617                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
6618                             speed, req_fec, fec, an, fc);
6619         } else {
6620                 netdev_info(vsi->netdev,
6621                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
6622                             speed, fc);
6623         }
6624
6625 }
6626
6627 /**
6628  * i40e_up_complete - Finish the last steps of bringing up a connection
6629  * @vsi: the VSI being configured
6630  **/
6631 static int i40e_up_complete(struct i40e_vsi *vsi)
6632 {
6633         struct i40e_pf *pf = vsi->back;
6634         int err;
6635
6636         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6637                 i40e_vsi_configure_msix(vsi);
6638         else
6639                 i40e_configure_msi_and_legacy(vsi);
6640
6641         /* start rings */
6642         err = i40e_vsi_start_rings(vsi);
6643         if (err)
6644                 return err;
6645
6646         clear_bit(__I40E_VSI_DOWN, vsi->state);
6647         i40e_napi_enable_all(vsi);
6648         i40e_vsi_enable_irq(vsi);
6649
6650         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
6651             (vsi->netdev)) {
6652                 i40e_print_link_message(vsi, true);
6653                 netif_tx_start_all_queues(vsi->netdev);
6654                 netif_carrier_on(vsi->netdev);
6655         }
6656
6657         /* replay FDIR SB filters */
6658         if (vsi->type == I40E_VSI_FDIR) {
6659                 /* reset fd counters */
6660                 pf->fd_add_err = 0;
6661                 pf->fd_atr_cnt = 0;
6662                 i40e_fdir_filter_restore(vsi);
6663         }
6664
6665         /* On the next run of the service_task, notify any clients of the new
6666          * opened netdev
6667          */
6668         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6669         i40e_service_event_schedule(pf);
6670
6671         return 0;
6672 }
6673
6674 /**
6675  * i40e_vsi_reinit_locked - Reset the VSI
6676  * @vsi: the VSI being configured
6677  *
6678  * Rebuild the ring structs after some configuration
6679  * has changed, e.g. MTU size.
6680  **/
6681 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
6682 {
6683         struct i40e_pf *pf = vsi->back;
6684
6685         WARN_ON(in_interrupt());
6686         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
6687                 usleep_range(1000, 2000);
6688         i40e_down(vsi);
6689
6690         i40e_up(vsi);
6691         clear_bit(__I40E_CONFIG_BUSY, pf->state);
6692 }
6693
6694 /**
6695  * i40e_up - Bring the connection back up after being down
6696  * @vsi: the VSI being configured
6697  **/
6698 int i40e_up(struct i40e_vsi *vsi)
6699 {
6700         int err;
6701
6702         err = i40e_vsi_configure(vsi);
6703         if (!err)
6704                 err = i40e_up_complete(vsi);
6705
6706         return err;
6707 }
6708
6709 /**
6710  * i40e_force_link_state - Force the link status
6711  * @pf: board private structure
6712  * @is_up: whether the link state should be forced up or down
6713  **/
6714 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
6715 {
6716         struct i40e_aq_get_phy_abilities_resp abilities;
6717         struct i40e_aq_set_phy_config config = {0};
6718         struct i40e_hw *hw = &pf->hw;
6719         i40e_status err;
6720         u64 mask;
6721         u8 speed;
6722
6723         /* Card might've been put in an unstable state by other drivers
6724          * and applications, which causes incorrect speed values being
6725          * set on startup. In order to clear speed registers, we call
6726          * get_phy_capabilities twice, once to get initial state of
6727          * available speeds, and once to get current PHY config.
6728          */
6729         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
6730                                            NULL);
6731         if (err) {
6732                 dev_err(&pf->pdev->dev,
6733                         "failed to get phy cap., ret =  %s last_status =  %s\n",
6734                         i40e_stat_str(hw, err),
6735                         i40e_aq_str(hw, hw->aq.asq_last_status));
6736                 return err;
6737         }
6738         speed = abilities.link_speed;
6739
6740         /* Get the current phy config */
6741         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
6742                                            NULL);
6743         if (err) {
6744                 dev_err(&pf->pdev->dev,
6745                         "failed to get phy cap., ret =  %s last_status =  %s\n",
6746                         i40e_stat_str(hw, err),
6747                         i40e_aq_str(hw, hw->aq.asq_last_status));
6748                 return err;
6749         }
6750
6751         /* If link needs to go up, but was not forced to go down,
6752          * and its speed values are OK, no need for a flap
6753          */
6754         if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
6755                 return I40E_SUCCESS;
6756
6757         /* To force link we need to set bits for all supported PHY types,
6758          * but there are now more than 32, so we need to split the bitmap
6759          * across two fields.
6760          */
6761         mask = I40E_PHY_TYPES_BITMASK;
6762         config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
6763         config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0;
6764         /* Copy the old settings, except of phy_type */
6765         config.abilities = abilities.abilities;
6766         if (abilities.link_speed != 0)
6767                 config.link_speed = abilities.link_speed;
6768         else
6769                 config.link_speed = speed;
6770         config.eee_capability = abilities.eee_capability;
6771         config.eeer = abilities.eeer_val;
6772         config.low_power_ctrl = abilities.d3_lpan;
6773         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
6774                             I40E_AQ_PHY_FEC_CONFIG_MASK;
6775         err = i40e_aq_set_phy_config(hw, &config, NULL);
6776
6777         if (err) {
6778                 dev_err(&pf->pdev->dev,
6779                         "set phy config ret =  %s last_status =  %s\n",
6780                         i40e_stat_str(&pf->hw, err),
6781                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6782                 return err;
6783         }
6784
6785         /* Update the link info */
6786         err = i40e_update_link_info(hw);
6787         if (err) {
6788                 /* Wait a little bit (on 40G cards it sometimes takes a really
6789                  * long time for link to come back from the atomic reset)
6790                  * and try once more
6791                  */
6792                 msleep(1000);
6793                 i40e_update_link_info(hw);
6794         }
6795
6796         i40e_aq_set_link_restart_an(hw, true, NULL);
6797
6798         return I40E_SUCCESS;
6799 }
6800
6801 /**
6802  * i40e_down - Shutdown the connection processing
6803  * @vsi: the VSI being stopped
6804  **/
6805 void i40e_down(struct i40e_vsi *vsi)
6806 {
6807         int i;
6808
6809         /* It is assumed that the caller of this function
6810          * sets the vsi->state __I40E_VSI_DOWN bit.
6811          */
6812         if (vsi->netdev) {
6813                 netif_carrier_off(vsi->netdev);
6814                 netif_tx_disable(vsi->netdev);
6815         }
6816         i40e_vsi_disable_irq(vsi);
6817         i40e_vsi_stop_rings(vsi);
6818         if (vsi->type == I40E_VSI_MAIN &&
6819             vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED)
6820                 i40e_force_link_state(vsi->back, false);
6821         i40e_napi_disable_all(vsi);
6822
6823         for (i = 0; i < vsi->num_queue_pairs; i++) {
6824                 i40e_clean_tx_ring(vsi->tx_rings[i]);
6825                 if (i40e_enabled_xdp_vsi(vsi)) {
6826                         /* Make sure that in-progress ndo_xdp_xmit and
6827                          * ndo_xsk_wakeup calls are completed.
6828                          */
6829                         synchronize_rcu();
6830                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
6831                 }
6832                 i40e_clean_rx_ring(vsi->rx_rings[i]);
6833         }
6834
6835 }
6836
6837 /**
6838  * i40e_validate_mqprio_qopt- validate queue mapping info
6839  * @vsi: the VSI being configured
6840  * @mqprio_qopt: queue parametrs
6841  **/
6842 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
6843                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
6844 {
6845         u64 sum_max_rate = 0;
6846         u64 max_rate = 0;
6847         int i;
6848
6849         if (mqprio_qopt->qopt.offset[0] != 0 ||
6850             mqprio_qopt->qopt.num_tc < 1 ||
6851             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
6852                 return -EINVAL;
6853         for (i = 0; ; i++) {
6854                 if (!mqprio_qopt->qopt.count[i])
6855                         return -EINVAL;
6856                 if (mqprio_qopt->min_rate[i]) {
6857                         dev_err(&vsi->back->pdev->dev,
6858                                 "Invalid min tx rate (greater than 0) specified\n");
6859                         return -EINVAL;
6860                 }
6861                 max_rate = mqprio_qopt->max_rate[i];
6862                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6863                 sum_max_rate += max_rate;
6864
6865                 if (i >= mqprio_qopt->qopt.num_tc - 1)
6866                         break;
6867                 if (mqprio_qopt->qopt.offset[i + 1] !=
6868                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
6869                         return -EINVAL;
6870         }
6871         if (vsi->num_queue_pairs <
6872             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
6873                 return -EINVAL;
6874         }
6875         if (sum_max_rate > i40e_get_link_speed(vsi)) {
6876                 dev_err(&vsi->back->pdev->dev,
6877                         "Invalid max tx rate specified\n");
6878                 return -EINVAL;
6879         }
6880         return 0;
6881 }
6882
6883 /**
6884  * i40e_vsi_set_default_tc_config - set default values for tc configuration
6885  * @vsi: the VSI being configured
6886  **/
6887 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
6888 {
6889         u16 qcount;
6890         int i;
6891
6892         /* Only TC0 is enabled */
6893         vsi->tc_config.numtc = 1;
6894         vsi->tc_config.enabled_tc = 1;
6895         qcount = min_t(int, vsi->alloc_queue_pairs,
6896                        i40e_pf_get_max_q_per_tc(vsi->back));
6897         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6898                 /* For the TC that is not enabled set the offset to to default
6899                  * queue and allocate one queue for the given TC.
6900                  */
6901                 vsi->tc_config.tc_info[i].qoffset = 0;
6902                 if (i == 0)
6903                         vsi->tc_config.tc_info[i].qcount = qcount;
6904                 else
6905                         vsi->tc_config.tc_info[i].qcount = 1;
6906                 vsi->tc_config.tc_info[i].netdev_tc = 0;
6907         }
6908 }
6909
6910 /**
6911  * i40e_del_macvlan_filter
6912  * @hw: pointer to the HW structure
6913  * @seid: seid of the channel VSI
6914  * @macaddr: the mac address to apply as a filter
6915  * @aq_err: store the admin Q error
6916  *
6917  * This function deletes a mac filter on the channel VSI which serves as the
6918  * macvlan. Returns 0 on success.
6919  **/
6920 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
6921                                            const u8 *macaddr, int *aq_err)
6922 {
6923         struct i40e_aqc_remove_macvlan_element_data element;
6924         i40e_status status;
6925
6926         memset(&element, 0, sizeof(element));
6927         ether_addr_copy(element.mac_addr, macaddr);
6928         element.vlan_tag = 0;
6929         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
6930         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
6931         *aq_err = hw->aq.asq_last_status;
6932
6933         return status;
6934 }
6935
6936 /**
6937  * i40e_add_macvlan_filter
6938  * @hw: pointer to the HW structure
6939  * @seid: seid of the channel VSI
6940  * @macaddr: the mac address to apply as a filter
6941  * @aq_err: store the admin Q error
6942  *
6943  * This function adds a mac filter on the channel VSI which serves as the
6944  * macvlan. Returns 0 on success.
6945  **/
6946 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
6947                                            const u8 *macaddr, int *aq_err)
6948 {
6949         struct i40e_aqc_add_macvlan_element_data element;
6950         i40e_status status;
6951         u16 cmd_flags = 0;
6952
6953         ether_addr_copy(element.mac_addr, macaddr);
6954         element.vlan_tag = 0;
6955         element.queue_number = 0;
6956         element.match_method = I40E_AQC_MM_ERR_NO_RES;
6957         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
6958         element.flags = cpu_to_le16(cmd_flags);
6959         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
6960         *aq_err = hw->aq.asq_last_status;
6961
6962         return status;
6963 }
6964
6965 /**
6966  * i40e_reset_ch_rings - Reset the queue contexts in a channel
6967  * @vsi: the VSI we want to access
6968  * @ch: the channel we want to access
6969  */
6970 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
6971 {
6972         struct i40e_ring *tx_ring, *rx_ring;
6973         u16 pf_q;
6974         int i;
6975
6976         for (i = 0; i < ch->num_queue_pairs; i++) {
6977                 pf_q = ch->base_queue + i;
6978                 tx_ring = vsi->tx_rings[pf_q];
6979                 tx_ring->ch = NULL;
6980                 rx_ring = vsi->rx_rings[pf_q];
6981                 rx_ring->ch = NULL;
6982         }
6983 }
6984
6985 /**
6986  * i40e_free_macvlan_channels
6987  * @vsi: the VSI we want to access
6988  *
6989  * This function frees the Qs of the channel VSI from
6990  * the stack and also deletes the channel VSIs which
6991  * serve as macvlans.
6992  */
6993 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
6994 {
6995         struct i40e_channel *ch, *ch_tmp;
6996         int ret;
6997
6998         if (list_empty(&vsi->macvlan_list))
6999                 return;
7000
7001         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7002                 struct i40e_vsi *parent_vsi;
7003
7004                 if (i40e_is_channel_macvlan(ch)) {
7005                         i40e_reset_ch_rings(vsi, ch);
7006                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7007                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7008                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7009                         kfree(ch->fwd);
7010                         ch->fwd = NULL;
7011                 }
7012
7013                 list_del(&ch->list);
7014                 parent_vsi = ch->parent_vsi;
7015                 if (!parent_vsi || !ch->initialized) {
7016                         kfree(ch);
7017                         continue;
7018                 }
7019
7020                 /* remove the VSI */
7021                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7022                                              NULL);
7023                 if (ret)
7024                         dev_err(&vsi->back->pdev->dev,
7025                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7026                                 ch->seid, parent_vsi->seid);
7027                 kfree(ch);
7028         }
7029         vsi->macvlan_cnt = 0;
7030 }
7031
7032 /**
7033  * i40e_fwd_ring_up - bring the macvlan device up
7034  * @vsi: the VSI we want to access
7035  * @vdev: macvlan netdevice
7036  * @fwd: the private fwd structure
7037  */
7038 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7039                             struct i40e_fwd_adapter *fwd)
7040 {
7041         int ret = 0, num_tc = 1,  i, aq_err;
7042         struct i40e_channel *ch, *ch_tmp;
7043         struct i40e_pf *pf = vsi->back;
7044         struct i40e_hw *hw = &pf->hw;
7045
7046         if (list_empty(&vsi->macvlan_list))
7047                 return -EINVAL;
7048
7049         /* Go through the list and find an available channel */
7050         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7051                 if (!i40e_is_channel_macvlan(ch)) {
7052                         ch->fwd = fwd;
7053                         /* record configuration for macvlan interface in vdev */
7054                         for (i = 0; i < num_tc; i++)
7055                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7056                                                              i,
7057                                                              ch->num_queue_pairs,
7058                                                              ch->base_queue);
7059                         for (i = 0; i < ch->num_queue_pairs; i++) {
7060                                 struct i40e_ring *tx_ring, *rx_ring;
7061                                 u16 pf_q;
7062
7063                                 pf_q = ch->base_queue + i;
7064
7065                                 /* Get to TX ring ptr */
7066                                 tx_ring = vsi->tx_rings[pf_q];
7067                                 tx_ring->ch = ch;
7068
7069                                 /* Get the RX ring ptr */
7070                                 rx_ring = vsi->rx_rings[pf_q];
7071                                 rx_ring->ch = ch;
7072                         }
7073                         break;
7074                 }
7075         }
7076
7077         /* Guarantee all rings are updated before we update the
7078          * MAC address filter.
7079          */
7080         wmb();
7081
7082         /* Add a mac filter */
7083         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7084         if (ret) {
7085                 /* if we cannot add the MAC rule then disable the offload */
7086                 macvlan_release_l2fw_offload(vdev);
7087                 for (i = 0; i < ch->num_queue_pairs; i++) {
7088                         struct i40e_ring *rx_ring;
7089                         u16 pf_q;
7090
7091                         pf_q = ch->base_queue + i;
7092                         rx_ring = vsi->rx_rings[pf_q];
7093                         rx_ring->netdev = NULL;
7094                 }
7095                 dev_info(&pf->pdev->dev,
7096                          "Error adding mac filter on macvlan err %s, aq_err %s\n",
7097                           i40e_stat_str(hw, ret),
7098                           i40e_aq_str(hw, aq_err));
7099                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7100         }
7101
7102         return ret;
7103 }
7104
7105 /**
7106  * i40e_setup_macvlans - create the channels which will be macvlans
7107  * @vsi: the VSI we want to access
7108  * @macvlan_cnt: no. of macvlans to be setup
7109  * @qcnt: no. of Qs per macvlan
7110  * @vdev: macvlan netdevice
7111  */
7112 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7113                                struct net_device *vdev)
7114 {
7115         struct i40e_pf *pf = vsi->back;
7116         struct i40e_hw *hw = &pf->hw;
7117         struct i40e_vsi_context ctxt;
7118         u16 sections, qmap, num_qps;
7119         struct i40e_channel *ch;
7120         int i, pow, ret = 0;
7121         u8 offset = 0;
7122
7123         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7124                 return -EINVAL;
7125
7126         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7127
7128         /* find the next higher power-of-2 of num queue pairs */
7129         pow = fls(roundup_pow_of_two(num_qps) - 1);
7130
7131         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7132                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7133
7134         /* Setup context bits for the main VSI */
7135         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7136         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7137         memset(&ctxt, 0, sizeof(ctxt));
7138         ctxt.seid = vsi->seid;
7139         ctxt.pf_num = vsi->back->hw.pf_id;
7140         ctxt.vf_num = 0;
7141         ctxt.uplink_seid = vsi->uplink_seid;
7142         ctxt.info = vsi->info;
7143         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7144         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7145         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7146         ctxt.info.valid_sections |= cpu_to_le16(sections);
7147
7148         /* Reconfigure RSS for main VSI with new max queue count */
7149         vsi->rss_size = max_t(u16, num_qps, qcnt);
7150         ret = i40e_vsi_config_rss(vsi);
7151         if (ret) {
7152                 dev_info(&pf->pdev->dev,
7153                          "Failed to reconfig RSS for num_queues (%u)\n",
7154                          vsi->rss_size);
7155                 return ret;
7156         }
7157         vsi->reconfig_rss = true;
7158         dev_dbg(&vsi->back->pdev->dev,
7159                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7160         vsi->next_base_queue = num_qps;
7161         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7162
7163         /* Update the VSI after updating the VSI queue-mapping
7164          * information
7165          */
7166         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7167         if (ret) {
7168                 dev_info(&pf->pdev->dev,
7169                          "Update vsi tc config failed, err %s aq_err %s\n",
7170                          i40e_stat_str(hw, ret),
7171                          i40e_aq_str(hw, hw->aq.asq_last_status));
7172                 return ret;
7173         }
7174         /* update the local VSI info with updated queue map */
7175         i40e_vsi_update_queue_map(vsi, &ctxt);
7176         vsi->info.valid_sections = 0;
7177
7178         /* Create channels for macvlans */
7179         INIT_LIST_HEAD(&vsi->macvlan_list);
7180         for (i = 0; i < macvlan_cnt; i++) {
7181                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7182                 if (!ch) {
7183                         ret = -ENOMEM;
7184                         goto err_free;
7185                 }
7186                 INIT_LIST_HEAD(&ch->list);
7187                 ch->num_queue_pairs = qcnt;
7188                 if (!i40e_setup_channel(pf, vsi, ch)) {
7189                         ret = -EINVAL;
7190                         kfree(ch);
7191                         goto err_free;
7192                 }
7193                 ch->parent_vsi = vsi;
7194                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7195                 vsi->macvlan_cnt++;
7196                 list_add_tail(&ch->list, &vsi->macvlan_list);
7197         }
7198
7199         return ret;
7200
7201 err_free:
7202         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7203         i40e_free_macvlan_channels(vsi);
7204
7205         return ret;
7206 }
7207
7208 /**
7209  * i40e_fwd_add - configure macvlans
7210  * @netdev: net device to configure
7211  * @vdev: macvlan netdevice
7212  **/
7213 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7214 {
7215         struct i40e_netdev_priv *np = netdev_priv(netdev);
7216         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7217         struct i40e_vsi *vsi = np->vsi;
7218         struct i40e_pf *pf = vsi->back;
7219         struct i40e_fwd_adapter *fwd;
7220         int avail_macvlan, ret;
7221
7222         if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7223                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7224                 return ERR_PTR(-EINVAL);
7225         }
7226         if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7227                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7228                 return ERR_PTR(-EINVAL);
7229         }
7230         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7231                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7232                 return ERR_PTR(-EINVAL);
7233         }
7234
7235         /* The macvlan device has to be a single Q device so that the
7236          * tc_to_txq field can be reused to pick the tx queue.
7237          */
7238         if (netif_is_multiqueue(vdev))
7239                 return ERR_PTR(-ERANGE);
7240
7241         if (!vsi->macvlan_cnt) {
7242                 /* reserve bit 0 for the pf device */
7243                 set_bit(0, vsi->fwd_bitmask);
7244
7245                 /* Try to reserve as many queues as possible for macvlans. First
7246                  * reserve 3/4th of max vectors, then half, then quarter and
7247                  * calculate Qs per macvlan as you go
7248                  */
7249                 vectors = pf->num_lan_msix;
7250                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7251                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7252                         q_per_macvlan = 4;
7253                         macvlan_cnt = (vectors - 32) / 4;
7254                 } else if (vectors <= 64 && vectors > 32) {
7255                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7256                         q_per_macvlan = 2;
7257                         macvlan_cnt = (vectors - 16) / 2;
7258                 } else if (vectors <= 32 && vectors > 16) {
7259                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7260                         q_per_macvlan = 1;
7261                         macvlan_cnt = vectors - 16;
7262                 } else if (vectors <= 16 && vectors > 8) {
7263                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
7264                         q_per_macvlan = 1;
7265                         macvlan_cnt = vectors - 8;
7266                 } else {
7267                         /* allocate 1 Q per macvlan and 1 Q to the PF */
7268                         q_per_macvlan = 1;
7269                         macvlan_cnt = vectors - 1;
7270                 }
7271
7272                 if (macvlan_cnt == 0)
7273                         return ERR_PTR(-EBUSY);
7274
7275                 /* Quiesce VSI queues */
7276                 i40e_quiesce_vsi(vsi);
7277
7278                 /* sets up the macvlans but does not "enable" them */
7279                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7280                                           vdev);
7281                 if (ret)
7282                         return ERR_PTR(ret);
7283
7284                 /* Unquiesce VSI */
7285                 i40e_unquiesce_vsi(vsi);
7286         }
7287         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7288                                             vsi->macvlan_cnt);
7289         if (avail_macvlan >= I40E_MAX_MACVLANS)
7290                 return ERR_PTR(-EBUSY);
7291
7292         /* create the fwd struct */
7293         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7294         if (!fwd)
7295                 return ERR_PTR(-ENOMEM);
7296
7297         set_bit(avail_macvlan, vsi->fwd_bitmask);
7298         fwd->bit_no = avail_macvlan;
7299         netdev_set_sb_channel(vdev, avail_macvlan);
7300         fwd->netdev = vdev;
7301
7302         if (!netif_running(netdev))
7303                 return fwd;
7304
7305         /* Set fwd ring up */
7306         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7307         if (ret) {
7308                 /* unbind the queues and drop the subordinate channel config */
7309                 netdev_unbind_sb_channel(netdev, vdev);
7310                 netdev_set_sb_channel(vdev, 0);
7311
7312                 kfree(fwd);
7313                 return ERR_PTR(-EINVAL);
7314         }
7315
7316         return fwd;
7317 }
7318
7319 /**
7320  * i40e_del_all_macvlans - Delete all the mac filters on the channels
7321  * @vsi: the VSI we want to access
7322  */
7323 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7324 {
7325         struct i40e_channel *ch, *ch_tmp;
7326         struct i40e_pf *pf = vsi->back;
7327         struct i40e_hw *hw = &pf->hw;
7328         int aq_err, ret = 0;
7329
7330         if (list_empty(&vsi->macvlan_list))
7331                 return;
7332
7333         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7334                 if (i40e_is_channel_macvlan(ch)) {
7335                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7336                                                       i40e_channel_mac(ch),
7337                                                       &aq_err);
7338                         if (!ret) {
7339                                 /* Reset queue contexts */
7340                                 i40e_reset_ch_rings(vsi, ch);
7341                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7342                                 netdev_unbind_sb_channel(vsi->netdev,
7343                                                          ch->fwd->netdev);
7344                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
7345                                 kfree(ch->fwd);
7346                                 ch->fwd = NULL;
7347                         }
7348                 }
7349         }
7350 }
7351
7352 /**
7353  * i40e_fwd_del - delete macvlan interfaces
7354  * @netdev: net device to configure
7355  * @vdev: macvlan netdevice
7356  */
7357 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7358 {
7359         struct i40e_netdev_priv *np = netdev_priv(netdev);
7360         struct i40e_fwd_adapter *fwd = vdev;
7361         struct i40e_channel *ch, *ch_tmp;
7362         struct i40e_vsi *vsi = np->vsi;
7363         struct i40e_pf *pf = vsi->back;
7364         struct i40e_hw *hw = &pf->hw;
7365         int aq_err, ret = 0;
7366
7367         /* Find the channel associated with the macvlan and del mac filter */
7368         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7369                 if (i40e_is_channel_macvlan(ch) &&
7370                     ether_addr_equal(i40e_channel_mac(ch),
7371                                      fwd->netdev->dev_addr)) {
7372                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7373                                                       i40e_channel_mac(ch),
7374                                                       &aq_err);
7375                         if (!ret) {
7376                                 /* Reset queue contexts */
7377                                 i40e_reset_ch_rings(vsi, ch);
7378                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7379                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
7380                                 netdev_set_sb_channel(fwd->netdev, 0);
7381                                 kfree(ch->fwd);
7382                                 ch->fwd = NULL;
7383                         } else {
7384                                 dev_info(&pf->pdev->dev,
7385                                          "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7386                                           i40e_stat_str(hw, ret),
7387                                           i40e_aq_str(hw, aq_err));
7388                         }
7389                         break;
7390                 }
7391         }
7392 }
7393
7394 /**
7395  * i40e_setup_tc - configure multiple traffic classes
7396  * @netdev: net device to configure
7397  * @type_data: tc offload data
7398  **/
7399 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7400 {
7401         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7402         struct i40e_netdev_priv *np = netdev_priv(netdev);
7403         struct i40e_vsi *vsi = np->vsi;
7404         struct i40e_pf *pf = vsi->back;
7405         u8 enabled_tc = 0, num_tc, hw;
7406         bool need_reset = false;
7407         int old_queue_pairs;
7408         int ret = -EINVAL;
7409         u16 mode;
7410         int i;
7411
7412         old_queue_pairs = vsi->num_queue_pairs;
7413         num_tc = mqprio_qopt->qopt.num_tc;
7414         hw = mqprio_qopt->qopt.hw;
7415         mode = mqprio_qopt->mode;
7416         if (!hw) {
7417                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7418                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7419                 goto config_tc;
7420         }
7421
7422         /* Check if MFP enabled */
7423         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7424                 netdev_info(netdev,
7425                             "Configuring TC not supported in MFP mode\n");
7426                 return ret;
7427         }
7428         switch (mode) {
7429         case TC_MQPRIO_MODE_DCB:
7430                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7431
7432                 /* Check if DCB enabled to continue */
7433                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7434                         netdev_info(netdev,
7435                                     "DCB is not enabled for adapter\n");
7436                         return ret;
7437                 }
7438
7439                 /* Check whether tc count is within enabled limit */
7440                 if (num_tc > i40e_pf_get_num_tc(pf)) {
7441                         netdev_info(netdev,
7442                                     "TC count greater than enabled on link for adapter\n");
7443                         return ret;
7444                 }
7445                 break;
7446         case TC_MQPRIO_MODE_CHANNEL:
7447                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7448                         netdev_info(netdev,
7449                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7450                         return ret;
7451                 }
7452                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7453                         return ret;
7454                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7455                 if (ret)
7456                         return ret;
7457                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7458                        sizeof(*mqprio_qopt));
7459                 pf->flags |= I40E_FLAG_TC_MQPRIO;
7460                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7461                 break;
7462         default:
7463                 return -EINVAL;
7464         }
7465
7466 config_tc:
7467         /* Generate TC map for number of tc requested */
7468         for (i = 0; i < num_tc; i++)
7469                 enabled_tc |= BIT(i);
7470
7471         /* Requesting same TC configuration as already enabled */
7472         if (enabled_tc == vsi->tc_config.enabled_tc &&
7473             mode != TC_MQPRIO_MODE_CHANNEL)
7474                 return 0;
7475
7476         /* Quiesce VSI queues */
7477         i40e_quiesce_vsi(vsi);
7478
7479         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7480                 i40e_remove_queue_channels(vsi);
7481
7482         /* Configure VSI for enabled TCs */
7483         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7484         if (ret) {
7485                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7486                             vsi->seid);
7487                 need_reset = true;
7488                 goto exit;
7489         } else {
7490                 dev_info(&vsi->back->pdev->dev,
7491                          "Setup channel (id:%u) utilizing num_queues %d\n",
7492                          vsi->seid, vsi->tc_config.tc_info[0].qcount);
7493         }
7494
7495         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7496                 if (vsi->mqprio_qopt.max_rate[0]) {
7497                         u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
7498
7499                         do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
7500                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
7501                         if (!ret) {
7502                                 u64 credits = max_tx_rate;
7503
7504                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
7505                                 dev_dbg(&vsi->back->pdev->dev,
7506                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
7507                                         max_tx_rate,
7508                                         credits,
7509                                         vsi->seid);
7510                         } else {
7511                                 need_reset = true;
7512                                 goto exit;
7513                         }
7514                 }
7515                 ret = i40e_configure_queue_channels(vsi);
7516                 if (ret) {
7517                         vsi->num_queue_pairs = old_queue_pairs;
7518                         netdev_info(netdev,
7519                                     "Failed configuring queue channels\n");
7520                         need_reset = true;
7521                         goto exit;
7522                 }
7523         }
7524
7525 exit:
7526         /* Reset the configuration data to defaults, only TC0 is enabled */
7527         if (need_reset) {
7528                 i40e_vsi_set_default_tc_config(vsi);
7529                 need_reset = false;
7530         }
7531
7532         /* Unquiesce VSI */
7533         i40e_unquiesce_vsi(vsi);
7534         return ret;
7535 }
7536
7537 /**
7538  * i40e_set_cld_element - sets cloud filter element data
7539  * @filter: cloud filter rule
7540  * @cld: ptr to cloud filter element data
7541  *
7542  * This is helper function to copy data into cloud filter element
7543  **/
7544 static inline void
7545 i40e_set_cld_element(struct i40e_cloud_filter *filter,
7546                      struct i40e_aqc_cloud_filters_element_data *cld)
7547 {
7548         int i, j;
7549         u32 ipa;
7550
7551         memset(cld, 0, sizeof(*cld));
7552         ether_addr_copy(cld->outer_mac, filter->dst_mac);
7553         ether_addr_copy(cld->inner_mac, filter->src_mac);
7554
7555         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
7556                 return;
7557
7558         if (filter->n_proto == ETH_P_IPV6) {
7559 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
7560                 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6);
7561                      i++, j += 2) {
7562                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
7563                         ipa = cpu_to_le32(ipa);
7564                         memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa));
7565                 }
7566         } else {
7567                 ipa = be32_to_cpu(filter->dst_ipv4);
7568                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
7569         }
7570
7571         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
7572
7573         /* tenant_id is not supported by FW now, once the support is enabled
7574          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
7575          */
7576         if (filter->tenant_id)
7577                 return;
7578 }
7579
7580 /**
7581  * i40e_add_del_cloud_filter - Add/del cloud filter
7582  * @vsi: pointer to VSI
7583  * @filter: cloud filter rule
7584  * @add: if true, add, if false, delete
7585  *
7586  * Add or delete a cloud filter for a specific flow spec.
7587  * Returns 0 if the filter were successfully added.
7588  **/
7589 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
7590                               struct i40e_cloud_filter *filter, bool add)
7591 {
7592         struct i40e_aqc_cloud_filters_element_data cld_filter;
7593         struct i40e_pf *pf = vsi->back;
7594         int ret;
7595         static const u16 flag_table[128] = {
7596                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
7597                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
7598                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
7599                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
7600                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
7601                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
7602                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
7603                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
7604                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
7605                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
7606                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
7607                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
7608                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
7609                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
7610         };
7611
7612         if (filter->flags >= ARRAY_SIZE(flag_table))
7613                 return I40E_ERR_CONFIG;
7614
7615         /* copy element needed to add cloud filter from filter */
7616         i40e_set_cld_element(filter, &cld_filter);
7617
7618         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
7619                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
7620                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
7621
7622         if (filter->n_proto == ETH_P_IPV6)
7623                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7624                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7625         else
7626                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7627                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7628
7629         if (add)
7630                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
7631                                                 &cld_filter, 1);
7632         else
7633                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
7634                                                 &cld_filter, 1);
7635         if (ret)
7636                 dev_dbg(&pf->pdev->dev,
7637                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
7638                         add ? "add" : "delete", filter->dst_port, ret,
7639                         pf->hw.aq.asq_last_status);
7640         else
7641                 dev_info(&pf->pdev->dev,
7642                          "%s cloud filter for VSI: %d\n",
7643                          add ? "Added" : "Deleted", filter->seid);
7644         return ret;
7645 }
7646
7647 /**
7648  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
7649  * @vsi: pointer to VSI
7650  * @filter: cloud filter rule
7651  * @add: if true, add, if false, delete
7652  *
7653  * Add or delete a cloud filter for a specific flow spec using big buffer.
7654  * Returns 0 if the filter were successfully added.
7655  **/
7656 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
7657                                       struct i40e_cloud_filter *filter,
7658                                       bool add)
7659 {
7660         struct i40e_aqc_cloud_filters_element_bb cld_filter;
7661         struct i40e_pf *pf = vsi->back;
7662         int ret;
7663
7664         /* Both (src/dst) valid mac_addr are not supported */
7665         if ((is_valid_ether_addr(filter->dst_mac) &&
7666              is_valid_ether_addr(filter->src_mac)) ||
7667             (is_multicast_ether_addr(filter->dst_mac) &&
7668              is_multicast_ether_addr(filter->src_mac)))
7669                 return -EOPNOTSUPP;
7670
7671         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
7672          * ports are not supported via big buffer now.
7673          */
7674         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
7675                 return -EOPNOTSUPP;
7676
7677         /* adding filter using src_port/src_ip is not supported at this stage */
7678         if (filter->src_port || filter->src_ipv4 ||
7679             !ipv6_addr_any(&filter->ip.v6.src_ip6))
7680                 return -EOPNOTSUPP;
7681
7682         /* copy element needed to add cloud filter from filter */
7683         i40e_set_cld_element(filter, &cld_filter.element);
7684
7685         if (is_valid_ether_addr(filter->dst_mac) ||
7686             is_valid_ether_addr(filter->src_mac) ||
7687             is_multicast_ether_addr(filter->dst_mac) ||
7688             is_multicast_ether_addr(filter->src_mac)) {
7689                 /* MAC + IP : unsupported mode */
7690                 if (filter->dst_ipv4)
7691                         return -EOPNOTSUPP;
7692
7693                 /* since we validated that L4 port must be valid before
7694                  * we get here, start with respective "flags" value
7695                  * and update if vlan is present or not
7696                  */
7697                 cld_filter.element.flags =
7698                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
7699
7700                 if (filter->vlan_id) {
7701                         cld_filter.element.flags =
7702                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
7703                 }
7704
7705         } else if (filter->dst_ipv4 ||
7706                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
7707                 cld_filter.element.flags =
7708                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
7709                 if (filter->n_proto == ETH_P_IPV6)
7710                         cld_filter.element.flags |=
7711                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7712                 else
7713                         cld_filter.element.flags |=
7714                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7715         } else {
7716                 dev_err(&pf->pdev->dev,
7717                         "either mac or ip has to be valid for cloud filter\n");
7718                 return -EINVAL;
7719         }
7720
7721         /* Now copy L4 port in Byte 6..7 in general fields */
7722         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
7723                                                 be16_to_cpu(filter->dst_port);
7724
7725         if (add) {
7726                 /* Validate current device switch mode, change if necessary */
7727                 ret = i40e_validate_and_set_switch_mode(vsi);
7728                 if (ret) {
7729                         dev_err(&pf->pdev->dev,
7730                                 "failed to set switch mode, ret %d\n",
7731                                 ret);
7732                         return ret;
7733                 }
7734
7735                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
7736                                                    &cld_filter, 1);
7737         } else {
7738                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
7739                                                    &cld_filter, 1);
7740         }
7741
7742         if (ret)
7743                 dev_dbg(&pf->pdev->dev,
7744                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
7745                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
7746         else
7747                 dev_info(&pf->pdev->dev,
7748                          "%s cloud filter for VSI: %d, L4 port: %d\n",
7749                          add ? "add" : "delete", filter->seid,
7750                          ntohs(filter->dst_port));
7751         return ret;
7752 }
7753
7754 /**
7755  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
7756  * @vsi: Pointer to VSI
7757  * @cls_flower: Pointer to struct flow_cls_offload
7758  * @filter: Pointer to cloud filter structure
7759  *
7760  **/
7761 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
7762                                  struct flow_cls_offload *f,
7763                                  struct i40e_cloud_filter *filter)
7764 {
7765         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
7766         struct flow_dissector *dissector = rule->match.dissector;
7767         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
7768         struct i40e_pf *pf = vsi->back;
7769         u8 field_flags = 0;
7770
7771         if (dissector->used_keys &
7772             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
7773               BIT(FLOW_DISSECTOR_KEY_BASIC) |
7774               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
7775               BIT(FLOW_DISSECTOR_KEY_VLAN) |
7776               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
7777               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
7778               BIT(FLOW_DISSECTOR_KEY_PORTS) |
7779               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
7780                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
7781                         dissector->used_keys);
7782                 return -EOPNOTSUPP;
7783         }
7784
7785         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
7786                 struct flow_match_enc_keyid match;
7787
7788                 flow_rule_match_enc_keyid(rule, &match);
7789                 if (match.mask->keyid != 0)
7790                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
7791
7792                 filter->tenant_id = be32_to_cpu(match.key->keyid);
7793         }
7794
7795         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
7796                 struct flow_match_basic match;
7797
7798                 flow_rule_match_basic(rule, &match);
7799                 n_proto_key = ntohs(match.key->n_proto);
7800                 n_proto_mask = ntohs(match.mask->n_proto);
7801
7802                 if (n_proto_key == ETH_P_ALL) {
7803                         n_proto_key = 0;
7804                         n_proto_mask = 0;
7805                 }
7806                 filter->n_proto = n_proto_key & n_proto_mask;
7807                 filter->ip_proto = match.key->ip_proto;
7808         }
7809
7810         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
7811                 struct flow_match_eth_addrs match;
7812
7813                 flow_rule_match_eth_addrs(rule, &match);
7814
7815                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
7816                 if (!is_zero_ether_addr(match.mask->dst)) {
7817                         if (is_broadcast_ether_addr(match.mask->dst)) {
7818                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
7819                         } else {
7820                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
7821                                         match.mask->dst);
7822                                 return I40E_ERR_CONFIG;
7823                         }
7824                 }
7825
7826                 if (!is_zero_ether_addr(match.mask->src)) {
7827                         if (is_broadcast_ether_addr(match.mask->src)) {
7828                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
7829                         } else {
7830                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
7831                                         match.mask->src);
7832                                 return I40E_ERR_CONFIG;
7833                         }
7834                 }
7835                 ether_addr_copy(filter->dst_mac, match.key->dst);
7836                 ether_addr_copy(filter->src_mac, match.key->src);
7837         }
7838
7839         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
7840                 struct flow_match_vlan match;
7841
7842                 flow_rule_match_vlan(rule, &match);
7843                 if (match.mask->vlan_id) {
7844                         if (match.mask->vlan_id == VLAN_VID_MASK) {
7845                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
7846
7847                         } else {
7848                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
7849                                         match.mask->vlan_id);
7850                                 return I40E_ERR_CONFIG;
7851                         }
7852                 }
7853
7854                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
7855         }
7856
7857         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
7858                 struct flow_match_control match;
7859
7860                 flow_rule_match_control(rule, &match);
7861                 addr_type = match.key->addr_type;
7862         }
7863
7864         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
7865                 struct flow_match_ipv4_addrs match;
7866
7867                 flow_rule_match_ipv4_addrs(rule, &match);
7868                 if (match.mask->dst) {
7869                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
7870                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7871                         } else {
7872                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
7873                                         &match.mask->dst);
7874                                 return I40E_ERR_CONFIG;
7875                         }
7876                 }
7877
7878                 if (match.mask->src) {
7879                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
7880                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7881                         } else {
7882                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
7883                                         &match.mask->src);
7884                                 return I40E_ERR_CONFIG;
7885                         }
7886                 }
7887
7888                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
7889                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
7890                         return I40E_ERR_CONFIG;
7891                 }
7892                 filter->dst_ipv4 = match.key->dst;
7893                 filter->src_ipv4 = match.key->src;
7894         }
7895
7896         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
7897                 struct flow_match_ipv6_addrs match;
7898
7899                 flow_rule_match_ipv6_addrs(rule, &match);
7900
7901                 /* src and dest IPV6 address should not be LOOPBACK
7902                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
7903                  */
7904                 if (ipv6_addr_loopback(&match.key->dst) ||
7905                     ipv6_addr_loopback(&match.key->src)) {
7906                         dev_err(&pf->pdev->dev,
7907                                 "Bad ipv6, addr is LOOPBACK\n");
7908                         return I40E_ERR_CONFIG;
7909                 }
7910                 if (!ipv6_addr_any(&match.mask->dst) ||
7911                     !ipv6_addr_any(&match.mask->src))
7912                         field_flags |= I40E_CLOUD_FIELD_IIP;
7913
7914                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
7915                        sizeof(filter->src_ipv6));
7916                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
7917                        sizeof(filter->dst_ipv6));
7918         }
7919
7920         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
7921                 struct flow_match_ports match;
7922
7923                 flow_rule_match_ports(rule, &match);
7924                 if (match.mask->src) {
7925                         if (match.mask->src == cpu_to_be16(0xffff)) {
7926                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7927                         } else {
7928                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
7929                                         be16_to_cpu(match.mask->src));
7930                                 return I40E_ERR_CONFIG;
7931                         }
7932                 }
7933
7934                 if (match.mask->dst) {
7935                         if (match.mask->dst == cpu_to_be16(0xffff)) {
7936                                 field_flags |= I40E_CLOUD_FIELD_IIP;
7937                         } else {
7938                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
7939                                         be16_to_cpu(match.mask->dst));
7940                                 return I40E_ERR_CONFIG;
7941                         }
7942                 }
7943
7944                 filter->dst_port = match.key->dst;
7945                 filter->src_port = match.key->src;
7946
7947                 switch (filter->ip_proto) {
7948                 case IPPROTO_TCP:
7949                 case IPPROTO_UDP:
7950                         break;
7951                 default:
7952                         dev_err(&pf->pdev->dev,
7953                                 "Only UDP and TCP transport are supported\n");
7954                         return -EINVAL;
7955                 }
7956         }
7957         filter->flags = field_flags;
7958         return 0;
7959 }
7960
7961 /**
7962  * i40e_handle_tclass: Forward to a traffic class on the device
7963  * @vsi: Pointer to VSI
7964  * @tc: traffic class index on the device
7965  * @filter: Pointer to cloud filter structure
7966  *
7967  **/
7968 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
7969                               struct i40e_cloud_filter *filter)
7970 {
7971         struct i40e_channel *ch, *ch_tmp;
7972
7973         /* direct to a traffic class on the same device */
7974         if (tc == 0) {
7975                 filter->seid = vsi->seid;
7976                 return 0;
7977         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
7978                 if (!filter->dst_port) {
7979                         dev_err(&vsi->back->pdev->dev,
7980                                 "Specify destination port to direct to traffic class that is not default\n");
7981                         return -EINVAL;
7982                 }
7983                 if (list_empty(&vsi->ch_list))
7984                         return -EINVAL;
7985                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
7986                                          list) {
7987                         if (ch->seid == vsi->tc_seid_map[tc])
7988                                 filter->seid = ch->seid;
7989                 }
7990                 return 0;
7991         }
7992         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
7993         return -EINVAL;
7994 }
7995
7996 /**
7997  * i40e_configure_clsflower - Configure tc flower filters
7998  * @vsi: Pointer to VSI
7999  * @cls_flower: Pointer to struct flow_cls_offload
8000  *
8001  **/
8002 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8003                                     struct flow_cls_offload *cls_flower)
8004 {
8005         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8006         struct i40e_cloud_filter *filter = NULL;
8007         struct i40e_pf *pf = vsi->back;
8008         int err = 0;
8009
8010         if (tc < 0) {
8011                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8012                 return -EOPNOTSUPP;
8013         }
8014
8015         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8016             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8017                 return -EBUSY;
8018
8019         if (pf->fdir_pf_active_filters ||
8020             (!hlist_empty(&pf->fdir_filter_list))) {
8021                 dev_err(&vsi->back->pdev->dev,
8022                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8023                 return -EINVAL;
8024         }
8025
8026         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8027                 dev_err(&vsi->back->pdev->dev,
8028                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8029                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8030                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8031         }
8032
8033         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8034         if (!filter)
8035                 return -ENOMEM;
8036
8037         filter->cookie = cls_flower->cookie;
8038
8039         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8040         if (err < 0)
8041                 goto err;
8042
8043         err = i40e_handle_tclass(vsi, tc, filter);
8044         if (err < 0)
8045                 goto err;
8046
8047         /* Add cloud filter */
8048         if (filter->dst_port)
8049                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8050         else
8051                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8052
8053         if (err) {
8054                 dev_err(&pf->pdev->dev,
8055                         "Failed to add cloud filter, err %s\n",
8056                         i40e_stat_str(&pf->hw, err));
8057                 goto err;
8058         }
8059
8060         /* add filter to the ordered list */
8061         INIT_HLIST_NODE(&filter->cloud_node);
8062
8063         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8064
8065         pf->num_cloud_filters++;
8066
8067         return err;
8068 err:
8069         kfree(filter);
8070         return err;
8071 }
8072
8073 /**
8074  * i40e_find_cloud_filter - Find the could filter in the list
8075  * @vsi: Pointer to VSI
8076  * @cookie: filter specific cookie
8077  *
8078  **/
8079 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8080                                                         unsigned long *cookie)
8081 {
8082         struct i40e_cloud_filter *filter = NULL;
8083         struct hlist_node *node2;
8084
8085         hlist_for_each_entry_safe(filter, node2,
8086                                   &vsi->back->cloud_filter_list, cloud_node)
8087                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8088                         return filter;
8089         return NULL;
8090 }
8091
8092 /**
8093  * i40e_delete_clsflower - Remove tc flower filters
8094  * @vsi: Pointer to VSI
8095  * @cls_flower: Pointer to struct flow_cls_offload
8096  *
8097  **/
8098 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8099                                  struct flow_cls_offload *cls_flower)
8100 {
8101         struct i40e_cloud_filter *filter = NULL;
8102         struct i40e_pf *pf = vsi->back;
8103         int err = 0;
8104
8105         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8106
8107         if (!filter)
8108                 return -EINVAL;
8109
8110         hash_del(&filter->cloud_node);
8111
8112         if (filter->dst_port)
8113                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8114         else
8115                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8116
8117         kfree(filter);
8118         if (err) {
8119                 dev_err(&pf->pdev->dev,
8120                         "Failed to delete cloud filter, err %s\n",
8121                         i40e_stat_str(&pf->hw, err));
8122                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8123         }
8124
8125         pf->num_cloud_filters--;
8126         if (!pf->num_cloud_filters)
8127                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8128                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8129                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8130                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8131                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8132                 }
8133         return 0;
8134 }
8135
8136 /**
8137  * i40e_setup_tc_cls_flower - flower classifier offloads
8138  * @netdev: net device to configure
8139  * @type_data: offload data
8140  **/
8141 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8142                                     struct flow_cls_offload *cls_flower)
8143 {
8144         struct i40e_vsi *vsi = np->vsi;
8145
8146         switch (cls_flower->command) {
8147         case FLOW_CLS_REPLACE:
8148                 return i40e_configure_clsflower(vsi, cls_flower);
8149         case FLOW_CLS_DESTROY:
8150                 return i40e_delete_clsflower(vsi, cls_flower);
8151         case FLOW_CLS_STATS:
8152                 return -EOPNOTSUPP;
8153         default:
8154                 return -EOPNOTSUPP;
8155         }
8156 }
8157
8158 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8159                                   void *cb_priv)
8160 {
8161         struct i40e_netdev_priv *np = cb_priv;
8162
8163         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8164                 return -EOPNOTSUPP;
8165
8166         switch (type) {
8167         case TC_SETUP_CLSFLOWER:
8168                 return i40e_setup_tc_cls_flower(np, type_data);
8169
8170         default:
8171                 return -EOPNOTSUPP;
8172         }
8173 }
8174
8175 static LIST_HEAD(i40e_block_cb_list);
8176
8177 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8178                            void *type_data)
8179 {
8180         struct i40e_netdev_priv *np = netdev_priv(netdev);
8181
8182         switch (type) {
8183         case TC_SETUP_QDISC_MQPRIO:
8184                 return i40e_setup_tc(netdev, type_data);
8185         case TC_SETUP_BLOCK:
8186                 return flow_block_cb_setup_simple(type_data,
8187                                                   &i40e_block_cb_list,
8188                                                   i40e_setup_tc_block_cb,
8189                                                   np, np, true);
8190         default:
8191                 return -EOPNOTSUPP;
8192         }
8193 }
8194
8195 /**
8196  * i40e_open - Called when a network interface is made active
8197  * @netdev: network interface device structure
8198  *
8199  * The open entry point is called when a network interface is made
8200  * active by the system (IFF_UP).  At this point all resources needed
8201  * for transmit and receive operations are allocated, the interrupt
8202  * handler is registered with the OS, the netdev watchdog subtask is
8203  * enabled, and the stack is notified that the interface is ready.
8204  *
8205  * Returns 0 on success, negative value on failure
8206  **/
8207 int i40e_open(struct net_device *netdev)
8208 {
8209         struct i40e_netdev_priv *np = netdev_priv(netdev);
8210         struct i40e_vsi *vsi = np->vsi;
8211         struct i40e_pf *pf = vsi->back;
8212         int err;
8213
8214         /* disallow open during test or if eeprom is broken */
8215         if (test_bit(__I40E_TESTING, pf->state) ||
8216             test_bit(__I40E_BAD_EEPROM, pf->state))
8217                 return -EBUSY;
8218
8219         netif_carrier_off(netdev);
8220
8221         if (i40e_force_link_state(pf, true))
8222                 return -EAGAIN;
8223
8224         err = i40e_vsi_open(vsi);
8225         if (err)
8226                 return err;
8227
8228         /* configure global TSO hardware offload settings */
8229         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8230                                                        TCP_FLAG_FIN) >> 16);
8231         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8232                                                        TCP_FLAG_FIN |
8233                                                        TCP_FLAG_CWR) >> 16);
8234         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8235
8236         udp_tunnel_get_rx_info(netdev);
8237
8238         return 0;
8239 }
8240
8241 /**
8242  * i40e_vsi_open -
8243  * @vsi: the VSI to open
8244  *
8245  * Finish initialization of the VSI.
8246  *
8247  * Returns 0 on success, negative value on failure
8248  *
8249  * Note: expects to be called while under rtnl_lock()
8250  **/
8251 int i40e_vsi_open(struct i40e_vsi *vsi)
8252 {
8253         struct i40e_pf *pf = vsi->back;
8254         char int_name[I40E_INT_NAME_STR_LEN];
8255         int err;
8256
8257         /* allocate descriptors */
8258         err = i40e_vsi_setup_tx_resources(vsi);
8259         if (err)
8260                 goto err_setup_tx;
8261         err = i40e_vsi_setup_rx_resources(vsi);
8262         if (err)
8263                 goto err_setup_rx;
8264
8265         err = i40e_vsi_configure(vsi);
8266         if (err)
8267                 goto err_setup_rx;
8268
8269         if (vsi->netdev) {
8270                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8271                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8272                 err = i40e_vsi_request_irq(vsi, int_name);
8273                 if (err)
8274                         goto err_setup_rx;
8275
8276                 /* Notify the stack of the actual queue counts. */
8277                 err = netif_set_real_num_tx_queues(vsi->netdev,
8278                                                    vsi->num_queue_pairs);
8279                 if (err)
8280                         goto err_set_queues;
8281
8282                 err = netif_set_real_num_rx_queues(vsi->netdev,
8283                                                    vsi->num_queue_pairs);
8284                 if (err)
8285                         goto err_set_queues;
8286
8287         } else if (vsi->type == I40E_VSI_FDIR) {
8288                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8289                          dev_driver_string(&pf->pdev->dev),
8290                          dev_name(&pf->pdev->dev));
8291                 err = i40e_vsi_request_irq(vsi, int_name);
8292
8293         } else {
8294                 err = -EINVAL;
8295                 goto err_setup_rx;
8296         }
8297
8298         err = i40e_up_complete(vsi);
8299         if (err)
8300                 goto err_up_complete;
8301
8302         return 0;
8303
8304 err_up_complete:
8305         i40e_down(vsi);
8306 err_set_queues:
8307         i40e_vsi_free_irq(vsi);
8308 err_setup_rx:
8309         i40e_vsi_free_rx_resources(vsi);
8310 err_setup_tx:
8311         i40e_vsi_free_tx_resources(vsi);
8312         if (vsi == pf->vsi[pf->lan_vsi])
8313                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8314
8315         return err;
8316 }
8317
8318 /**
8319  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
8320  * @pf: Pointer to PF
8321  *
8322  * This function destroys the hlist where all the Flow Director
8323  * filters were saved.
8324  **/
8325 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8326 {
8327         struct i40e_fdir_filter *filter;
8328         struct i40e_flex_pit *pit_entry, *tmp;
8329         struct hlist_node *node2;
8330
8331         hlist_for_each_entry_safe(filter, node2,
8332                                   &pf->fdir_filter_list, fdir_node) {
8333                 hlist_del(&filter->fdir_node);
8334                 kfree(filter);
8335         }
8336
8337         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8338                 list_del(&pit_entry->list);
8339                 kfree(pit_entry);
8340         }
8341         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8342
8343         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8344                 list_del(&pit_entry->list);
8345                 kfree(pit_entry);
8346         }
8347         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8348
8349         pf->fdir_pf_active_filters = 0;
8350         pf->fd_tcp4_filter_cnt = 0;
8351         pf->fd_udp4_filter_cnt = 0;
8352         pf->fd_sctp4_filter_cnt = 0;
8353         pf->fd_ip4_filter_cnt = 0;
8354
8355         /* Reprogram the default input set for TCP/IPv4 */
8356         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8357                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8358                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8359
8360         /* Reprogram the default input set for UDP/IPv4 */
8361         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8362                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8363                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8364
8365         /* Reprogram the default input set for SCTP/IPv4 */
8366         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8367                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8368                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8369
8370         /* Reprogram the default input set for Other/IPv4 */
8371         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8372                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8373
8374         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8375                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8376 }
8377
8378 /**
8379  * i40e_cloud_filter_exit - Cleans up the cloud filters
8380  * @pf: Pointer to PF
8381  *
8382  * This function destroys the hlist where all the cloud filters
8383  * were saved.
8384  **/
8385 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8386 {
8387         struct i40e_cloud_filter *cfilter;
8388         struct hlist_node *node;
8389
8390         hlist_for_each_entry_safe(cfilter, node,
8391                                   &pf->cloud_filter_list, cloud_node) {
8392                 hlist_del(&cfilter->cloud_node);
8393                 kfree(cfilter);
8394         }
8395         pf->num_cloud_filters = 0;
8396
8397         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8398             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8399                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8400                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8401                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8402         }
8403 }
8404
8405 /**
8406  * i40e_close - Disables a network interface
8407  * @netdev: network interface device structure
8408  *
8409  * The close entry point is called when an interface is de-activated
8410  * by the OS.  The hardware is still under the driver's control, but
8411  * this netdev interface is disabled.
8412  *
8413  * Returns 0, this is not allowed to fail
8414  **/
8415 int i40e_close(struct net_device *netdev)
8416 {
8417         struct i40e_netdev_priv *np = netdev_priv(netdev);
8418         struct i40e_vsi *vsi = np->vsi;
8419
8420         i40e_vsi_close(vsi);
8421
8422         return 0;
8423 }
8424
8425 /**
8426  * i40e_do_reset - Start a PF or Core Reset sequence
8427  * @pf: board private structure
8428  * @reset_flags: which reset is requested
8429  * @lock_acquired: indicates whether or not the lock has been acquired
8430  * before this function was called.
8431  *
8432  * The essential difference in resets is that the PF Reset
8433  * doesn't clear the packet buffers, doesn't reset the PE
8434  * firmware, and doesn't bother the other PFs on the chip.
8435  **/
8436 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8437 {
8438         u32 val;
8439
8440         WARN_ON(in_interrupt());
8441
8442
8443         /* do the biggest reset indicated */
8444         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8445
8446                 /* Request a Global Reset
8447                  *
8448                  * This will start the chip's countdown to the actual full
8449                  * chip reset event, and a warning interrupt to be sent
8450                  * to all PFs, including the requestor.  Our handler
8451                  * for the warning interrupt will deal with the shutdown
8452                  * and recovery of the switch setup.
8453                  */
8454                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8455                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8456                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8457                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8458
8459         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8460
8461                 /* Request a Core Reset
8462                  *
8463                  * Same as Global Reset, except does *not* include the MAC/PHY
8464                  */
8465                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8466                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8467                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8468                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8469                 i40e_flush(&pf->hw);
8470
8471         } else if (reset_flags & I40E_PF_RESET_FLAG) {
8472
8473                 /* Request a PF Reset
8474                  *
8475                  * Resets only the PF-specific registers
8476                  *
8477                  * This goes directly to the tear-down and rebuild of
8478                  * the switch, since we need to do all the recovery as
8479                  * for the Core Reset.
8480                  */
8481                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8482                 i40e_handle_reset_warning(pf, lock_acquired);
8483
8484                 dev_info(&pf->pdev->dev,
8485                          pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
8486                          "FW LLDP is disabled\n" :
8487                          "FW LLDP is enabled\n");
8488
8489         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8490                 int v;
8491
8492                 /* Find the VSI(s) that requested a re-init */
8493                 dev_info(&pf->pdev->dev,
8494                          "VSI reinit requested\n");
8495                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8496                         struct i40e_vsi *vsi = pf->vsi[v];
8497
8498                         if (vsi != NULL &&
8499                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
8500                                                vsi->state))
8501                                 i40e_vsi_reinit_locked(pf->vsi[v]);
8502                 }
8503         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
8504                 int v;
8505
8506                 /* Find the VSI(s) that needs to be brought down */
8507                 dev_info(&pf->pdev->dev, "VSI down requested\n");
8508                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8509                         struct i40e_vsi *vsi = pf->vsi[v];
8510
8511                         if (vsi != NULL &&
8512                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
8513                                                vsi->state)) {
8514                                 set_bit(__I40E_VSI_DOWN, vsi->state);
8515                                 i40e_down(vsi);
8516                         }
8517                 }
8518         } else {
8519                 dev_info(&pf->pdev->dev,
8520                          "bad reset request 0x%08x\n", reset_flags);
8521         }
8522 }
8523
8524 #ifdef CONFIG_I40E_DCB
8525 /**
8526  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
8527  * @pf: board private structure
8528  * @old_cfg: current DCB config
8529  * @new_cfg: new DCB config
8530  **/
8531 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
8532                             struct i40e_dcbx_config *old_cfg,
8533                             struct i40e_dcbx_config *new_cfg)
8534 {
8535         bool need_reconfig = false;
8536
8537         /* Check if ETS configuration has changed */
8538         if (memcmp(&new_cfg->etscfg,
8539                    &old_cfg->etscfg,
8540                    sizeof(new_cfg->etscfg))) {
8541                 /* If Priority Table has changed reconfig is needed */
8542                 if (memcmp(&new_cfg->etscfg.prioritytable,
8543                            &old_cfg->etscfg.prioritytable,
8544                            sizeof(new_cfg->etscfg.prioritytable))) {
8545                         need_reconfig = true;
8546                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
8547                 }
8548
8549                 if (memcmp(&new_cfg->etscfg.tcbwtable,
8550                            &old_cfg->etscfg.tcbwtable,
8551                            sizeof(new_cfg->etscfg.tcbwtable)))
8552                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
8553
8554                 if (memcmp(&new_cfg->etscfg.tsatable,
8555                            &old_cfg->etscfg.tsatable,
8556                            sizeof(new_cfg->etscfg.tsatable)))
8557                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
8558         }
8559
8560         /* Check if PFC configuration has changed */
8561         if (memcmp(&new_cfg->pfc,
8562                    &old_cfg->pfc,
8563                    sizeof(new_cfg->pfc))) {
8564                 need_reconfig = true;
8565                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
8566         }
8567
8568         /* Check if APP Table has changed */
8569         if (memcmp(&new_cfg->app,
8570                    &old_cfg->app,
8571                    sizeof(new_cfg->app))) {
8572                 need_reconfig = true;
8573                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
8574         }
8575
8576         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
8577         return need_reconfig;
8578 }
8579
8580 /**
8581  * i40e_handle_lldp_event - Handle LLDP Change MIB event
8582  * @pf: board private structure
8583  * @e: event info posted on ARQ
8584  **/
8585 static int i40e_handle_lldp_event(struct i40e_pf *pf,
8586                                   struct i40e_arq_event_info *e)
8587 {
8588         struct i40e_aqc_lldp_get_mib *mib =
8589                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
8590         struct i40e_hw *hw = &pf->hw;
8591         struct i40e_dcbx_config tmp_dcbx_cfg;
8592         bool need_reconfig = false;
8593         int ret = 0;
8594         u8 type;
8595
8596         /* Not DCB capable or capability disabled */
8597         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
8598                 return ret;
8599
8600         /* Ignore if event is not for Nearest Bridge */
8601         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
8602                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
8603         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
8604         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
8605                 return ret;
8606
8607         /* Check MIB Type and return if event for Remote MIB update */
8608         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
8609         dev_dbg(&pf->pdev->dev,
8610                 "LLDP event mib type %s\n", type ? "remote" : "local");
8611         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
8612                 /* Update the remote cached instance and return */
8613                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
8614                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
8615                                 &hw->remote_dcbx_config);
8616                 goto exit;
8617         }
8618
8619         /* Store the old configuration */
8620         tmp_dcbx_cfg = hw->local_dcbx_config;
8621
8622         /* Reset the old DCBx configuration data */
8623         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
8624         /* Get updated DCBX data from firmware */
8625         ret = i40e_get_dcb_config(&pf->hw);
8626         if (ret) {
8627                 dev_info(&pf->pdev->dev,
8628                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
8629                          i40e_stat_str(&pf->hw, ret),
8630                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8631                 goto exit;
8632         }
8633
8634         /* No change detected in DCBX configs */
8635         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
8636                     sizeof(tmp_dcbx_cfg))) {
8637                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
8638                 goto exit;
8639         }
8640
8641         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
8642                                                &hw->local_dcbx_config);
8643
8644         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
8645
8646         if (!need_reconfig)
8647                 goto exit;
8648
8649         /* Enable DCB tagging only when more than one TC */
8650         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
8651                 pf->flags |= I40E_FLAG_DCB_ENABLED;
8652         else
8653                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8654
8655         set_bit(__I40E_PORT_SUSPENDED, pf->state);
8656         /* Reconfiguration needed quiesce all VSIs */
8657         i40e_pf_quiesce_all_vsi(pf);
8658
8659         /* Changes in configuration update VEB/VSI */
8660         i40e_dcb_reconfigure(pf);
8661
8662         ret = i40e_resume_port_tx(pf);
8663
8664         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
8665         /* In case of error no point in resuming VSIs */
8666         if (ret)
8667                 goto exit;
8668
8669         /* Wait for the PF's queues to be disabled */
8670         ret = i40e_pf_wait_queues_disabled(pf);
8671         if (ret) {
8672                 /* Schedule PF reset to recover */
8673                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8674                 i40e_service_event_schedule(pf);
8675         } else {
8676                 i40e_pf_unquiesce_all_vsi(pf);
8677                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
8678                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
8679         }
8680
8681 exit:
8682         return ret;
8683 }
8684 #endif /* CONFIG_I40E_DCB */
8685
8686 /**
8687  * i40e_do_reset_safe - Protected reset path for userland calls.
8688  * @pf: board private structure
8689  * @reset_flags: which reset is requested
8690  *
8691  **/
8692 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
8693 {
8694         rtnl_lock();
8695         i40e_do_reset(pf, reset_flags, true);
8696         rtnl_unlock();
8697 }
8698
8699 /**
8700  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
8701  * @pf: board private structure
8702  * @e: event info posted on ARQ
8703  *
8704  * Handler for LAN Queue Overflow Event generated by the firmware for PF
8705  * and VF queues
8706  **/
8707 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
8708                                            struct i40e_arq_event_info *e)
8709 {
8710         struct i40e_aqc_lan_overflow *data =
8711                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
8712         u32 queue = le32_to_cpu(data->prtdcb_rupto);
8713         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
8714         struct i40e_hw *hw = &pf->hw;
8715         struct i40e_vf *vf;
8716         u16 vf_id;
8717
8718         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
8719                 queue, qtx_ctl);
8720
8721         /* Queue belongs to VF, find the VF and issue VF reset */
8722         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
8723             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
8724                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
8725                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
8726                 vf_id -= hw->func_caps.vf_base_id;
8727                 vf = &pf->vf[vf_id];
8728                 i40e_vc_notify_vf_reset(vf);
8729                 /* Allow VF to process pending reset notification */
8730                 msleep(20);
8731                 i40e_reset_vf(vf, false);
8732         }
8733 }
8734
8735 /**
8736  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
8737  * @pf: board private structure
8738  **/
8739 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
8740 {
8741         u32 val, fcnt_prog;
8742
8743         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8744         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
8745         return fcnt_prog;
8746 }
8747
8748 /**
8749  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
8750  * @pf: board private structure
8751  **/
8752 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
8753 {
8754         u32 val, fcnt_prog;
8755
8756         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8757         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
8758                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
8759                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
8760         return fcnt_prog;
8761 }
8762
8763 /**
8764  * i40e_get_global_fd_count - Get total FD filters programmed on device
8765  * @pf: board private structure
8766  **/
8767 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
8768 {
8769         u32 val, fcnt_prog;
8770
8771         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
8772         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
8773                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
8774                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
8775         return fcnt_prog;
8776 }
8777
8778 /**
8779  * i40e_reenable_fdir_sb - Restore FDir SB capability
8780  * @pf: board private structure
8781  **/
8782 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
8783 {
8784         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
8785                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8786                     (I40E_DEBUG_FD & pf->hw.debug_mask))
8787                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
8788 }
8789
8790 /**
8791  * i40e_reenable_fdir_atr - Restore FDir ATR capability
8792  * @pf: board private structure
8793  **/
8794 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
8795 {
8796         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
8797                 /* ATR uses the same filtering logic as SB rules. It only
8798                  * functions properly if the input set mask is at the default
8799                  * settings. It is safe to restore the default input set
8800                  * because there are no active TCPv4 filter rules.
8801                  */
8802                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8803                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8804                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8805
8806                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8807                     (I40E_DEBUG_FD & pf->hw.debug_mask))
8808                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
8809         }
8810 }
8811
8812 /**
8813  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
8814  * @pf: board private structure
8815  * @filter: FDir filter to remove
8816  */
8817 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
8818                                        struct i40e_fdir_filter *filter)
8819 {
8820         /* Update counters */
8821         pf->fdir_pf_active_filters--;
8822         pf->fd_inv = 0;
8823
8824         switch (filter->flow_type) {
8825         case TCP_V4_FLOW:
8826                 pf->fd_tcp4_filter_cnt--;
8827                 break;
8828         case UDP_V4_FLOW:
8829                 pf->fd_udp4_filter_cnt--;
8830                 break;
8831         case SCTP_V4_FLOW:
8832                 pf->fd_sctp4_filter_cnt--;
8833                 break;
8834         case IP_USER_FLOW:
8835                 switch (filter->ip4_proto) {
8836                 case IPPROTO_TCP:
8837                         pf->fd_tcp4_filter_cnt--;
8838                         break;
8839                 case IPPROTO_UDP:
8840                         pf->fd_udp4_filter_cnt--;
8841                         break;
8842                 case IPPROTO_SCTP:
8843                         pf->fd_sctp4_filter_cnt--;
8844                         break;
8845                 case IPPROTO_IP:
8846                         pf->fd_ip4_filter_cnt--;
8847                         break;
8848                 }
8849                 break;
8850         }
8851
8852         /* Remove the filter from the list and free memory */
8853         hlist_del(&filter->fdir_node);
8854         kfree(filter);
8855 }
8856
8857 /**
8858  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
8859  * @pf: board private structure
8860  **/
8861 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
8862 {
8863         struct i40e_fdir_filter *filter;
8864         u32 fcnt_prog, fcnt_avail;
8865         struct hlist_node *node;
8866
8867         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8868                 return;
8869
8870         /* Check if we have enough room to re-enable FDir SB capability. */
8871         fcnt_prog = i40e_get_global_fd_count(pf);
8872         fcnt_avail = pf->fdir_pf_filter_count;
8873         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
8874             (pf->fd_add_err == 0) ||
8875             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
8876                 i40e_reenable_fdir_sb(pf);
8877
8878         /* We should wait for even more space before re-enabling ATR.
8879          * Additionally, we cannot enable ATR as long as we still have TCP SB
8880          * rules active.
8881          */
8882         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
8883             (pf->fd_tcp4_filter_cnt == 0))
8884                 i40e_reenable_fdir_atr(pf);
8885
8886         /* if hw had a problem adding a filter, delete it */
8887         if (pf->fd_inv > 0) {
8888                 hlist_for_each_entry_safe(filter, node,
8889                                           &pf->fdir_filter_list, fdir_node)
8890                         if (filter->fd_id == pf->fd_inv)
8891                                 i40e_delete_invalid_filter(pf, filter);
8892         }
8893 }
8894
8895 #define I40E_MIN_FD_FLUSH_INTERVAL 10
8896 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
8897 /**
8898  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
8899  * @pf: board private structure
8900  **/
8901 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
8902 {
8903         unsigned long min_flush_time;
8904         int flush_wait_retry = 50;
8905         bool disable_atr = false;
8906         int fd_room;
8907         int reg;
8908
8909         if (!time_after(jiffies, pf->fd_flush_timestamp +
8910                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
8911                 return;
8912
8913         /* If the flush is happening too quick and we have mostly SB rules we
8914          * should not re-enable ATR for some time.
8915          */
8916         min_flush_time = pf->fd_flush_timestamp +
8917                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
8918         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
8919
8920         if (!(time_after(jiffies, min_flush_time)) &&
8921             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
8922                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8923                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
8924                 disable_atr = true;
8925         }
8926
8927         pf->fd_flush_timestamp = jiffies;
8928         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8929         /* flush all filters */
8930         wr32(&pf->hw, I40E_PFQF_CTL_1,
8931              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
8932         i40e_flush(&pf->hw);
8933         pf->fd_flush_cnt++;
8934         pf->fd_add_err = 0;
8935         do {
8936                 /* Check FD flush status every 5-6msec */
8937                 usleep_range(5000, 6000);
8938                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
8939                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
8940                         break;
8941         } while (flush_wait_retry--);
8942         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
8943                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
8944         } else {
8945                 /* replay sideband filters */
8946                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
8947                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
8948                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8949                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
8950                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8951                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
8952         }
8953 }
8954
8955 /**
8956  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
8957  * @pf: board private structure
8958  **/
8959 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
8960 {
8961         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
8962 }
8963
8964 /* We can see up to 256 filter programming desc in transit if the filters are
8965  * being applied really fast; before we see the first
8966  * filter miss error on Rx queue 0. Accumulating enough error messages before
8967  * reacting will make sure we don't cause flush too often.
8968  */
8969 #define I40E_MAX_FD_PROGRAM_ERROR 256
8970
8971 /**
8972  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
8973  * @pf: board private structure
8974  **/
8975 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
8976 {
8977
8978         /* if interface is down do nothing */
8979         if (test_bit(__I40E_DOWN, pf->state))
8980                 return;
8981
8982         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8983                 i40e_fdir_flush_and_replay(pf);
8984
8985         i40e_fdir_check_and_reenable(pf);
8986
8987 }
8988
8989 /**
8990  * i40e_vsi_link_event - notify VSI of a link event
8991  * @vsi: vsi to be notified
8992  * @link_up: link up or down
8993  **/
8994 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
8995 {
8996         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
8997                 return;
8998
8999         switch (vsi->type) {
9000         case I40E_VSI_MAIN:
9001                 if (!vsi->netdev || !vsi->netdev_registered)
9002                         break;
9003
9004                 if (link_up) {
9005                         netif_carrier_on(vsi->netdev);
9006                         netif_tx_wake_all_queues(vsi->netdev);
9007                 } else {
9008                         netif_carrier_off(vsi->netdev);
9009                         netif_tx_stop_all_queues(vsi->netdev);
9010                 }
9011                 break;
9012
9013         case I40E_VSI_SRIOV:
9014         case I40E_VSI_VMDQ2:
9015         case I40E_VSI_CTRL:
9016         case I40E_VSI_IWARP:
9017         case I40E_VSI_MIRROR:
9018         default:
9019                 /* there is no notification for other VSIs */
9020                 break;
9021         }
9022 }
9023
9024 /**
9025  * i40e_veb_link_event - notify elements on the veb of a link event
9026  * @veb: veb to be notified
9027  * @link_up: link up or down
9028  **/
9029 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9030 {
9031         struct i40e_pf *pf;
9032         int i;
9033
9034         if (!veb || !veb->pf)
9035                 return;
9036         pf = veb->pf;
9037
9038         /* depth first... */
9039         for (i = 0; i < I40E_MAX_VEB; i++)
9040                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9041                         i40e_veb_link_event(pf->veb[i], link_up);
9042
9043         /* ... now the local VSIs */
9044         for (i = 0; i < pf->num_alloc_vsi; i++)
9045                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9046                         i40e_vsi_link_event(pf->vsi[i], link_up);
9047 }
9048
9049 /**
9050  * i40e_link_event - Update netif_carrier status
9051  * @pf: board private structure
9052  **/
9053 static void i40e_link_event(struct i40e_pf *pf)
9054 {
9055         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9056         u8 new_link_speed, old_link_speed;
9057         i40e_status status;
9058         bool new_link, old_link;
9059
9060         /* set this to force the get_link_status call to refresh state */
9061         pf->hw.phy.get_link_info = true;
9062         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9063         status = i40e_get_link_status(&pf->hw, &new_link);
9064
9065         /* On success, disable temp link polling */
9066         if (status == I40E_SUCCESS) {
9067                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9068         } else {
9069                 /* Enable link polling temporarily until i40e_get_link_status
9070                  * returns I40E_SUCCESS
9071                  */
9072                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9073                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9074                         status);
9075                 return;
9076         }
9077
9078         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9079         new_link_speed = pf->hw.phy.link_info.link_speed;
9080
9081         if (new_link == old_link &&
9082             new_link_speed == old_link_speed &&
9083             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9084              new_link == netif_carrier_ok(vsi->netdev)))
9085                 return;
9086
9087         i40e_print_link_message(vsi, new_link);
9088
9089         /* Notify the base of the switch tree connected to
9090          * the link.  Floating VEBs are not notified.
9091          */
9092         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9093                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9094         else
9095                 i40e_vsi_link_event(vsi, new_link);
9096
9097         if (pf->vf)
9098                 i40e_vc_notify_link_state(pf);
9099
9100         if (pf->flags & I40E_FLAG_PTP)
9101                 i40e_ptp_set_increment(pf);
9102 }
9103
9104 /**
9105  * i40e_watchdog_subtask - periodic checks not using event driven response
9106  * @pf: board private structure
9107  **/
9108 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9109 {
9110         int i;
9111
9112         /* if interface is down do nothing */
9113         if (test_bit(__I40E_DOWN, pf->state) ||
9114             test_bit(__I40E_CONFIG_BUSY, pf->state))
9115                 return;
9116
9117         /* make sure we don't do these things too often */
9118         if (time_before(jiffies, (pf->service_timer_previous +
9119                                   pf->service_timer_period)))
9120                 return;
9121         pf->service_timer_previous = jiffies;
9122
9123         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9124             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9125                 i40e_link_event(pf);
9126
9127         /* Update the stats for active netdevs so the network stack
9128          * can look at updated numbers whenever it cares to
9129          */
9130         for (i = 0; i < pf->num_alloc_vsi; i++)
9131                 if (pf->vsi[i] && pf->vsi[i]->netdev)
9132                         i40e_update_stats(pf->vsi[i]);
9133
9134         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9135                 /* Update the stats for the active switching components */
9136                 for (i = 0; i < I40E_MAX_VEB; i++)
9137                         if (pf->veb[i])
9138                                 i40e_update_veb_stats(pf->veb[i]);
9139         }
9140
9141         i40e_ptp_rx_hang(pf);
9142         i40e_ptp_tx_hang(pf);
9143 }
9144
9145 /**
9146  * i40e_reset_subtask - Set up for resetting the device and driver
9147  * @pf: board private structure
9148  **/
9149 static void i40e_reset_subtask(struct i40e_pf *pf)
9150 {
9151         u32 reset_flags = 0;
9152
9153         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9154                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9155                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9156         }
9157         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9158                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9159                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9160         }
9161         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9162                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9163                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9164         }
9165         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9166                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9167                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9168         }
9169         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9170                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9171                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9172         }
9173
9174         /* If there's a recovery already waiting, it takes
9175          * precedence before starting a new reset sequence.
9176          */
9177         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9178                 i40e_prep_for_reset(pf, false);
9179                 i40e_reset(pf);
9180                 i40e_rebuild(pf, false, false);
9181         }
9182
9183         /* If we're already down or resetting, just bail */
9184         if (reset_flags &&
9185             !test_bit(__I40E_DOWN, pf->state) &&
9186             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9187                 i40e_do_reset(pf, reset_flags, false);
9188         }
9189 }
9190
9191 /**
9192  * i40e_handle_link_event - Handle link event
9193  * @pf: board private structure
9194  * @e: event info posted on ARQ
9195  **/
9196 static void i40e_handle_link_event(struct i40e_pf *pf,
9197                                    struct i40e_arq_event_info *e)
9198 {
9199         struct i40e_aqc_get_link_status *status =
9200                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9201
9202         /* Do a new status request to re-enable LSE reporting
9203          * and load new status information into the hw struct
9204          * This completely ignores any state information
9205          * in the ARQ event info, instead choosing to always
9206          * issue the AQ update link status command.
9207          */
9208         i40e_link_event(pf);
9209
9210         /* Check if module meets thermal requirements */
9211         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9212                 dev_err(&pf->pdev->dev,
9213                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9214                 dev_err(&pf->pdev->dev,
9215                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9216         } else {
9217                 /* check for unqualified module, if link is down, suppress
9218                  * the message if link was forced to be down.
9219                  */
9220                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9221                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9222                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
9223                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9224                         dev_err(&pf->pdev->dev,
9225                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9226                         dev_err(&pf->pdev->dev,
9227                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9228                 }
9229         }
9230 }
9231
9232 /**
9233  * i40e_clean_adminq_subtask - Clean the AdminQ rings
9234  * @pf: board private structure
9235  **/
9236 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9237 {
9238         struct i40e_arq_event_info event;
9239         struct i40e_hw *hw = &pf->hw;
9240         u16 pending, i = 0;
9241         i40e_status ret;
9242         u16 opcode;
9243         u32 oldval;
9244         u32 val;
9245
9246         /* Do not run clean AQ when PF reset fails */
9247         if (test_bit(__I40E_RESET_FAILED, pf->state))
9248                 return;
9249
9250         /* check for error indications */
9251         val = rd32(&pf->hw, pf->hw.aq.arq.len);
9252         oldval = val;
9253         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9254                 if (hw->debug_mask & I40E_DEBUG_AQ)
9255                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9256                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9257         }
9258         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9259                 if (hw->debug_mask & I40E_DEBUG_AQ)
9260                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9261                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9262                 pf->arq_overflows++;
9263         }
9264         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9265                 if (hw->debug_mask & I40E_DEBUG_AQ)
9266                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9267                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9268         }
9269         if (oldval != val)
9270                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9271
9272         val = rd32(&pf->hw, pf->hw.aq.asq.len);
9273         oldval = val;
9274         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9275                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9276                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9277                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9278         }
9279         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9280                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9281                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9282                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9283         }
9284         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9285                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9286                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9287                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9288         }
9289         if (oldval != val)
9290                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9291
9292         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9293         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9294         if (!event.msg_buf)
9295                 return;
9296
9297         do {
9298                 ret = i40e_clean_arq_element(hw, &event, &pending);
9299                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9300                         break;
9301                 else if (ret) {
9302                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9303                         break;
9304                 }
9305
9306                 opcode = le16_to_cpu(event.desc.opcode);
9307                 switch (opcode) {
9308
9309                 case i40e_aqc_opc_get_link_status:
9310                         i40e_handle_link_event(pf, &event);
9311                         break;
9312                 case i40e_aqc_opc_send_msg_to_pf:
9313                         ret = i40e_vc_process_vf_msg(pf,
9314                                         le16_to_cpu(event.desc.retval),
9315                                         le32_to_cpu(event.desc.cookie_high),
9316                                         le32_to_cpu(event.desc.cookie_low),
9317                                         event.msg_buf,
9318                                         event.msg_len);
9319                         break;
9320                 case i40e_aqc_opc_lldp_update_mib:
9321                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9322 #ifdef CONFIG_I40E_DCB
9323                         rtnl_lock();
9324                         ret = i40e_handle_lldp_event(pf, &event);
9325                         rtnl_unlock();
9326 #endif /* CONFIG_I40E_DCB */
9327                         break;
9328                 case i40e_aqc_opc_event_lan_overflow:
9329                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9330                         i40e_handle_lan_overflow_event(pf, &event);
9331                         break;
9332                 case i40e_aqc_opc_send_msg_to_peer:
9333                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9334                         break;
9335                 case i40e_aqc_opc_nvm_erase:
9336                 case i40e_aqc_opc_nvm_update:
9337                 case i40e_aqc_opc_oem_post_update:
9338                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9339                                    "ARQ NVM operation 0x%04x completed\n",
9340                                    opcode);
9341                         break;
9342                 default:
9343                         dev_info(&pf->pdev->dev,
9344                                  "ARQ: Unknown event 0x%04x ignored\n",
9345                                  opcode);
9346                         break;
9347                 }
9348         } while (i++ < pf->adminq_work_limit);
9349
9350         if (i < pf->adminq_work_limit)
9351                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9352
9353         /* re-enable Admin queue interrupt cause */
9354         val = rd32(hw, I40E_PFINT_ICR0_ENA);
9355         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9356         wr32(hw, I40E_PFINT_ICR0_ENA, val);
9357         i40e_flush(hw);
9358
9359         kfree(event.msg_buf);
9360 }
9361
9362 /**
9363  * i40e_verify_eeprom - make sure eeprom is good to use
9364  * @pf: board private structure
9365  **/
9366 static void i40e_verify_eeprom(struct i40e_pf *pf)
9367 {
9368         int err;
9369
9370         err = i40e_diag_eeprom_test(&pf->hw);
9371         if (err) {
9372                 /* retry in case of garbage read */
9373                 err = i40e_diag_eeprom_test(&pf->hw);
9374                 if (err) {
9375                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9376                                  err);
9377                         set_bit(__I40E_BAD_EEPROM, pf->state);
9378                 }
9379         }
9380
9381         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9382                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9383                 clear_bit(__I40E_BAD_EEPROM, pf->state);
9384         }
9385 }
9386
9387 /**
9388  * i40e_enable_pf_switch_lb
9389  * @pf: pointer to the PF structure
9390  *
9391  * enable switch loop back or die - no point in a return value
9392  **/
9393 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
9394 {
9395         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9396         struct i40e_vsi_context ctxt;
9397         int ret;
9398
9399         ctxt.seid = pf->main_vsi_seid;
9400         ctxt.pf_num = pf->hw.pf_id;
9401         ctxt.vf_num = 0;
9402         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9403         if (ret) {
9404                 dev_info(&pf->pdev->dev,
9405                          "couldn't get PF vsi config, err %s aq_err %s\n",
9406                          i40e_stat_str(&pf->hw, ret),
9407                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9408                 return;
9409         }
9410         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9411         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9412         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9413
9414         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9415         if (ret) {
9416                 dev_info(&pf->pdev->dev,
9417                          "update vsi switch failed, err %s aq_err %s\n",
9418                          i40e_stat_str(&pf->hw, ret),
9419                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9420         }
9421 }
9422
9423 /**
9424  * i40e_disable_pf_switch_lb
9425  * @pf: pointer to the PF structure
9426  *
9427  * disable switch loop back or die - no point in a return value
9428  **/
9429 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
9430 {
9431         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9432         struct i40e_vsi_context ctxt;
9433         int ret;
9434
9435         ctxt.seid = pf->main_vsi_seid;
9436         ctxt.pf_num = pf->hw.pf_id;
9437         ctxt.vf_num = 0;
9438         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9439         if (ret) {
9440                 dev_info(&pf->pdev->dev,
9441                          "couldn't get PF vsi config, err %s aq_err %s\n",
9442                          i40e_stat_str(&pf->hw, ret),
9443                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9444                 return;
9445         }
9446         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9447         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9448         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9449
9450         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9451         if (ret) {
9452                 dev_info(&pf->pdev->dev,
9453                          "update vsi switch failed, err %s aq_err %s\n",
9454                          i40e_stat_str(&pf->hw, ret),
9455                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9456         }
9457 }
9458
9459 /**
9460  * i40e_config_bridge_mode - Configure the HW bridge mode
9461  * @veb: pointer to the bridge instance
9462  *
9463  * Configure the loop back mode for the LAN VSI that is downlink to the
9464  * specified HW bridge instance. It is expected this function is called
9465  * when a new HW bridge is instantiated.
9466  **/
9467 static void i40e_config_bridge_mode(struct i40e_veb *veb)
9468 {
9469         struct i40e_pf *pf = veb->pf;
9470
9471         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
9472                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
9473                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9474         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
9475                 i40e_disable_pf_switch_lb(pf);
9476         else
9477                 i40e_enable_pf_switch_lb(pf);
9478 }
9479
9480 /**
9481  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
9482  * @veb: pointer to the VEB instance
9483  *
9484  * This is a recursive function that first builds the attached VSIs then
9485  * recurses in to build the next layer of VEB.  We track the connections
9486  * through our own index numbers because the seid's from the HW could
9487  * change across the reset.
9488  **/
9489 static int i40e_reconstitute_veb(struct i40e_veb *veb)
9490 {
9491         struct i40e_vsi *ctl_vsi = NULL;
9492         struct i40e_pf *pf = veb->pf;
9493         int v, veb_idx;
9494         int ret;
9495
9496         /* build VSI that owns this VEB, temporarily attached to base VEB */
9497         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
9498                 if (pf->vsi[v] &&
9499                     pf->vsi[v]->veb_idx == veb->idx &&
9500                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
9501                         ctl_vsi = pf->vsi[v];
9502                         break;
9503                 }
9504         }
9505         if (!ctl_vsi) {
9506                 dev_info(&pf->pdev->dev,
9507                          "missing owner VSI for veb_idx %d\n", veb->idx);
9508                 ret = -ENOENT;
9509                 goto end_reconstitute;
9510         }
9511         if (ctl_vsi != pf->vsi[pf->lan_vsi])
9512                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
9513         ret = i40e_add_vsi(ctl_vsi);
9514         if (ret) {
9515                 dev_info(&pf->pdev->dev,
9516                          "rebuild of veb_idx %d owner VSI failed: %d\n",
9517                          veb->idx, ret);
9518                 goto end_reconstitute;
9519         }
9520         i40e_vsi_reset_stats(ctl_vsi);
9521
9522         /* create the VEB in the switch and move the VSI onto the VEB */
9523         ret = i40e_add_veb(veb, ctl_vsi);
9524         if (ret)
9525                 goto end_reconstitute;
9526
9527         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
9528                 veb->bridge_mode = BRIDGE_MODE_VEB;
9529         else
9530                 veb->bridge_mode = BRIDGE_MODE_VEPA;
9531         i40e_config_bridge_mode(veb);
9532
9533         /* create the remaining VSIs attached to this VEB */
9534         for (v = 0; v < pf->num_alloc_vsi; v++) {
9535                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
9536                         continue;
9537
9538                 if (pf->vsi[v]->veb_idx == veb->idx) {
9539                         struct i40e_vsi *vsi = pf->vsi[v];
9540
9541                         vsi->uplink_seid = veb->seid;
9542                         ret = i40e_add_vsi(vsi);
9543                         if (ret) {
9544                                 dev_info(&pf->pdev->dev,
9545                                          "rebuild of vsi_idx %d failed: %d\n",
9546                                          v, ret);
9547                                 goto end_reconstitute;
9548                         }
9549                         i40e_vsi_reset_stats(vsi);
9550                 }
9551         }
9552
9553         /* create any VEBs attached to this VEB - RECURSION */
9554         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
9555                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
9556                         pf->veb[veb_idx]->uplink_seid = veb->seid;
9557                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
9558                         if (ret)
9559                                 break;
9560                 }
9561         }
9562
9563 end_reconstitute:
9564         return ret;
9565 }
9566
9567 /**
9568  * i40e_get_capabilities - get info about the HW
9569  * @pf: the PF struct
9570  **/
9571 static int i40e_get_capabilities(struct i40e_pf *pf,
9572                                  enum i40e_admin_queue_opc list_type)
9573 {
9574         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
9575         u16 data_size;
9576         int buf_len;
9577         int err;
9578
9579         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
9580         do {
9581                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
9582                 if (!cap_buf)
9583                         return -ENOMEM;
9584
9585                 /* this loads the data into the hw struct for us */
9586                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
9587                                                     &data_size, list_type,
9588                                                     NULL);
9589                 /* data loaded, buffer no longer needed */
9590                 kfree(cap_buf);
9591
9592                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
9593                         /* retry with a larger buffer */
9594                         buf_len = data_size;
9595                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
9596                         dev_info(&pf->pdev->dev,
9597                                  "capability discovery failed, err %s aq_err %s\n",
9598                                  i40e_stat_str(&pf->hw, err),
9599                                  i40e_aq_str(&pf->hw,
9600                                              pf->hw.aq.asq_last_status));
9601                         return -ENODEV;
9602                 }
9603         } while (err);
9604
9605         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
9606                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9607                         dev_info(&pf->pdev->dev,
9608                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
9609                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
9610                                  pf->hw.func_caps.num_msix_vectors,
9611                                  pf->hw.func_caps.num_msix_vectors_vf,
9612                                  pf->hw.func_caps.fd_filters_guaranteed,
9613                                  pf->hw.func_caps.fd_filters_best_effort,
9614                                  pf->hw.func_caps.num_tx_qp,
9615                                  pf->hw.func_caps.num_vsis);
9616                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
9617                         dev_info(&pf->pdev->dev,
9618                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
9619                                  pf->hw.dev_caps.switch_mode,
9620                                  pf->hw.dev_caps.valid_functions);
9621                         dev_info(&pf->pdev->dev,
9622                                  "SR-IOV=%d, num_vfs for all function=%u\n",
9623                                  pf->hw.dev_caps.sr_iov_1_1,
9624                                  pf->hw.dev_caps.num_vfs);
9625                         dev_info(&pf->pdev->dev,
9626                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
9627                                  pf->hw.dev_caps.num_vsis,
9628                                  pf->hw.dev_caps.num_rx_qp,
9629                                  pf->hw.dev_caps.num_tx_qp);
9630                 }
9631         }
9632         if (list_type == i40e_aqc_opc_list_func_capabilities) {
9633 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
9634                        + pf->hw.func_caps.num_vfs)
9635                 if (pf->hw.revision_id == 0 &&
9636                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
9637                         dev_info(&pf->pdev->dev,
9638                                  "got num_vsis %d, setting num_vsis to %d\n",
9639                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
9640                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
9641                 }
9642         }
9643         return 0;
9644 }
9645
9646 static int i40e_vsi_clear(struct i40e_vsi *vsi);
9647
9648 /**
9649  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
9650  * @pf: board private structure
9651  **/
9652 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
9653 {
9654         struct i40e_vsi *vsi;
9655
9656         /* quick workaround for an NVM issue that leaves a critical register
9657          * uninitialized
9658          */
9659         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
9660                 static const u32 hkey[] = {
9661                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
9662                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
9663                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
9664                         0x95b3a76d};
9665                 int i;
9666
9667                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
9668                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
9669         }
9670
9671         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9672                 return;
9673
9674         /* find existing VSI and see if it needs configuring */
9675         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9676
9677         /* create a new VSI if none exists */
9678         if (!vsi) {
9679                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
9680                                      pf->vsi[pf->lan_vsi]->seid, 0);
9681                 if (!vsi) {
9682                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
9683                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
9684                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
9685                         return;
9686                 }
9687         }
9688
9689         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
9690 }
9691
9692 /**
9693  * i40e_fdir_teardown - release the Flow Director resources
9694  * @pf: board private structure
9695  **/
9696 static void i40e_fdir_teardown(struct i40e_pf *pf)
9697 {
9698         struct i40e_vsi *vsi;
9699
9700         i40e_fdir_filter_exit(pf);
9701         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9702         if (vsi)
9703                 i40e_vsi_release(vsi);
9704 }
9705
9706 /**
9707  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
9708  * @vsi: PF main vsi
9709  * @seid: seid of main or channel VSIs
9710  *
9711  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
9712  * existed before reset
9713  **/
9714 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
9715 {
9716         struct i40e_cloud_filter *cfilter;
9717         struct i40e_pf *pf = vsi->back;
9718         struct hlist_node *node;
9719         i40e_status ret;
9720
9721         /* Add cloud filters back if they exist */
9722         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
9723                                   cloud_node) {
9724                 if (cfilter->seid != seid)
9725                         continue;
9726
9727                 if (cfilter->dst_port)
9728                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
9729                                                                 true);
9730                 else
9731                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
9732
9733                 if (ret) {
9734                         dev_dbg(&pf->pdev->dev,
9735                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
9736                                 i40e_stat_str(&pf->hw, ret),
9737                                 i40e_aq_str(&pf->hw,
9738                                             pf->hw.aq.asq_last_status));
9739                         return ret;
9740                 }
9741         }
9742         return 0;
9743 }
9744
9745 /**
9746  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
9747  * @vsi: PF main vsi
9748  *
9749  * Rebuilds channel VSIs if they existed before reset
9750  **/
9751 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
9752 {
9753         struct i40e_channel *ch, *ch_tmp;
9754         i40e_status ret;
9755
9756         if (list_empty(&vsi->ch_list))
9757                 return 0;
9758
9759         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
9760                 if (!ch->initialized)
9761                         break;
9762                 /* Proceed with creation of channel (VMDq2) VSI */
9763                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
9764                 if (ret) {
9765                         dev_info(&vsi->back->pdev->dev,
9766                                  "failed to rebuild channels using uplink_seid %u\n",
9767                                  vsi->uplink_seid);
9768                         return ret;
9769                 }
9770                 /* Reconfigure TX queues using QTX_CTL register */
9771                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
9772                 if (ret) {
9773                         dev_info(&vsi->back->pdev->dev,
9774                                  "failed to configure TX rings for channel %u\n",
9775                                  ch->seid);
9776                         return ret;
9777                 }
9778                 /* update 'next_base_queue' */
9779                 vsi->next_base_queue = vsi->next_base_queue +
9780                                                         ch->num_queue_pairs;
9781                 if (ch->max_tx_rate) {
9782                         u64 credits = ch->max_tx_rate;
9783
9784                         if (i40e_set_bw_limit(vsi, ch->seid,
9785                                               ch->max_tx_rate))
9786                                 return -EINVAL;
9787
9788                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
9789                         dev_dbg(&vsi->back->pdev->dev,
9790                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9791                                 ch->max_tx_rate,
9792                                 credits,
9793                                 ch->seid);
9794                 }
9795                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
9796                 if (ret) {
9797                         dev_dbg(&vsi->back->pdev->dev,
9798                                 "Failed to rebuild cloud filters for channel VSI %u\n",
9799                                 ch->seid);
9800                         return ret;
9801                 }
9802         }
9803         return 0;
9804 }
9805
9806 /**
9807  * i40e_prep_for_reset - prep for the core to reset
9808  * @pf: board private structure
9809  * @lock_acquired: indicates whether or not the lock has been acquired
9810  * before this function was called.
9811  *
9812  * Close up the VFs and other things in prep for PF Reset.
9813   **/
9814 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
9815 {
9816         struct i40e_hw *hw = &pf->hw;
9817         i40e_status ret = 0;
9818         u32 v;
9819
9820         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
9821         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
9822                 return;
9823         if (i40e_check_asq_alive(&pf->hw))
9824                 i40e_vc_notify_reset(pf);
9825
9826         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
9827
9828         /* quiesce the VSIs and their queues that are not already DOWN */
9829         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
9830         if (!lock_acquired)
9831                 rtnl_lock();
9832         i40e_pf_quiesce_all_vsi(pf);
9833         if (!lock_acquired)
9834                 rtnl_unlock();
9835
9836         for (v = 0; v < pf->num_alloc_vsi; v++) {
9837                 if (pf->vsi[v])
9838                         pf->vsi[v]->seid = 0;
9839         }
9840
9841         i40e_shutdown_adminq(&pf->hw);
9842
9843         /* call shutdown HMC */
9844         if (hw->hmc.hmc_obj) {
9845                 ret = i40e_shutdown_lan_hmc(hw);
9846                 if (ret)
9847                         dev_warn(&pf->pdev->dev,
9848                                  "shutdown_lan_hmc failed: %d\n", ret);
9849         }
9850
9851         /* Save the current PTP time so that we can restore the time after the
9852          * reset completes.
9853          */
9854         i40e_ptp_save_hw_time(pf);
9855 }
9856
9857 /**
9858  * i40e_send_version - update firmware with driver version
9859  * @pf: PF struct
9860  */
9861 static void i40e_send_version(struct i40e_pf *pf)
9862 {
9863         struct i40e_driver_version dv;
9864
9865         dv.major_version = DRV_VERSION_MAJOR;
9866         dv.minor_version = DRV_VERSION_MINOR;
9867         dv.build_version = DRV_VERSION_BUILD;
9868         dv.subbuild_version = 0;
9869         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
9870         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
9871 }
9872
9873 /**
9874  * i40e_get_oem_version - get OEM specific version information
9875  * @hw: pointer to the hardware structure
9876  **/
9877 static void i40e_get_oem_version(struct i40e_hw *hw)
9878 {
9879         u16 block_offset = 0xffff;
9880         u16 block_length = 0;
9881         u16 capabilities = 0;
9882         u16 gen_snap = 0;
9883         u16 release = 0;
9884
9885 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
9886 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
9887 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
9888 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
9889 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
9890 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
9891 #define I40E_NVM_OEM_LENGTH                     3
9892
9893         /* Check if pointer to OEM version block is valid. */
9894         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
9895         if (block_offset == 0xffff)
9896                 return;
9897
9898         /* Check if OEM version block has correct length. */
9899         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
9900                            &block_length);
9901         if (block_length < I40E_NVM_OEM_LENGTH)
9902                 return;
9903
9904         /* Check if OEM version format is as expected. */
9905         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
9906                            &capabilities);
9907         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
9908                 return;
9909
9910         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
9911                            &gen_snap);
9912         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
9913                            &release);
9914         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
9915         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
9916 }
9917
9918 /**
9919  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
9920  * @pf: board private structure
9921  **/
9922 static int i40e_reset(struct i40e_pf *pf)
9923 {
9924         struct i40e_hw *hw = &pf->hw;
9925         i40e_status ret;
9926
9927         ret = i40e_pf_reset(hw);
9928         if (ret) {
9929                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
9930                 set_bit(__I40E_RESET_FAILED, pf->state);
9931                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9932         } else {
9933                 pf->pfr_count++;
9934         }
9935         return ret;
9936 }
9937
9938 /**
9939  * i40e_rebuild - rebuild using a saved config
9940  * @pf: board private structure
9941  * @reinit: if the Main VSI needs to re-initialized.
9942  * @lock_acquired: indicates whether or not the lock has been acquired
9943  * before this function was called.
9944  **/
9945 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
9946 {
9947         int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state);
9948         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9949         struct i40e_hw *hw = &pf->hw;
9950         u8 set_fc_aq_fail = 0;
9951         i40e_status ret;
9952         u32 val;
9953         int v;
9954
9955         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
9956             i40e_check_recovery_mode(pf)) {
9957                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
9958         }
9959
9960         if (test_bit(__I40E_DOWN, pf->state) &&
9961             !test_bit(__I40E_RECOVERY_MODE, pf->state) &&
9962             !old_recovery_mode_bit)
9963                 goto clear_recovery;
9964         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
9965
9966         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
9967         ret = i40e_init_adminq(&pf->hw);
9968         if (ret) {
9969                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
9970                          i40e_stat_str(&pf->hw, ret),
9971                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9972                 goto clear_recovery;
9973         }
9974         i40e_get_oem_version(&pf->hw);
9975
9976         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
9977             ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
9978              hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
9979                 /* The following delay is necessary for 4.33 firmware and older
9980                  * to recover after EMP reset. 200 ms should suffice but we
9981                  * put here 300 ms to be sure that FW is ready to operate
9982                  * after reset.
9983                  */
9984                 mdelay(300);
9985         }
9986
9987         /* re-verify the eeprom if we just had an EMP reset */
9988         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
9989                 i40e_verify_eeprom(pf);
9990
9991         /* if we are going out of or into recovery mode we have to act
9992          * accordingly with regard to resources initialization
9993          * and deinitialization
9994          */
9995         if (test_bit(__I40E_RECOVERY_MODE, pf->state) ||
9996             old_recovery_mode_bit) {
9997                 if (i40e_get_capabilities(pf,
9998                                           i40e_aqc_opc_list_func_capabilities))
9999                         goto end_unlock;
10000
10001                 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10002                         /* we're staying in recovery mode so we'll reinitialize
10003                          * misc vector here
10004                          */
10005                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10006                                 goto end_unlock;
10007                 } else {
10008                         if (!lock_acquired)
10009                                 rtnl_lock();
10010                         /* we're going out of recovery mode so we'll free
10011                          * the IRQ allocated specifically for recovery mode
10012                          * and restore the interrupt scheme
10013                          */
10014                         free_irq(pf->pdev->irq, pf);
10015                         i40e_clear_interrupt_scheme(pf);
10016                         if (i40e_restore_interrupt_scheme(pf))
10017                                 goto end_unlock;
10018                 }
10019
10020                 /* tell the firmware that we're starting */
10021                 i40e_send_version(pf);
10022
10023                 /* bail out in case recovery mode was detected, as there is
10024                  * no need for further configuration.
10025                  */
10026                 goto end_unlock;
10027         }
10028
10029         i40e_clear_pxe_mode(hw);
10030         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10031         if (ret)
10032                 goto end_core_reset;
10033
10034         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10035                                 hw->func_caps.num_rx_qp, 0, 0);
10036         if (ret) {
10037                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10038                 goto end_core_reset;
10039         }
10040         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10041         if (ret) {
10042                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10043                 goto end_core_reset;
10044         }
10045
10046         /* Enable FW to write a default DCB config on link-up */
10047         i40e_aq_set_dcb_parameters(hw, true, NULL);
10048
10049 #ifdef CONFIG_I40E_DCB
10050         ret = i40e_init_pf_dcb(pf);
10051         if (ret) {
10052                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
10053                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10054                 /* Continue without DCB enabled */
10055         }
10056 #endif /* CONFIG_I40E_DCB */
10057         /* do basic switch setup */
10058         if (!lock_acquired)
10059                 rtnl_lock();
10060         ret = i40e_setup_pf_switch(pf, reinit);
10061         if (ret)
10062                 goto end_unlock;
10063
10064         /* The driver only wants link up/down and module qualification
10065          * reports from firmware.  Note the negative logic.
10066          */
10067         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10068                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10069                                          I40E_AQ_EVENT_MEDIA_NA |
10070                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10071         if (ret)
10072                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10073                          i40e_stat_str(&pf->hw, ret),
10074                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10075
10076         /* make sure our flow control settings are restored */
10077         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
10078         if (ret)
10079                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
10080                         i40e_stat_str(&pf->hw, ret),
10081                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10082
10083         /* Rebuild the VSIs and VEBs that existed before reset.
10084          * They are still in our local switch element arrays, so only
10085          * need to rebuild the switch model in the HW.
10086          *
10087          * If there were VEBs but the reconstitution failed, we'll try
10088          * try to recover minimal use by getting the basic PF VSI working.
10089          */
10090         if (vsi->uplink_seid != pf->mac_seid) {
10091                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10092                 /* find the one VEB connected to the MAC, and find orphans */
10093                 for (v = 0; v < I40E_MAX_VEB; v++) {
10094                         if (!pf->veb[v])
10095                                 continue;
10096
10097                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10098                             pf->veb[v]->uplink_seid == 0) {
10099                                 ret = i40e_reconstitute_veb(pf->veb[v]);
10100
10101                                 if (!ret)
10102                                         continue;
10103
10104                                 /* If Main VEB failed, we're in deep doodoo,
10105                                  * so give up rebuilding the switch and set up
10106                                  * for minimal rebuild of PF VSI.
10107                                  * If orphan failed, we'll report the error
10108                                  * but try to keep going.
10109                                  */
10110                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10111                                         dev_info(&pf->pdev->dev,
10112                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10113                                                  ret);
10114                                         vsi->uplink_seid = pf->mac_seid;
10115                                         break;
10116                                 } else if (pf->veb[v]->uplink_seid == 0) {
10117                                         dev_info(&pf->pdev->dev,
10118                                                  "rebuild of orphan VEB failed: %d\n",
10119                                                  ret);
10120                                 }
10121                         }
10122                 }
10123         }
10124
10125         if (vsi->uplink_seid == pf->mac_seid) {
10126                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10127                 /* no VEB, so rebuild only the Main VSI */
10128                 ret = i40e_add_vsi(vsi);
10129                 if (ret) {
10130                         dev_info(&pf->pdev->dev,
10131                                  "rebuild of Main VSI failed: %d\n", ret);
10132                         goto end_unlock;
10133                 }
10134         }
10135
10136         if (vsi->mqprio_qopt.max_rate[0]) {
10137                 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
10138                 u64 credits = 0;
10139
10140                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
10141                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10142                 if (ret)
10143                         goto end_unlock;
10144
10145                 credits = max_tx_rate;
10146                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10147                 dev_dbg(&vsi->back->pdev->dev,
10148                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10149                         max_tx_rate,
10150                         credits,
10151                         vsi->seid);
10152         }
10153
10154         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10155         if (ret)
10156                 goto end_unlock;
10157
10158         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
10159          * for this main VSI if they exist
10160          */
10161         ret = i40e_rebuild_channels(vsi);
10162         if (ret)
10163                 goto end_unlock;
10164
10165         /* Reconfigure hardware for allowing smaller MSS in the case
10166          * of TSO, so that we avoid the MDD being fired and causing
10167          * a reset in the case of small MSS+TSO.
10168          */
10169 #define I40E_REG_MSS          0x000E64DC
10170 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
10171 #define I40E_64BYTE_MSS       0x400000
10172         val = rd32(hw, I40E_REG_MSS);
10173         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10174                 val &= ~I40E_REG_MSS_MIN_MASK;
10175                 val |= I40E_64BYTE_MSS;
10176                 wr32(hw, I40E_REG_MSS, val);
10177         }
10178
10179         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10180                 msleep(75);
10181                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10182                 if (ret)
10183                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10184                                  i40e_stat_str(&pf->hw, ret),
10185                                  i40e_aq_str(&pf->hw,
10186                                              pf->hw.aq.asq_last_status));
10187         }
10188         /* reinit the misc interrupt */
10189         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10190                 ret = i40e_setup_misc_vector(pf);
10191
10192         /* Add a filter to drop all Flow control frames from any VSI from being
10193          * transmitted. By doing so we stop a malicious VF from sending out
10194          * PAUSE or PFC frames and potentially controlling traffic for other
10195          * PF/VF VSIs.
10196          * The FW can still send Flow control frames if enabled.
10197          */
10198         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10199                                                        pf->main_vsi_seid);
10200
10201         /* restart the VSIs that were rebuilt and running before the reset */
10202         i40e_pf_unquiesce_all_vsi(pf);
10203
10204         /* Release the RTNL lock before we start resetting VFs */
10205         if (!lock_acquired)
10206                 rtnl_unlock();
10207
10208         /* Restore promiscuous settings */
10209         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10210         if (ret)
10211                 dev_warn(&pf->pdev->dev,
10212                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10213                          pf->cur_promisc ? "on" : "off",
10214                          i40e_stat_str(&pf->hw, ret),
10215                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10216
10217         i40e_reset_all_vfs(pf, true);
10218
10219         /* tell the firmware that we're starting */
10220         i40e_send_version(pf);
10221
10222         /* We've already released the lock, so don't do it again */
10223         goto end_core_reset;
10224
10225 end_unlock:
10226         if (!lock_acquired)
10227                 rtnl_unlock();
10228 end_core_reset:
10229         clear_bit(__I40E_RESET_FAILED, pf->state);
10230 clear_recovery:
10231         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10232         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10233 }
10234
10235 /**
10236  * i40e_reset_and_rebuild - reset and rebuild using a saved config
10237  * @pf: board private structure
10238  * @reinit: if the Main VSI needs to re-initialized.
10239  * @lock_acquired: indicates whether or not the lock has been acquired
10240  * before this function was called.
10241  **/
10242 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10243                                    bool lock_acquired)
10244 {
10245         int ret;
10246         /* Now we wait for GRST to settle out.
10247          * We don't have to delete the VEBs or VSIs from the hw switch
10248          * because the reset will make them disappear.
10249          */
10250         ret = i40e_reset(pf);
10251         if (!ret)
10252                 i40e_rebuild(pf, reinit, lock_acquired);
10253 }
10254
10255 /**
10256  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
10257  * @pf: board private structure
10258  *
10259  * Close up the VFs and other things in prep for a Core Reset,
10260  * then get ready to rebuild the world.
10261  * @lock_acquired: indicates whether or not the lock has been acquired
10262  * before this function was called.
10263  **/
10264 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10265 {
10266         i40e_prep_for_reset(pf, lock_acquired);
10267         i40e_reset_and_rebuild(pf, false, lock_acquired);
10268 }
10269
10270 /**
10271  * i40e_handle_mdd_event
10272  * @pf: pointer to the PF structure
10273  *
10274  * Called from the MDD irq handler to identify possibly malicious vfs
10275  **/
10276 static void i40e_handle_mdd_event(struct i40e_pf *pf)
10277 {
10278         struct i40e_hw *hw = &pf->hw;
10279         bool mdd_detected = false;
10280         struct i40e_vf *vf;
10281         u32 reg;
10282         int i;
10283
10284         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10285                 return;
10286
10287         /* find what triggered the MDD event */
10288         reg = rd32(hw, I40E_GL_MDET_TX);
10289         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10290                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10291                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10292                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10293                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10294                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10295                                 I40E_GL_MDET_TX_EVENT_SHIFT;
10296                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10297                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10298                                 pf->hw.func_caps.base_queue;
10299                 if (netif_msg_tx_err(pf))
10300                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10301                                  event, queue, pf_num, vf_num);
10302                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10303                 mdd_detected = true;
10304         }
10305         reg = rd32(hw, I40E_GL_MDET_RX);
10306         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10307                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10308                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10309                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10310                                 I40E_GL_MDET_RX_EVENT_SHIFT;
10311                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10312                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10313                                 pf->hw.func_caps.base_queue;
10314                 if (netif_msg_rx_err(pf))
10315                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10316                                  event, queue, func);
10317                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10318                 mdd_detected = true;
10319         }
10320
10321         if (mdd_detected) {
10322                 reg = rd32(hw, I40E_PF_MDET_TX);
10323                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10324                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10325                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10326                 }
10327                 reg = rd32(hw, I40E_PF_MDET_RX);
10328                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10329                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10330                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10331                 }
10332         }
10333
10334         /* see if one of the VFs needs its hand slapped */
10335         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10336                 vf = &(pf->vf[i]);
10337                 reg = rd32(hw, I40E_VP_MDET_TX(i));
10338                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10339                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10340                         vf->num_mdd_events++;
10341                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10342                                  i);
10343                         dev_info(&pf->pdev->dev,
10344                                  "Use PF Control I/F to re-enable the VF\n");
10345                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10346                 }
10347
10348                 reg = rd32(hw, I40E_VP_MDET_RX(i));
10349                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10350                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10351                         vf->num_mdd_events++;
10352                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10353                                  i);
10354                         dev_info(&pf->pdev->dev,
10355                                  "Use PF Control I/F to re-enable the VF\n");
10356                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10357                 }
10358         }
10359
10360         /* re-enable mdd interrupt cause */
10361         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10362         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10363         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10364         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10365         i40e_flush(hw);
10366 }
10367
10368 static const char *i40e_tunnel_name(u8 type)
10369 {
10370         switch (type) {
10371         case UDP_TUNNEL_TYPE_VXLAN:
10372                 return "vxlan";
10373         case UDP_TUNNEL_TYPE_GENEVE:
10374                 return "geneve";
10375         default:
10376                 return "unknown";
10377         }
10378 }
10379
10380 /**
10381  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
10382  * @pf: board private structure
10383  **/
10384 static void i40e_sync_udp_filters(struct i40e_pf *pf)
10385 {
10386         int i;
10387
10388         /* loop through and set pending bit for all active UDP filters */
10389         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
10390                 if (pf->udp_ports[i].port)
10391                         pf->pending_udp_bitmap |= BIT_ULL(i);
10392         }
10393
10394         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
10395 }
10396
10397 /**
10398  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
10399  * @pf: board private structure
10400  **/
10401 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
10402 {
10403         struct i40e_hw *hw = &pf->hw;
10404         u8 filter_index, type;
10405         u16 port;
10406         int i;
10407
10408         if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state))
10409                 return;
10410
10411         /* acquire RTNL to maintain state of flags and port requests */
10412         rtnl_lock();
10413
10414         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
10415                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
10416                         struct i40e_udp_port_config *udp_port;
10417                         i40e_status ret = 0;
10418
10419                         udp_port = &pf->udp_ports[i];
10420                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
10421
10422                         port = READ_ONCE(udp_port->port);
10423                         type = READ_ONCE(udp_port->type);
10424                         filter_index = READ_ONCE(udp_port->filter_index);
10425
10426                         /* release RTNL while we wait on AQ command */
10427                         rtnl_unlock();
10428
10429                         if (port)
10430                                 ret = i40e_aq_add_udp_tunnel(hw, port,
10431                                                              type,
10432                                                              &filter_index,
10433                                                              NULL);
10434                         else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED)
10435                                 ret = i40e_aq_del_udp_tunnel(hw, filter_index,
10436                                                              NULL);
10437
10438                         /* reacquire RTNL so we can update filter_index */
10439                         rtnl_lock();
10440
10441                         if (ret) {
10442                                 dev_info(&pf->pdev->dev,
10443                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
10444                                          i40e_tunnel_name(type),
10445                                          port ? "add" : "delete",
10446                                          port,
10447                                          filter_index,
10448                                          i40e_stat_str(&pf->hw, ret),
10449                                          i40e_aq_str(&pf->hw,
10450                                                      pf->hw.aq.asq_last_status));
10451                                 if (port) {
10452                                         /* failed to add, just reset port,
10453                                          * drop pending bit for any deletion
10454                                          */
10455                                         udp_port->port = 0;
10456                                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
10457                                 }
10458                         } else if (port) {
10459                                 /* record filter index on success */
10460                                 udp_port->filter_index = filter_index;
10461                         }
10462                 }
10463         }
10464
10465         rtnl_unlock();
10466 }
10467
10468 /**
10469  * i40e_service_task - Run the driver's async subtasks
10470  * @work: pointer to work_struct containing our data
10471  **/
10472 static void i40e_service_task(struct work_struct *work)
10473 {
10474         struct i40e_pf *pf = container_of(work,
10475                                           struct i40e_pf,
10476                                           service_task);
10477         unsigned long start_time = jiffies;
10478
10479         /* don't bother with service tasks if a reset is in progress */
10480         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10481             test_bit(__I40E_SUSPENDED, pf->state))
10482                 return;
10483
10484         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10485                 return;
10486
10487         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10488                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10489                 i40e_sync_filters_subtask(pf);
10490                 i40e_reset_subtask(pf);
10491                 i40e_handle_mdd_event(pf);
10492                 i40e_vc_process_vflr_event(pf);
10493                 i40e_watchdog_subtask(pf);
10494                 i40e_fdir_reinit_subtask(pf);
10495                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10496                         /* Client subtask will reopen next time through. */
10497                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
10498                                                            true);
10499                 } else {
10500                         i40e_client_subtask(pf);
10501                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10502                                                pf->state))
10503                                 i40e_notify_client_of_l2_param_changes(
10504                                                                 pf->vsi[pf->lan_vsi]);
10505                 }
10506                 i40e_sync_filters_subtask(pf);
10507                 i40e_sync_udp_filters_subtask(pf);
10508         } else {
10509                 i40e_reset_subtask(pf);
10510         }
10511
10512         i40e_clean_adminq_subtask(pf);
10513
10514         /* flush memory to make sure state is correct before next watchdog */
10515         smp_mb__before_atomic();
10516         clear_bit(__I40E_SERVICE_SCHED, pf->state);
10517
10518         /* If the tasks have taken longer than one timer cycle or there
10519          * is more work to be done, reschedule the service task now
10520          * rather than wait for the timer to tick again.
10521          */
10522         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10523             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
10524             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
10525             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10526                 i40e_service_event_schedule(pf);
10527 }
10528
10529 /**
10530  * i40e_service_timer - timer callback
10531  * @data: pointer to PF struct
10532  **/
10533 static void i40e_service_timer(struct timer_list *t)
10534 {
10535         struct i40e_pf *pf = from_timer(pf, t, service_timer);
10536
10537         mod_timer(&pf->service_timer,
10538                   round_jiffies(jiffies + pf->service_timer_period));
10539         i40e_service_event_schedule(pf);
10540 }
10541
10542 /**
10543  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
10544  * @vsi: the VSI being configured
10545  **/
10546 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
10547 {
10548         struct i40e_pf *pf = vsi->back;
10549
10550         switch (vsi->type) {
10551         case I40E_VSI_MAIN:
10552                 vsi->alloc_queue_pairs = pf->num_lan_qps;
10553                 if (!vsi->num_tx_desc)
10554                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10555                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10556                 if (!vsi->num_rx_desc)
10557                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10558                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10559                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10560                         vsi->num_q_vectors = pf->num_lan_msix;
10561                 else
10562                         vsi->num_q_vectors = 1;
10563
10564                 break;
10565
10566         case I40E_VSI_FDIR:
10567                 vsi->alloc_queue_pairs = 1;
10568                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10569                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
10570                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10571                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
10572                 vsi->num_q_vectors = pf->num_fdsb_msix;
10573                 break;
10574
10575         case I40E_VSI_VMDQ2:
10576                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
10577                 if (!vsi->num_tx_desc)
10578                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10579                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10580                 if (!vsi->num_rx_desc)
10581                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10582                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10583                 vsi->num_q_vectors = pf->num_vmdq_msix;
10584                 break;
10585
10586         case I40E_VSI_SRIOV:
10587                 vsi->alloc_queue_pairs = pf->num_vf_qps;
10588                 if (!vsi->num_tx_desc)
10589                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10590                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10591                 if (!vsi->num_rx_desc)
10592                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10593                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10594                 break;
10595
10596         default:
10597                 WARN_ON(1);
10598                 return -ENODATA;
10599         }
10600
10601         return 0;
10602 }
10603
10604 /**
10605  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
10606  * @vsi: VSI pointer
10607  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
10608  *
10609  * On error: returns error code (negative)
10610  * On success: returns 0
10611  **/
10612 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
10613 {
10614         struct i40e_ring **next_rings;
10615         int size;
10616         int ret = 0;
10617
10618         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
10619         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
10620                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
10621         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
10622         if (!vsi->tx_rings)
10623                 return -ENOMEM;
10624         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
10625         if (i40e_enabled_xdp_vsi(vsi)) {
10626                 vsi->xdp_rings = next_rings;
10627                 next_rings += vsi->alloc_queue_pairs;
10628         }
10629         vsi->rx_rings = next_rings;
10630
10631         if (alloc_qvectors) {
10632                 /* allocate memory for q_vector pointers */
10633                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
10634                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
10635                 if (!vsi->q_vectors) {
10636                         ret = -ENOMEM;
10637                         goto err_vectors;
10638                 }
10639         }
10640         return ret;
10641
10642 err_vectors:
10643         kfree(vsi->tx_rings);
10644         return ret;
10645 }
10646
10647 /**
10648  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
10649  * @pf: board private structure
10650  * @type: type of VSI
10651  *
10652  * On error: returns error code (negative)
10653  * On success: returns vsi index in PF (positive)
10654  **/
10655 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
10656 {
10657         int ret = -ENODEV;
10658         struct i40e_vsi *vsi;
10659         int vsi_idx;
10660         int i;
10661
10662         /* Need to protect the allocation of the VSIs at the PF level */
10663         mutex_lock(&pf->switch_mutex);
10664
10665         /* VSI list may be fragmented if VSI creation/destruction has
10666          * been happening.  We can afford to do a quick scan to look
10667          * for any free VSIs in the list.
10668          *
10669          * find next empty vsi slot, looping back around if necessary
10670          */
10671         i = pf->next_vsi;
10672         while (i < pf->num_alloc_vsi && pf->vsi[i])
10673                 i++;
10674         if (i >= pf->num_alloc_vsi) {
10675                 i = 0;
10676                 while (i < pf->next_vsi && pf->vsi[i])
10677                         i++;
10678         }
10679
10680         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
10681                 vsi_idx = i;             /* Found one! */
10682         } else {
10683                 ret = -ENODEV;
10684                 goto unlock_pf;  /* out of VSI slots! */
10685         }
10686         pf->next_vsi = ++i;
10687
10688         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
10689         if (!vsi) {
10690                 ret = -ENOMEM;
10691                 goto unlock_pf;
10692         }
10693         vsi->type = type;
10694         vsi->back = pf;
10695         set_bit(__I40E_VSI_DOWN, vsi->state);
10696         vsi->flags = 0;
10697         vsi->idx = vsi_idx;
10698         vsi->int_rate_limit = 0;
10699         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
10700                                 pf->rss_table_size : 64;
10701         vsi->netdev_registered = false;
10702         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
10703         hash_init(vsi->mac_filter_hash);
10704         vsi->irqs_ready = false;
10705
10706         if (type == I40E_VSI_MAIN) {
10707                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
10708                 if (!vsi->af_xdp_zc_qps)
10709                         goto err_rings;
10710         }
10711
10712         ret = i40e_set_num_rings_in_vsi(vsi);
10713         if (ret)
10714                 goto err_rings;
10715
10716         ret = i40e_vsi_alloc_arrays(vsi, true);
10717         if (ret)
10718                 goto err_rings;
10719
10720         /* Setup default MSIX irq handler for VSI */
10721         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
10722
10723         /* Initialize VSI lock */
10724         spin_lock_init(&vsi->mac_filter_hash_lock);
10725         pf->vsi[vsi_idx] = vsi;
10726         ret = vsi_idx;
10727         goto unlock_pf;
10728
10729 err_rings:
10730         bitmap_free(vsi->af_xdp_zc_qps);
10731         pf->next_vsi = i - 1;
10732         kfree(vsi);
10733 unlock_pf:
10734         mutex_unlock(&pf->switch_mutex);
10735         return ret;
10736 }
10737
10738 /**
10739  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
10740  * @vsi: VSI pointer
10741  * @free_qvectors: a bool to specify if q_vectors need to be freed.
10742  *
10743  * On error: returns error code (negative)
10744  * On success: returns 0
10745  **/
10746 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
10747 {
10748         /* free the ring and vector containers */
10749         if (free_qvectors) {
10750                 kfree(vsi->q_vectors);
10751                 vsi->q_vectors = NULL;
10752         }
10753         kfree(vsi->tx_rings);
10754         vsi->tx_rings = NULL;
10755         vsi->rx_rings = NULL;
10756         vsi->xdp_rings = NULL;
10757 }
10758
10759 /**
10760  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
10761  * and lookup table
10762  * @vsi: Pointer to VSI structure
10763  */
10764 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
10765 {
10766         if (!vsi)
10767                 return;
10768
10769         kfree(vsi->rss_hkey_user);
10770         vsi->rss_hkey_user = NULL;
10771
10772         kfree(vsi->rss_lut_user);
10773         vsi->rss_lut_user = NULL;
10774 }
10775
10776 /**
10777  * i40e_vsi_clear - Deallocate the VSI provided
10778  * @vsi: the VSI being un-configured
10779  **/
10780 static int i40e_vsi_clear(struct i40e_vsi *vsi)
10781 {
10782         struct i40e_pf *pf;
10783
10784         if (!vsi)
10785                 return 0;
10786
10787         if (!vsi->back)
10788                 goto free_vsi;
10789         pf = vsi->back;
10790
10791         mutex_lock(&pf->switch_mutex);
10792         if (!pf->vsi[vsi->idx]) {
10793                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
10794                         vsi->idx, vsi->idx, vsi->type);
10795                 goto unlock_vsi;
10796         }
10797
10798         if (pf->vsi[vsi->idx] != vsi) {
10799                 dev_err(&pf->pdev->dev,
10800                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
10801                         pf->vsi[vsi->idx]->idx,
10802                         pf->vsi[vsi->idx]->type,
10803                         vsi->idx, vsi->type);
10804                 goto unlock_vsi;
10805         }
10806
10807         /* updates the PF for this cleared vsi */
10808         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10809         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
10810
10811         bitmap_free(vsi->af_xdp_zc_qps);
10812         i40e_vsi_free_arrays(vsi, true);
10813         i40e_clear_rss_config_user(vsi);
10814
10815         pf->vsi[vsi->idx] = NULL;
10816         if (vsi->idx < pf->next_vsi)
10817                 pf->next_vsi = vsi->idx;
10818
10819 unlock_vsi:
10820         mutex_unlock(&pf->switch_mutex);
10821 free_vsi:
10822         kfree(vsi);
10823
10824         return 0;
10825 }
10826
10827 /**
10828  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
10829  * @vsi: the VSI being cleaned
10830  **/
10831 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
10832 {
10833         int i;
10834
10835         if (vsi->tx_rings && vsi->tx_rings[0]) {
10836                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10837                         kfree_rcu(vsi->tx_rings[i], rcu);
10838                         vsi->tx_rings[i] = NULL;
10839                         vsi->rx_rings[i] = NULL;
10840                         if (vsi->xdp_rings)
10841                                 vsi->xdp_rings[i] = NULL;
10842                 }
10843         }
10844 }
10845
10846 /**
10847  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
10848  * @vsi: the VSI being configured
10849  **/
10850 static int i40e_alloc_rings(struct i40e_vsi *vsi)
10851 {
10852         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
10853         struct i40e_pf *pf = vsi->back;
10854         struct i40e_ring *ring;
10855
10856         /* Set basic values in the rings to be used later during open() */
10857         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10858                 /* allocate space for both Tx and Rx in one shot */
10859                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
10860                 if (!ring)
10861                         goto err_out;
10862
10863                 ring->queue_index = i;
10864                 ring->reg_idx = vsi->base_queue + i;
10865                 ring->ring_active = false;
10866                 ring->vsi = vsi;
10867                 ring->netdev = vsi->netdev;
10868                 ring->dev = &pf->pdev->dev;
10869                 ring->count = vsi->num_tx_desc;
10870                 ring->size = 0;
10871                 ring->dcb_tc = 0;
10872                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10873                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10874                 ring->itr_setting = pf->tx_itr_default;
10875                 vsi->tx_rings[i] = ring++;
10876
10877                 if (!i40e_enabled_xdp_vsi(vsi))
10878                         goto setup_rx;
10879
10880                 ring->queue_index = vsi->alloc_queue_pairs + i;
10881                 ring->reg_idx = vsi->base_queue + ring->queue_index;
10882                 ring->ring_active = false;
10883                 ring->vsi = vsi;
10884                 ring->netdev = NULL;
10885                 ring->dev = &pf->pdev->dev;
10886                 ring->count = vsi->num_tx_desc;
10887                 ring->size = 0;
10888                 ring->dcb_tc = 0;
10889                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10890                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10891                 set_ring_xdp(ring);
10892                 ring->itr_setting = pf->tx_itr_default;
10893                 vsi->xdp_rings[i] = ring++;
10894
10895 setup_rx:
10896                 ring->queue_index = i;
10897                 ring->reg_idx = vsi->base_queue + i;
10898                 ring->ring_active = false;
10899                 ring->vsi = vsi;
10900                 ring->netdev = vsi->netdev;
10901                 ring->dev = &pf->pdev->dev;
10902                 ring->count = vsi->num_rx_desc;
10903                 ring->size = 0;
10904                 ring->dcb_tc = 0;
10905                 ring->itr_setting = pf->rx_itr_default;
10906                 vsi->rx_rings[i] = ring;
10907         }
10908
10909         return 0;
10910
10911 err_out:
10912         i40e_vsi_clear_rings(vsi);
10913         return -ENOMEM;
10914 }
10915
10916 /**
10917  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
10918  * @pf: board private structure
10919  * @vectors: the number of MSI-X vectors to request
10920  *
10921  * Returns the number of vectors reserved, or error
10922  **/
10923 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
10924 {
10925         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
10926                                         I40E_MIN_MSIX, vectors);
10927         if (vectors < 0) {
10928                 dev_info(&pf->pdev->dev,
10929                          "MSI-X vector reservation failed: %d\n", vectors);
10930                 vectors = 0;
10931         }
10932
10933         return vectors;
10934 }
10935
10936 /**
10937  * i40e_init_msix - Setup the MSIX capability
10938  * @pf: board private structure
10939  *
10940  * Work with the OS to set up the MSIX vectors needed.
10941  *
10942  * Returns the number of vectors reserved or negative on failure
10943  **/
10944 static int i40e_init_msix(struct i40e_pf *pf)
10945 {
10946         struct i40e_hw *hw = &pf->hw;
10947         int cpus, extra_vectors;
10948         int vectors_left;
10949         int v_budget, i;
10950         int v_actual;
10951         int iwarp_requested = 0;
10952
10953         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10954                 return -ENODEV;
10955
10956         /* The number of vectors we'll request will be comprised of:
10957          *   - Add 1 for "other" cause for Admin Queue events, etc.
10958          *   - The number of LAN queue pairs
10959          *      - Queues being used for RSS.
10960          *              We don't need as many as max_rss_size vectors.
10961          *              use rss_size instead in the calculation since that
10962          *              is governed by number of cpus in the system.
10963          *      - assumes symmetric Tx/Rx pairing
10964          *   - The number of VMDq pairs
10965          *   - The CPU count within the NUMA node if iWARP is enabled
10966          * Once we count this up, try the request.
10967          *
10968          * If we can't get what we want, we'll simplify to nearly nothing
10969          * and try again.  If that still fails, we punt.
10970          */
10971         vectors_left = hw->func_caps.num_msix_vectors;
10972         v_budget = 0;
10973
10974         /* reserve one vector for miscellaneous handler */
10975         if (vectors_left) {
10976                 v_budget++;
10977                 vectors_left--;
10978         }
10979
10980         /* reserve some vectors for the main PF traffic queues. Initially we
10981          * only reserve at most 50% of the available vectors, in the case that
10982          * the number of online CPUs is large. This ensures that we can enable
10983          * extra features as well. Once we've enabled the other features, we
10984          * will use any remaining vectors to reach as close as we can to the
10985          * number of online CPUs.
10986          */
10987         cpus = num_online_cpus();
10988         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
10989         vectors_left -= pf->num_lan_msix;
10990
10991         /* reserve one vector for sideband flow director */
10992         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10993                 if (vectors_left) {
10994                         pf->num_fdsb_msix = 1;
10995                         v_budget++;
10996                         vectors_left--;
10997                 } else {
10998                         pf->num_fdsb_msix = 0;
10999                 }
11000         }
11001
11002         /* can we reserve enough for iWARP? */
11003         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11004                 iwarp_requested = pf->num_iwarp_msix;
11005
11006                 if (!vectors_left)
11007                         pf->num_iwarp_msix = 0;
11008                 else if (vectors_left < pf->num_iwarp_msix)
11009                         pf->num_iwarp_msix = 1;
11010                 v_budget += pf->num_iwarp_msix;
11011                 vectors_left -= pf->num_iwarp_msix;
11012         }
11013
11014         /* any vectors left over go for VMDq support */
11015         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11016                 if (!vectors_left) {
11017                         pf->num_vmdq_msix = 0;
11018                         pf->num_vmdq_qps = 0;
11019                 } else {
11020                         int vmdq_vecs_wanted =
11021                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11022                         int vmdq_vecs =
11023                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11024
11025                         /* if we're short on vectors for what's desired, we limit
11026                          * the queues per vmdq.  If this is still more than are
11027                          * available, the user will need to change the number of
11028                          * queues/vectors used by the PF later with the ethtool
11029                          * channels command
11030                          */
11031                         if (vectors_left < vmdq_vecs_wanted) {
11032                                 pf->num_vmdq_qps = 1;
11033                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11034                                 vmdq_vecs = min_t(int,
11035                                                   vectors_left,
11036                                                   vmdq_vecs_wanted);
11037                         }
11038                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11039
11040                         v_budget += vmdq_vecs;
11041                         vectors_left -= vmdq_vecs;
11042                 }
11043         }
11044
11045         /* On systems with a large number of SMP cores, we previously limited
11046          * the number of vectors for num_lan_msix to be at most 50% of the
11047          * available vectors, to allow for other features. Now, we add back
11048          * the remaining vectors. However, we ensure that the total
11049          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11050          * calculate the number of vectors we can add without going over the
11051          * cap of CPUs. For systems with a small number of CPUs this will be
11052          * zero.
11053          */
11054         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11055         pf->num_lan_msix += extra_vectors;
11056         vectors_left -= extra_vectors;
11057
11058         WARN(vectors_left < 0,
11059              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11060
11061         v_budget += pf->num_lan_msix;
11062         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11063                                    GFP_KERNEL);
11064         if (!pf->msix_entries)
11065                 return -ENOMEM;
11066
11067         for (i = 0; i < v_budget; i++)
11068                 pf->msix_entries[i].entry = i;
11069         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11070
11071         if (v_actual < I40E_MIN_MSIX) {
11072                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11073                 kfree(pf->msix_entries);
11074                 pf->msix_entries = NULL;
11075                 pci_disable_msix(pf->pdev);
11076                 return -ENODEV;
11077
11078         } else if (v_actual == I40E_MIN_MSIX) {
11079                 /* Adjust for minimal MSIX use */
11080                 pf->num_vmdq_vsis = 0;
11081                 pf->num_vmdq_qps = 0;
11082                 pf->num_lan_qps = 1;
11083                 pf->num_lan_msix = 1;
11084
11085         } else if (v_actual != v_budget) {
11086                 /* If we have limited resources, we will start with no vectors
11087                  * for the special features and then allocate vectors to some
11088                  * of these features based on the policy and at the end disable
11089                  * the features that did not get any vectors.
11090                  */
11091                 int vec;
11092
11093                 dev_info(&pf->pdev->dev,
11094                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11095                          v_actual, v_budget);
11096                 /* reserve the misc vector */
11097                 vec = v_actual - 1;
11098
11099                 /* Scale vector usage down */
11100                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11101                 pf->num_vmdq_vsis = 1;
11102                 pf->num_vmdq_qps = 1;
11103
11104                 /* partition out the remaining vectors */
11105                 switch (vec) {
11106                 case 2:
11107                         pf->num_lan_msix = 1;
11108                         break;
11109                 case 3:
11110                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11111                                 pf->num_lan_msix = 1;
11112                                 pf->num_iwarp_msix = 1;
11113                         } else {
11114                                 pf->num_lan_msix = 2;
11115                         }
11116                         break;
11117                 default:
11118                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11119                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11120                                                  iwarp_requested);
11121                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11122                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11123                         } else {
11124                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11125                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11126                         }
11127                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11128                                 pf->num_fdsb_msix = 1;
11129                                 vec--;
11130                         }
11131                         pf->num_lan_msix = min_t(int,
11132                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11133                                                               pf->num_lan_msix);
11134                         pf->num_lan_qps = pf->num_lan_msix;
11135                         break;
11136                 }
11137         }
11138
11139         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11140             (pf->num_fdsb_msix == 0)) {
11141                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11142                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11143                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11144         }
11145         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11146             (pf->num_vmdq_msix == 0)) {
11147                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11148                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11149         }
11150
11151         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11152             (pf->num_iwarp_msix == 0)) {
11153                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11154                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11155         }
11156         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11157                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11158                    pf->num_lan_msix,
11159                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11160                    pf->num_fdsb_msix,
11161                    pf->num_iwarp_msix);
11162
11163         return v_actual;
11164 }
11165
11166 /**
11167  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11168  * @vsi: the VSI being configured
11169  * @v_idx: index of the vector in the vsi struct
11170  * @cpu: cpu to be used on affinity_mask
11171  *
11172  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11173  **/
11174 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
11175 {
11176         struct i40e_q_vector *q_vector;
11177
11178         /* allocate q_vector */
11179         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11180         if (!q_vector)
11181                 return -ENOMEM;
11182
11183         q_vector->vsi = vsi;
11184         q_vector->v_idx = v_idx;
11185         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11186
11187         if (vsi->netdev)
11188                 netif_napi_add(vsi->netdev, &q_vector->napi,
11189                                i40e_napi_poll, NAPI_POLL_WEIGHT);
11190
11191         /* tie q_vector and vsi together */
11192         vsi->q_vectors[v_idx] = q_vector;
11193
11194         return 0;
11195 }
11196
11197 /**
11198  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11199  * @vsi: the VSI being configured
11200  *
11201  * We allocate one q_vector per queue interrupt.  If allocation fails we
11202  * return -ENOMEM.
11203  **/
11204 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11205 {
11206         struct i40e_pf *pf = vsi->back;
11207         int err, v_idx, num_q_vectors, current_cpu;
11208
11209         /* if not MSIX, give the one vector only to the LAN VSI */
11210         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11211                 num_q_vectors = vsi->num_q_vectors;
11212         else if (vsi == pf->vsi[pf->lan_vsi])
11213                 num_q_vectors = 1;
11214         else
11215                 return -EINVAL;
11216
11217         current_cpu = cpumask_first(cpu_online_mask);
11218
11219         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11220                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
11221                 if (err)
11222                         goto err_out;
11223                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
11224                 if (unlikely(current_cpu >= nr_cpu_ids))
11225                         current_cpu = cpumask_first(cpu_online_mask);
11226         }
11227
11228         return 0;
11229
11230 err_out:
11231         while (v_idx--)
11232                 i40e_free_q_vector(vsi, v_idx);
11233
11234         return err;
11235 }
11236
11237 /**
11238  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
11239  * @pf: board private structure to initialize
11240  **/
11241 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11242 {
11243         int vectors = 0;
11244         ssize_t size;
11245
11246         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11247                 vectors = i40e_init_msix(pf);
11248                 if (vectors < 0) {
11249                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
11250                                        I40E_FLAG_IWARP_ENABLED  |
11251                                        I40E_FLAG_RSS_ENABLED    |
11252                                        I40E_FLAG_DCB_CAPABLE    |
11253                                        I40E_FLAG_DCB_ENABLED    |
11254                                        I40E_FLAG_SRIOV_ENABLED  |
11255                                        I40E_FLAG_FD_SB_ENABLED  |
11256                                        I40E_FLAG_FD_ATR_ENABLED |
11257                                        I40E_FLAG_VMDQ_ENABLED);
11258                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11259
11260                         /* rework the queue expectations without MSIX */
11261                         i40e_determine_queue_usage(pf);
11262                 }
11263         }
11264
11265         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11266             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11267                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11268                 vectors = pci_enable_msi(pf->pdev);
11269                 if (vectors < 0) {
11270                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11271                                  vectors);
11272                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11273                 }
11274                 vectors = 1;  /* one MSI or Legacy vector */
11275         }
11276
11277         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11278                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11279
11280         /* set up vector assignment tracking */
11281         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11282         pf->irq_pile = kzalloc(size, GFP_KERNEL);
11283         if (!pf->irq_pile)
11284                 return -ENOMEM;
11285
11286         pf->irq_pile->num_entries = vectors;
11287         pf->irq_pile->search_hint = 0;
11288
11289         /* track first vector for misc interrupts, ignore return */
11290         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11291
11292         return 0;
11293 }
11294
11295 /**
11296  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
11297  * @pf: private board data structure
11298  *
11299  * Restore the interrupt scheme that was cleared when we suspended the
11300  * device. This should be called during resume to re-allocate the q_vectors
11301  * and reacquire IRQs.
11302  */
11303 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11304 {
11305         int err, i;
11306
11307         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
11308          * scheme. We need to re-enabled them here in order to attempt to
11309          * re-acquire the MSI or MSI-X vectors
11310          */
11311         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11312
11313         err = i40e_init_interrupt_scheme(pf);
11314         if (err)
11315                 return err;
11316
11317         /* Now that we've re-acquired IRQs, we need to remap the vectors and
11318          * rings together again.
11319          */
11320         for (i = 0; i < pf->num_alloc_vsi; i++) {
11321                 if (pf->vsi[i]) {
11322                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11323                         if (err)
11324                                 goto err_unwind;
11325                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11326                 }
11327         }
11328
11329         err = i40e_setup_misc_vector(pf);
11330         if (err)
11331                 goto err_unwind;
11332
11333         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11334                 i40e_client_update_msix_info(pf);
11335
11336         return 0;
11337
11338 err_unwind:
11339         while (i--) {
11340                 if (pf->vsi[i])
11341                         i40e_vsi_free_q_vectors(pf->vsi[i]);
11342         }
11343
11344         return err;
11345 }
11346
11347 /**
11348  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
11349  * non queue events in recovery mode
11350  * @pf: board private structure
11351  *
11352  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
11353  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
11354  * This is handled differently than in recovery mode since no Tx/Rx resources
11355  * are being allocated.
11356  **/
11357 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11358 {
11359         int err;
11360
11361         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11362                 err = i40e_setup_misc_vector(pf);
11363
11364                 if (err) {
11365                         dev_info(&pf->pdev->dev,
11366                                  "MSI-X misc vector request failed, error %d\n",
11367                                  err);
11368                         return err;
11369                 }
11370         } else {
11371                 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11372
11373                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11374                                   pf->int_name, pf);
11375
11376                 if (err) {
11377                         dev_info(&pf->pdev->dev,
11378                                  "MSI/legacy misc vector request failed, error %d\n",
11379                                  err);
11380                         return err;
11381                 }
11382                 i40e_enable_misc_int_causes(pf);
11383                 i40e_irq_dynamic_enable_icr0(pf);
11384         }
11385
11386         return 0;
11387 }
11388
11389 /**
11390  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
11391  * @pf: board private structure
11392  *
11393  * This sets up the handler for MSIX 0, which is used to manage the
11394  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
11395  * when in MSI or Legacy interrupt mode.
11396  **/
11397 static int i40e_setup_misc_vector(struct i40e_pf *pf)
11398 {
11399         struct i40e_hw *hw = &pf->hw;
11400         int err = 0;
11401
11402         /* Only request the IRQ once, the first time through. */
11403         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11404                 err = request_irq(pf->msix_entries[0].vector,
11405                                   i40e_intr, 0, pf->int_name, pf);
11406                 if (err) {
11407                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11408                         dev_info(&pf->pdev->dev,
11409                                  "request_irq for %s failed: %d\n",
11410                                  pf->int_name, err);
11411                         return -EFAULT;
11412                 }
11413         }
11414
11415         i40e_enable_misc_int_causes(pf);
11416
11417         /* associate no queues to the misc vector */
11418         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11419         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
11420
11421         i40e_flush(hw);
11422
11423         i40e_irq_dynamic_enable_icr0(pf);
11424
11425         return err;
11426 }
11427
11428 /**
11429  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
11430  * @vsi: Pointer to vsi structure
11431  * @seed: Buffter to store the hash keys
11432  * @lut: Buffer to store the lookup table entries
11433  * @lut_size: Size of buffer to store the lookup table entries
11434  *
11435  * Return 0 on success, negative on failure
11436  */
11437 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11438                            u8 *lut, u16 lut_size)
11439 {
11440         struct i40e_pf *pf = vsi->back;
11441         struct i40e_hw *hw = &pf->hw;
11442         int ret = 0;
11443
11444         if (seed) {
11445                 ret = i40e_aq_get_rss_key(hw, vsi->id,
11446                         (struct i40e_aqc_get_set_rss_key_data *)seed);
11447                 if (ret) {
11448                         dev_info(&pf->pdev->dev,
11449                                  "Cannot get RSS key, err %s aq_err %s\n",
11450                                  i40e_stat_str(&pf->hw, ret),
11451                                  i40e_aq_str(&pf->hw,
11452                                              pf->hw.aq.asq_last_status));
11453                         return ret;
11454                 }
11455         }
11456
11457         if (lut) {
11458                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
11459
11460                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11461                 if (ret) {
11462                         dev_info(&pf->pdev->dev,
11463                                  "Cannot get RSS lut, err %s aq_err %s\n",
11464                                  i40e_stat_str(&pf->hw, ret),
11465                                  i40e_aq_str(&pf->hw,
11466                                              pf->hw.aq.asq_last_status));
11467                         return ret;
11468                 }
11469         }
11470
11471         return ret;
11472 }
11473
11474 /**
11475  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
11476  * @vsi: Pointer to vsi structure
11477  * @seed: RSS hash seed
11478  * @lut: Lookup table
11479  * @lut_size: Lookup table size
11480  *
11481  * Returns 0 on success, negative on failure
11482  **/
11483 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11484                                const u8 *lut, u16 lut_size)
11485 {
11486         struct i40e_pf *pf = vsi->back;
11487         struct i40e_hw *hw = &pf->hw;
11488         u16 vf_id = vsi->vf_id;
11489         u8 i;
11490
11491         /* Fill out hash function seed */
11492         if (seed) {
11493                 u32 *seed_dw = (u32 *)seed;
11494
11495                 if (vsi->type == I40E_VSI_MAIN) {
11496                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11497                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
11498                 } else if (vsi->type == I40E_VSI_SRIOV) {
11499                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
11500                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
11501                 } else {
11502                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
11503                 }
11504         }
11505
11506         if (lut) {
11507                 u32 *lut_dw = (u32 *)lut;
11508
11509                 if (vsi->type == I40E_VSI_MAIN) {
11510                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
11511                                 return -EINVAL;
11512                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11513                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
11514                 } else if (vsi->type == I40E_VSI_SRIOV) {
11515                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
11516                                 return -EINVAL;
11517                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11518                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
11519                 } else {
11520                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11521                 }
11522         }
11523         i40e_flush(hw);
11524
11525         return 0;
11526 }
11527
11528 /**
11529  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
11530  * @vsi: Pointer to VSI structure
11531  * @seed: Buffer to store the keys
11532  * @lut: Buffer to store the lookup table entries
11533  * @lut_size: Size of buffer to store the lookup table entries
11534  *
11535  * Returns 0 on success, negative on failure
11536  */
11537 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
11538                             u8 *lut, u16 lut_size)
11539 {
11540         struct i40e_pf *pf = vsi->back;
11541         struct i40e_hw *hw = &pf->hw;
11542         u16 i;
11543
11544         if (seed) {
11545                 u32 *seed_dw = (u32 *)seed;
11546
11547                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11548                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
11549         }
11550         if (lut) {
11551                 u32 *lut_dw = (u32 *)lut;
11552
11553                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11554                         return -EINVAL;
11555                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11556                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
11557         }
11558
11559         return 0;
11560 }
11561
11562 /**
11563  * i40e_config_rss - Configure RSS keys and lut
11564  * @vsi: Pointer to VSI structure
11565  * @seed: RSS hash seed
11566  * @lut: Lookup table
11567  * @lut_size: Lookup table size
11568  *
11569  * Returns 0 on success, negative on failure
11570  */
11571 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11572 {
11573         struct i40e_pf *pf = vsi->back;
11574
11575         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11576                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
11577         else
11578                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
11579 }
11580
11581 /**
11582  * i40e_get_rss - Get RSS keys and lut
11583  * @vsi: Pointer to VSI structure
11584  * @seed: Buffer to store the keys
11585  * @lut: Buffer to store the lookup table entries
11586  * @lut_size: Size of buffer to store the lookup table entries
11587  *
11588  * Returns 0 on success, negative on failure
11589  */
11590 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11591 {
11592         struct i40e_pf *pf = vsi->back;
11593
11594         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11595                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
11596         else
11597                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
11598 }
11599
11600 /**
11601  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
11602  * @pf: Pointer to board private structure
11603  * @lut: Lookup table
11604  * @rss_table_size: Lookup table size
11605  * @rss_size: Range of queue number for hashing
11606  */
11607 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
11608                        u16 rss_table_size, u16 rss_size)
11609 {
11610         u16 i;
11611
11612         for (i = 0; i < rss_table_size; i++)
11613                 lut[i] = i % rss_size;
11614 }
11615
11616 /**
11617  * i40e_pf_config_rss - Prepare for RSS if used
11618  * @pf: board private structure
11619  **/
11620 static int i40e_pf_config_rss(struct i40e_pf *pf)
11621 {
11622         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11623         u8 seed[I40E_HKEY_ARRAY_SIZE];
11624         u8 *lut;
11625         struct i40e_hw *hw = &pf->hw;
11626         u32 reg_val;
11627         u64 hena;
11628         int ret;
11629
11630         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
11631         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
11632                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
11633         hena |= i40e_pf_get_default_rss_hena(pf);
11634
11635         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
11636         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
11637
11638         /* Determine the RSS table size based on the hardware capabilities */
11639         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
11640         reg_val = (pf->rss_table_size == 512) ?
11641                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
11642                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
11643         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
11644
11645         /* Determine the RSS size of the VSI */
11646         if (!vsi->rss_size) {
11647                 u16 qcount;
11648                 /* If the firmware does something weird during VSI init, we
11649                  * could end up with zero TCs. Check for that to avoid
11650                  * divide-by-zero. It probably won't pass traffic, but it also
11651                  * won't panic.
11652                  */
11653                 qcount = vsi->num_queue_pairs /
11654                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
11655                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11656         }
11657         if (!vsi->rss_size)
11658                 return -EINVAL;
11659
11660         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
11661         if (!lut)
11662                 return -ENOMEM;
11663
11664         /* Use user configured lut if there is one, otherwise use default */
11665         if (vsi->rss_lut_user)
11666                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
11667         else
11668                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
11669
11670         /* Use user configured hash key if there is one, otherwise
11671          * use default.
11672          */
11673         if (vsi->rss_hkey_user)
11674                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
11675         else
11676                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
11677         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
11678         kfree(lut);
11679
11680         return ret;
11681 }
11682
11683 /**
11684  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
11685  * @pf: board private structure
11686  * @queue_count: the requested queue count for rss.
11687  *
11688  * returns 0 if rss is not enabled, if enabled returns the final rss queue
11689  * count which may be different from the requested queue count.
11690  * Note: expects to be called while under rtnl_lock()
11691  **/
11692 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
11693 {
11694         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11695         int new_rss_size;
11696
11697         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
11698                 return 0;
11699
11700         queue_count = min_t(int, queue_count, num_online_cpus());
11701         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
11702
11703         if (queue_count != vsi->num_queue_pairs) {
11704                 u16 qcount;
11705
11706                 vsi->req_queue_pairs = queue_count;
11707                 i40e_prep_for_reset(pf, true);
11708
11709                 pf->alloc_rss_size = new_rss_size;
11710
11711                 i40e_reset_and_rebuild(pf, true, true);
11712
11713                 /* Discard the user configured hash keys and lut, if less
11714                  * queues are enabled.
11715                  */
11716                 if (queue_count < vsi->rss_size) {
11717                         i40e_clear_rss_config_user(vsi);
11718                         dev_dbg(&pf->pdev->dev,
11719                                 "discard user configured hash keys and lut\n");
11720                 }
11721
11722                 /* Reset vsi->rss_size, as number of enabled queues changed */
11723                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
11724                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11725
11726                 i40e_pf_config_rss(pf);
11727         }
11728         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
11729                  vsi->req_queue_pairs, pf->rss_size_max);
11730         return pf->alloc_rss_size;
11731 }
11732
11733 /**
11734  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
11735  * @pf: board private structure
11736  **/
11737 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
11738 {
11739         i40e_status status;
11740         bool min_valid, max_valid;
11741         u32 max_bw, min_bw;
11742
11743         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
11744                                            &min_valid, &max_valid);
11745
11746         if (!status) {
11747                 if (min_valid)
11748                         pf->min_bw = min_bw;
11749                 if (max_valid)
11750                         pf->max_bw = max_bw;
11751         }
11752
11753         return status;
11754 }
11755
11756 /**
11757  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
11758  * @pf: board private structure
11759  **/
11760 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
11761 {
11762         struct i40e_aqc_configure_partition_bw_data bw_data;
11763         i40e_status status;
11764
11765         /* Set the valid bit for this PF */
11766         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
11767         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
11768         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
11769
11770         /* Set the new bandwidths */
11771         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
11772
11773         return status;
11774 }
11775
11776 /**
11777  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
11778  * @pf: board private structure
11779  **/
11780 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
11781 {
11782         /* Commit temporary BW setting to permanent NVM image */
11783         enum i40e_admin_queue_err last_aq_status;
11784         i40e_status ret;
11785         u16 nvm_word;
11786
11787         if (pf->hw.partition_id != 1) {
11788                 dev_info(&pf->pdev->dev,
11789                          "Commit BW only works on partition 1! This is partition %d",
11790                          pf->hw.partition_id);
11791                 ret = I40E_NOT_SUPPORTED;
11792                 goto bw_commit_out;
11793         }
11794
11795         /* Acquire NVM for read access */
11796         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
11797         last_aq_status = pf->hw.aq.asq_last_status;
11798         if (ret) {
11799                 dev_info(&pf->pdev->dev,
11800                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
11801                          i40e_stat_str(&pf->hw, ret),
11802                          i40e_aq_str(&pf->hw, last_aq_status));
11803                 goto bw_commit_out;
11804         }
11805
11806         /* Read word 0x10 of NVM - SW compatibility word 1 */
11807         ret = i40e_aq_read_nvm(&pf->hw,
11808                                I40E_SR_NVM_CONTROL_WORD,
11809                                0x10, sizeof(nvm_word), &nvm_word,
11810                                false, NULL);
11811         /* Save off last admin queue command status before releasing
11812          * the NVM
11813          */
11814         last_aq_status = pf->hw.aq.asq_last_status;
11815         i40e_release_nvm(&pf->hw);
11816         if (ret) {
11817                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
11818                          i40e_stat_str(&pf->hw, ret),
11819                          i40e_aq_str(&pf->hw, last_aq_status));
11820                 goto bw_commit_out;
11821         }
11822
11823         /* Wait a bit for NVM release to complete */
11824         msleep(50);
11825
11826         /* Acquire NVM for write access */
11827         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
11828         last_aq_status = pf->hw.aq.asq_last_status;
11829         if (ret) {
11830                 dev_info(&pf->pdev->dev,
11831                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
11832                          i40e_stat_str(&pf->hw, ret),
11833                          i40e_aq_str(&pf->hw, last_aq_status));
11834                 goto bw_commit_out;
11835         }
11836         /* Write it back out unchanged to initiate update NVM,
11837          * which will force a write of the shadow (alt) RAM to
11838          * the NVM - thus storing the bandwidth values permanently.
11839          */
11840         ret = i40e_aq_update_nvm(&pf->hw,
11841                                  I40E_SR_NVM_CONTROL_WORD,
11842                                  0x10, sizeof(nvm_word),
11843                                  &nvm_word, true, 0, NULL);
11844         /* Save off last admin queue command status before releasing
11845          * the NVM
11846          */
11847         last_aq_status = pf->hw.aq.asq_last_status;
11848         i40e_release_nvm(&pf->hw);
11849         if (ret)
11850                 dev_info(&pf->pdev->dev,
11851                          "BW settings NOT SAVED, err %s aq_err %s\n",
11852                          i40e_stat_str(&pf->hw, ret),
11853                          i40e_aq_str(&pf->hw, last_aq_status));
11854 bw_commit_out:
11855
11856         return ret;
11857 }
11858
11859 /**
11860  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
11861  * @pf: board private structure to initialize
11862  *
11863  * i40e_sw_init initializes the Adapter private data structure.
11864  * Fields are initialized based on PCI device information and
11865  * OS network device settings (MTU size).
11866  **/
11867 static int i40e_sw_init(struct i40e_pf *pf)
11868 {
11869         int err = 0;
11870         int size;
11871
11872         /* Set default capability flags */
11873         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
11874                     I40E_FLAG_MSI_ENABLED     |
11875                     I40E_FLAG_MSIX_ENABLED;
11876
11877         /* Set default ITR */
11878         pf->rx_itr_default = I40E_ITR_RX_DEF;
11879         pf->tx_itr_default = I40E_ITR_TX_DEF;
11880
11881         /* Depending on PF configurations, it is possible that the RSS
11882          * maximum might end up larger than the available queues
11883          */
11884         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
11885         pf->alloc_rss_size = 1;
11886         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
11887         pf->rss_size_max = min_t(int, pf->rss_size_max,
11888                                  pf->hw.func_caps.num_tx_qp);
11889         if (pf->hw.func_caps.rss) {
11890                 pf->flags |= I40E_FLAG_RSS_ENABLED;
11891                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
11892                                            num_online_cpus());
11893         }
11894
11895         /* MFP mode enabled */
11896         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
11897                 pf->flags |= I40E_FLAG_MFP_ENABLED;
11898                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
11899                 if (i40e_get_partition_bw_setting(pf)) {
11900                         dev_warn(&pf->pdev->dev,
11901                                  "Could not get partition bw settings\n");
11902                 } else {
11903                         dev_info(&pf->pdev->dev,
11904                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
11905                                  pf->min_bw, pf->max_bw);
11906
11907                         /* nudge the Tx scheduler */
11908                         i40e_set_partition_bw_setting(pf);
11909                 }
11910         }
11911
11912         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
11913             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
11914                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
11915                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
11916                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
11917                     pf->hw.num_partitions > 1)
11918                         dev_info(&pf->pdev->dev,
11919                                  "Flow Director Sideband mode Disabled in MFP mode\n");
11920                 else
11921                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11922                 pf->fdir_pf_filter_count =
11923                                  pf->hw.func_caps.fd_filters_guaranteed;
11924                 pf->hw.fdir_shared_filter_count =
11925                                  pf->hw.func_caps.fd_filters_best_effort;
11926         }
11927
11928         if (pf->hw.mac.type == I40E_MAC_X722) {
11929                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
11930                                     I40E_HW_128_QP_RSS_CAPABLE |
11931                                     I40E_HW_ATR_EVICT_CAPABLE |
11932                                     I40E_HW_WB_ON_ITR_CAPABLE |
11933                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
11934                                     I40E_HW_NO_PCI_LINK_CHECK |
11935                                     I40E_HW_USE_SET_LLDP_MIB |
11936                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
11937                                     I40E_HW_PTP_L4_CAPABLE |
11938                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
11939                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
11940
11941 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
11942                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
11943                     I40E_FDEVICT_PCTYPE_DEFAULT) {
11944                         dev_warn(&pf->pdev->dev,
11945                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
11946                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
11947                 }
11948         } else if ((pf->hw.aq.api_maj_ver > 1) ||
11949                    ((pf->hw.aq.api_maj_ver == 1) &&
11950                     (pf->hw.aq.api_min_ver > 4))) {
11951                 /* Supported in FW API version higher than 1.4 */
11952                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
11953         }
11954
11955         /* Enable HW ATR eviction if possible */
11956         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
11957                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
11958
11959         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11960             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
11961             (pf->hw.aq.fw_maj_ver < 4))) {
11962                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
11963                 /* No DCB support  for FW < v4.33 */
11964                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
11965         }
11966
11967         /* Disable FW LLDP if FW < v4.3 */
11968         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11969             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
11970             (pf->hw.aq.fw_maj_ver < 4)))
11971                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
11972
11973         /* Use the FW Set LLDP MIB API if FW > v4.40 */
11974         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11975             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
11976             (pf->hw.aq.fw_maj_ver >= 5)))
11977                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
11978
11979         /* Enable PTP L4 if FW > v6.0 */
11980         if (pf->hw.mac.type == I40E_MAC_XL710 &&
11981             pf->hw.aq.fw_maj_ver >= 6)
11982                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
11983
11984         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
11985                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
11986                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
11987                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
11988         }
11989
11990         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
11991                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
11992                 /* IWARP needs one extra vector for CQP just like MISC.*/
11993                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
11994         }
11995         /* Stopping FW LLDP engine is supported on XL710 and X722
11996          * starting from FW versions determined in i40e_init_adminq.
11997          * Stopping the FW LLDP engine is not supported on XL710
11998          * if NPAR is functioning so unset this hw flag in this case.
11999          */
12000         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12001             pf->hw.func_caps.npar_enable &&
12002             (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12003                 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12004
12005 #ifdef CONFIG_PCI_IOV
12006         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12007                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12008                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12009                 pf->num_req_vfs = min_t(int,
12010                                         pf->hw.func_caps.num_vfs,
12011                                         I40E_MAX_VF_COUNT);
12012         }
12013 #endif /* CONFIG_PCI_IOV */
12014         pf->eeprom_version = 0xDEAD;
12015         pf->lan_veb = I40E_NO_VEB;
12016         pf->lan_vsi = I40E_NO_VSI;
12017
12018         /* By default FW has this off for performance reasons */
12019         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12020
12021         /* set up queue assignment tracking */
12022         size = sizeof(struct i40e_lump_tracking)
12023                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12024         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12025         if (!pf->qp_pile) {
12026                 err = -ENOMEM;
12027                 goto sw_init_done;
12028         }
12029         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12030         pf->qp_pile->search_hint = 0;
12031
12032         pf->tx_timeout_recovery_level = 1;
12033
12034         mutex_init(&pf->switch_mutex);
12035
12036 sw_init_done:
12037         return err;
12038 }
12039
12040 /**
12041  * i40e_set_ntuple - set the ntuple feature flag and take action
12042  * @pf: board private structure to initialize
12043  * @features: the feature set that the stack is suggesting
12044  *
12045  * returns a bool to indicate if reset needs to happen
12046  **/
12047 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12048 {
12049         bool need_reset = false;
12050
12051         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12052          * the state changed, we need to reset.
12053          */
12054         if (features & NETIF_F_NTUPLE) {
12055                 /* Enable filters and mark for reset */
12056                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12057                         need_reset = true;
12058                 /* enable FD_SB only if there is MSI-X vector and no cloud
12059                  * filters exist
12060                  */
12061                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12062                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12063                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12064                 }
12065         } else {
12066                 /* turn off filters, mark for reset and clear SW filter list */
12067                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12068                         need_reset = true;
12069                         i40e_fdir_filter_exit(pf);
12070                 }
12071                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12072                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12073                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12074
12075                 /* reset fd counters */
12076                 pf->fd_add_err = 0;
12077                 pf->fd_atr_cnt = 0;
12078                 /* if ATR was auto disabled it can be re-enabled. */
12079                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12080                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12081                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12082                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12083         }
12084         return need_reset;
12085 }
12086
12087 /**
12088  * i40e_clear_rss_lut - clear the rx hash lookup table
12089  * @vsi: the VSI being configured
12090  **/
12091 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12092 {
12093         struct i40e_pf *pf = vsi->back;
12094         struct i40e_hw *hw = &pf->hw;
12095         u16 vf_id = vsi->vf_id;
12096         u8 i;
12097
12098         if (vsi->type == I40E_VSI_MAIN) {
12099                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12100                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12101         } else if (vsi->type == I40E_VSI_SRIOV) {
12102                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12103                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12104         } else {
12105                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12106         }
12107 }
12108
12109 /**
12110  * i40e_set_features - set the netdev feature flags
12111  * @netdev: ptr to the netdev being adjusted
12112  * @features: the feature set that the stack is suggesting
12113  * Note: expects to be called while under rtnl_lock()
12114  **/
12115 static int i40e_set_features(struct net_device *netdev,
12116                              netdev_features_t features)
12117 {
12118         struct i40e_netdev_priv *np = netdev_priv(netdev);
12119         struct i40e_vsi *vsi = np->vsi;
12120         struct i40e_pf *pf = vsi->back;
12121         bool need_reset;
12122
12123         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12124                 i40e_pf_config_rss(pf);
12125         else if (!(features & NETIF_F_RXHASH) &&
12126                  netdev->features & NETIF_F_RXHASH)
12127                 i40e_clear_rss_lut(vsi);
12128
12129         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12130                 i40e_vlan_stripping_enable(vsi);
12131         else
12132                 i40e_vlan_stripping_disable(vsi);
12133
12134         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12135                 dev_err(&pf->pdev->dev,
12136                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12137                 return -EINVAL;
12138         }
12139
12140         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12141                 i40e_del_all_macvlans(vsi);
12142
12143         need_reset = i40e_set_ntuple(pf, features);
12144
12145         if (need_reset)
12146                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12147
12148         return 0;
12149 }
12150
12151 /**
12152  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
12153  * @pf: board private structure
12154  * @port: The UDP port to look up
12155  *
12156  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
12157  **/
12158 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
12159 {
12160         u8 i;
12161
12162         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
12163                 /* Do not report ports with pending deletions as
12164                  * being available.
12165                  */
12166                 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i)))
12167                         continue;
12168                 if (pf->udp_ports[i].port == port)
12169                         return i;
12170         }
12171
12172         return i;
12173 }
12174
12175 /**
12176  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
12177  * @netdev: This physical port's netdev
12178  * @ti: Tunnel endpoint information
12179  **/
12180 static void i40e_udp_tunnel_add(struct net_device *netdev,
12181                                 struct udp_tunnel_info *ti)
12182 {
12183         struct i40e_netdev_priv *np = netdev_priv(netdev);
12184         struct i40e_vsi *vsi = np->vsi;
12185         struct i40e_pf *pf = vsi->back;
12186         u16 port = ntohs(ti->port);
12187         u8 next_idx;
12188         u8 idx;
12189
12190         idx = i40e_get_udp_port_idx(pf, port);
12191
12192         /* Check if port already exists */
12193         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
12194                 netdev_info(netdev, "port %d already offloaded\n", port);
12195                 return;
12196         }
12197
12198         /* Now check if there is space to add the new port */
12199         next_idx = i40e_get_udp_port_idx(pf, 0);
12200
12201         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
12202                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
12203                             port);
12204                 return;
12205         }
12206
12207         switch (ti->type) {
12208         case UDP_TUNNEL_TYPE_VXLAN:
12209                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
12210                 break;
12211         case UDP_TUNNEL_TYPE_GENEVE:
12212                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
12213                         return;
12214                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
12215                 break;
12216         default:
12217                 return;
12218         }
12219
12220         /* New port: add it and mark its index in the bitmap */
12221         pf->udp_ports[next_idx].port = port;
12222         pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED;
12223         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
12224         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
12225 }
12226
12227 /**
12228  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
12229  * @netdev: This physical port's netdev
12230  * @ti: Tunnel endpoint information
12231  **/
12232 static void i40e_udp_tunnel_del(struct net_device *netdev,
12233                                 struct udp_tunnel_info *ti)
12234 {
12235         struct i40e_netdev_priv *np = netdev_priv(netdev);
12236         struct i40e_vsi *vsi = np->vsi;
12237         struct i40e_pf *pf = vsi->back;
12238         u16 port = ntohs(ti->port);
12239         u8 idx;
12240
12241         idx = i40e_get_udp_port_idx(pf, port);
12242
12243         /* Check if port already exists */
12244         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
12245                 goto not_found;
12246
12247         switch (ti->type) {
12248         case UDP_TUNNEL_TYPE_VXLAN:
12249                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
12250                         goto not_found;
12251                 break;
12252         case UDP_TUNNEL_TYPE_GENEVE:
12253                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
12254                         goto not_found;
12255                 break;
12256         default:
12257                 goto not_found;
12258         }
12259
12260         /* if port exists, set it to 0 (mark for deletion)
12261          * and make it pending
12262          */
12263         pf->udp_ports[idx].port = 0;
12264
12265         /* Toggle pending bit instead of setting it. This way if we are
12266          * deleting a port that has yet to be added we just clear the pending
12267          * bit and don't have to worry about it.
12268          */
12269         pf->pending_udp_bitmap ^= BIT_ULL(idx);
12270         set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
12271
12272         return;
12273 not_found:
12274         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
12275                     port);
12276 }
12277
12278 static int i40e_get_phys_port_id(struct net_device *netdev,
12279                                  struct netdev_phys_item_id *ppid)
12280 {
12281         struct i40e_netdev_priv *np = netdev_priv(netdev);
12282         struct i40e_pf *pf = np->vsi->back;
12283         struct i40e_hw *hw = &pf->hw;
12284
12285         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12286                 return -EOPNOTSUPP;
12287
12288         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12289         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12290
12291         return 0;
12292 }
12293
12294 /**
12295  * i40e_ndo_fdb_add - add an entry to the hardware database
12296  * @ndm: the input from the stack
12297  * @tb: pointer to array of nladdr (unused)
12298  * @dev: the net device pointer
12299  * @addr: the MAC address entry being added
12300  * @vid: VLAN ID
12301  * @flags: instructions from stack about fdb operation
12302  */
12303 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12304                             struct net_device *dev,
12305                             const unsigned char *addr, u16 vid,
12306                             u16 flags,
12307                             struct netlink_ext_ack *extack)
12308 {
12309         struct i40e_netdev_priv *np = netdev_priv(dev);
12310         struct i40e_pf *pf = np->vsi->back;
12311         int err = 0;
12312
12313         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12314                 return -EOPNOTSUPP;
12315
12316         if (vid) {
12317                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12318                 return -EINVAL;
12319         }
12320
12321         /* Hardware does not support aging addresses so if a
12322          * ndm_state is given only allow permanent addresses
12323          */
12324         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12325                 netdev_info(dev, "FDB only supports static addresses\n");
12326                 return -EINVAL;
12327         }
12328
12329         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12330                 err = dev_uc_add_excl(dev, addr);
12331         else if (is_multicast_ether_addr(addr))
12332                 err = dev_mc_add_excl(dev, addr);
12333         else
12334                 err = -EINVAL;
12335
12336         /* Only return duplicate errors if NLM_F_EXCL is set */
12337         if (err == -EEXIST && !(flags & NLM_F_EXCL))
12338                 err = 0;
12339
12340         return err;
12341 }
12342
12343 /**
12344  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
12345  * @dev: the netdev being configured
12346  * @nlh: RTNL message
12347  * @flags: bridge flags
12348  * @extack: netlink extended ack
12349  *
12350  * Inserts a new hardware bridge if not already created and
12351  * enables the bridging mode requested (VEB or VEPA). If the
12352  * hardware bridge has already been inserted and the request
12353  * is to change the mode then that requires a PF reset to
12354  * allow rebuild of the components with required hardware
12355  * bridge mode enabled.
12356  *
12357  * Note: expects to be called while under rtnl_lock()
12358  **/
12359 static int i40e_ndo_bridge_setlink(struct net_device *dev,
12360                                    struct nlmsghdr *nlh,
12361                                    u16 flags,
12362                                    struct netlink_ext_ack *extack)
12363 {
12364         struct i40e_netdev_priv *np = netdev_priv(dev);
12365         struct i40e_vsi *vsi = np->vsi;
12366         struct i40e_pf *pf = vsi->back;
12367         struct i40e_veb *veb = NULL;
12368         struct nlattr *attr, *br_spec;
12369         int i, rem;
12370
12371         /* Only for PF VSI for now */
12372         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12373                 return -EOPNOTSUPP;
12374
12375         /* Find the HW bridge for PF VSI */
12376         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12377                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12378                         veb = pf->veb[i];
12379         }
12380
12381         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12382
12383         nla_for_each_nested(attr, br_spec, rem) {
12384                 __u16 mode;
12385
12386                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12387                         continue;
12388
12389                 mode = nla_get_u16(attr);
12390                 if ((mode != BRIDGE_MODE_VEPA) &&
12391                     (mode != BRIDGE_MODE_VEB))
12392                         return -EINVAL;
12393
12394                 /* Insert a new HW bridge */
12395                 if (!veb) {
12396                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12397                                              vsi->tc_config.enabled_tc);
12398                         if (veb) {
12399                                 veb->bridge_mode = mode;
12400                                 i40e_config_bridge_mode(veb);
12401                         } else {
12402                                 /* No Bridge HW offload available */
12403                                 return -ENOENT;
12404                         }
12405                         break;
12406                 } else if (mode != veb->bridge_mode) {
12407                         /* Existing HW bridge but different mode needs reset */
12408                         veb->bridge_mode = mode;
12409                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
12410                         if (mode == BRIDGE_MODE_VEB)
12411                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12412                         else
12413                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12414                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12415                         break;
12416                 }
12417         }
12418
12419         return 0;
12420 }
12421
12422 /**
12423  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
12424  * @skb: skb buff
12425  * @pid: process id
12426  * @seq: RTNL message seq #
12427  * @dev: the netdev being configured
12428  * @filter_mask: unused
12429  * @nlflags: netlink flags passed in
12430  *
12431  * Return the mode in which the hardware bridge is operating in
12432  * i.e VEB or VEPA.
12433  **/
12434 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12435                                    struct net_device *dev,
12436                                    u32 __always_unused filter_mask,
12437                                    int nlflags)
12438 {
12439         struct i40e_netdev_priv *np = netdev_priv(dev);
12440         struct i40e_vsi *vsi = np->vsi;
12441         struct i40e_pf *pf = vsi->back;
12442         struct i40e_veb *veb = NULL;
12443         int i;
12444
12445         /* Only for PF VSI for now */
12446         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12447                 return -EOPNOTSUPP;
12448
12449         /* Find the HW bridge for the PF VSI */
12450         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12451                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12452                         veb = pf->veb[i];
12453         }
12454
12455         if (!veb)
12456                 return 0;
12457
12458         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12459                                        0, 0, nlflags, filter_mask, NULL);
12460 }
12461
12462 /**
12463  * i40e_features_check - Validate encapsulated packet conforms to limits
12464  * @skb: skb buff
12465  * @dev: This physical port's netdev
12466  * @features: Offload features that the stack believes apply
12467  **/
12468 static netdev_features_t i40e_features_check(struct sk_buff *skb,
12469                                              struct net_device *dev,
12470                                              netdev_features_t features)
12471 {
12472         size_t len;
12473
12474         /* No point in doing any of this if neither checksum nor GSO are
12475          * being requested for this frame.  We can rule out both by just
12476          * checking for CHECKSUM_PARTIAL
12477          */
12478         if (skb->ip_summed != CHECKSUM_PARTIAL)
12479                 return features;
12480
12481         /* We cannot support GSO if the MSS is going to be less than
12482          * 64 bytes.  If it is then we need to drop support for GSO.
12483          */
12484         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12485                 features &= ~NETIF_F_GSO_MASK;
12486
12487         /* MACLEN can support at most 63 words */
12488         len = skb_network_header(skb) - skb->data;
12489         if (len & ~(63 * 2))
12490                 goto out_err;
12491
12492         /* IPLEN and EIPLEN can support at most 127 dwords */
12493         len = skb_transport_header(skb) - skb_network_header(skb);
12494         if (len & ~(127 * 4))
12495                 goto out_err;
12496
12497         if (skb->encapsulation) {
12498                 /* L4TUNLEN can support 127 words */
12499                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12500                 if (len & ~(127 * 2))
12501                         goto out_err;
12502
12503                 /* IPLEN can support at most 127 dwords */
12504                 len = skb_inner_transport_header(skb) -
12505                       skb_inner_network_header(skb);
12506                 if (len & ~(127 * 4))
12507                         goto out_err;
12508         }
12509
12510         /* No need to validate L4LEN as TCP is the only protocol with a
12511          * a flexible value and we support all possible values supported
12512          * by TCP, which is at most 15 dwords
12513          */
12514
12515         return features;
12516 out_err:
12517         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12518 }
12519
12520 /**
12521  * i40e_xdp_setup - add/remove an XDP program
12522  * @vsi: VSI to changed
12523  * @prog: XDP program
12524  **/
12525 static int i40e_xdp_setup(struct i40e_vsi *vsi,
12526                           struct bpf_prog *prog)
12527 {
12528         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
12529         struct i40e_pf *pf = vsi->back;
12530         struct bpf_prog *old_prog;
12531         bool need_reset;
12532         int i;
12533
12534         /* Don't allow frames that span over multiple buffers */
12535         if (frame_size > vsi->rx_buf_len)
12536                 return -EINVAL;
12537
12538         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
12539                 return 0;
12540
12541         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
12542         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
12543
12544         if (need_reset)
12545                 i40e_prep_for_reset(pf, true);
12546
12547         old_prog = xchg(&vsi->xdp_prog, prog);
12548
12549         if (need_reset) {
12550                 if (!prog)
12551                         /* Wait until ndo_xsk_wakeup completes. */
12552                         synchronize_rcu();
12553                 i40e_reset_and_rebuild(pf, true, true);
12554         }
12555
12556         for (i = 0; i < vsi->num_queue_pairs; i++)
12557                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
12558
12559         if (old_prog)
12560                 bpf_prog_put(old_prog);
12561
12562         /* Kick start the NAPI context if there is an AF_XDP socket open
12563          * on that queue id. This so that receiving will start.
12564          */
12565         if (need_reset && prog)
12566                 for (i = 0; i < vsi->num_queue_pairs; i++)
12567                         if (vsi->xdp_rings[i]->xsk_umem)
12568                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
12569                                                       XDP_WAKEUP_RX);
12570
12571         return 0;
12572 }
12573
12574 /**
12575  * i40e_enter_busy_conf - Enters busy config state
12576  * @vsi: vsi
12577  *
12578  * Returns 0 on success, <0 for failure.
12579  **/
12580 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
12581 {
12582         struct i40e_pf *pf = vsi->back;
12583         int timeout = 50;
12584
12585         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
12586                 timeout--;
12587                 if (!timeout)
12588                         return -EBUSY;
12589                 usleep_range(1000, 2000);
12590         }
12591
12592         return 0;
12593 }
12594
12595 /**
12596  * i40e_exit_busy_conf - Exits busy config state
12597  * @vsi: vsi
12598  **/
12599 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
12600 {
12601         struct i40e_pf *pf = vsi->back;
12602
12603         clear_bit(__I40E_CONFIG_BUSY, pf->state);
12604 }
12605
12606 /**
12607  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
12608  * @vsi: vsi
12609  * @queue_pair: queue pair
12610  **/
12611 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
12612 {
12613         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
12614                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
12615         memset(&vsi->tx_rings[queue_pair]->stats, 0,
12616                sizeof(vsi->tx_rings[queue_pair]->stats));
12617         if (i40e_enabled_xdp_vsi(vsi)) {
12618                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
12619                        sizeof(vsi->xdp_rings[queue_pair]->stats));
12620         }
12621 }
12622
12623 /**
12624  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
12625  * @vsi: vsi
12626  * @queue_pair: queue pair
12627  **/
12628 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
12629 {
12630         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
12631         if (i40e_enabled_xdp_vsi(vsi)) {
12632                 /* Make sure that in-progress ndo_xdp_xmit calls are
12633                  * completed.
12634                  */
12635                 synchronize_rcu();
12636                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
12637         }
12638         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
12639 }
12640
12641 /**
12642  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
12643  * @vsi: vsi
12644  * @queue_pair: queue pair
12645  * @enable: true for enable, false for disable
12646  **/
12647 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
12648                                         bool enable)
12649 {
12650         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12651         struct i40e_q_vector *q_vector = rxr->q_vector;
12652
12653         if (!vsi->netdev)
12654                 return;
12655
12656         /* All rings in a qp belong to the same qvector. */
12657         if (q_vector->rx.ring || q_vector->tx.ring) {
12658                 if (enable)
12659                         napi_enable(&q_vector->napi);
12660                 else
12661                         napi_disable(&q_vector->napi);
12662         }
12663 }
12664
12665 /**
12666  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
12667  * @vsi: vsi
12668  * @queue_pair: queue pair
12669  * @enable: true for enable, false for disable
12670  *
12671  * Returns 0 on success, <0 on failure.
12672  **/
12673 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
12674                                         bool enable)
12675 {
12676         struct i40e_pf *pf = vsi->back;
12677         int pf_q, ret = 0;
12678
12679         pf_q = vsi->base_queue + queue_pair;
12680         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
12681                                      false /*is xdp*/, enable);
12682         if (ret) {
12683                 dev_info(&pf->pdev->dev,
12684                          "VSI seid %d Tx ring %d %sable timeout\n",
12685                          vsi->seid, pf_q, (enable ? "en" : "dis"));
12686                 return ret;
12687         }
12688
12689         i40e_control_rx_q(pf, pf_q, enable);
12690         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
12691         if (ret) {
12692                 dev_info(&pf->pdev->dev,
12693                          "VSI seid %d Rx ring %d %sable timeout\n",
12694                          vsi->seid, pf_q, (enable ? "en" : "dis"));
12695                 return ret;
12696         }
12697
12698         /* Due to HW errata, on Rx disable only, the register can
12699          * indicate done before it really is. Needs 50ms to be sure
12700          */
12701         if (!enable)
12702                 mdelay(50);
12703
12704         if (!i40e_enabled_xdp_vsi(vsi))
12705                 return ret;
12706
12707         ret = i40e_control_wait_tx_q(vsi->seid, pf,
12708                                      pf_q + vsi->alloc_queue_pairs,
12709                                      true /*is xdp*/, enable);
12710         if (ret) {
12711                 dev_info(&pf->pdev->dev,
12712                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
12713                          vsi->seid, pf_q, (enable ? "en" : "dis"));
12714         }
12715
12716         return ret;
12717 }
12718
12719 /**
12720  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
12721  * @vsi: vsi
12722  * @queue_pair: queue_pair
12723  **/
12724 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
12725 {
12726         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12727         struct i40e_pf *pf = vsi->back;
12728         struct i40e_hw *hw = &pf->hw;
12729
12730         /* All rings in a qp belong to the same qvector. */
12731         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
12732                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
12733         else
12734                 i40e_irq_dynamic_enable_icr0(pf);
12735
12736         i40e_flush(hw);
12737 }
12738
12739 /**
12740  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
12741  * @vsi: vsi
12742  * @queue_pair: queue_pair
12743  **/
12744 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
12745 {
12746         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12747         struct i40e_pf *pf = vsi->back;
12748         struct i40e_hw *hw = &pf->hw;
12749
12750         /* For simplicity, instead of removing the qp interrupt causes
12751          * from the interrupt linked list, we simply disable the interrupt, and
12752          * leave the list intact.
12753          *
12754          * All rings in a qp belong to the same qvector.
12755          */
12756         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12757                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
12758
12759                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
12760                 i40e_flush(hw);
12761                 synchronize_irq(pf->msix_entries[intpf].vector);
12762         } else {
12763                 /* Legacy and MSI mode - this stops all interrupt handling */
12764                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
12765                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
12766                 i40e_flush(hw);
12767                 synchronize_irq(pf->pdev->irq);
12768         }
12769 }
12770
12771 /**
12772  * i40e_queue_pair_disable - Disables a queue pair
12773  * @vsi: vsi
12774  * @queue_pair: queue pair
12775  *
12776  * Returns 0 on success, <0 on failure.
12777  **/
12778 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
12779 {
12780         int err;
12781
12782         err = i40e_enter_busy_conf(vsi);
12783         if (err)
12784                 return err;
12785
12786         i40e_queue_pair_disable_irq(vsi, queue_pair);
12787         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
12788         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
12789         i40e_queue_pair_clean_rings(vsi, queue_pair);
12790         i40e_queue_pair_reset_stats(vsi, queue_pair);
12791
12792         return err;
12793 }
12794
12795 /**
12796  * i40e_queue_pair_enable - Enables a queue pair
12797  * @vsi: vsi
12798  * @queue_pair: queue pair
12799  *
12800  * Returns 0 on success, <0 on failure.
12801  **/
12802 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
12803 {
12804         int err;
12805
12806         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
12807         if (err)
12808                 return err;
12809
12810         if (i40e_enabled_xdp_vsi(vsi)) {
12811                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
12812                 if (err)
12813                         return err;
12814         }
12815
12816         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
12817         if (err)
12818                 return err;
12819
12820         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
12821         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
12822         i40e_queue_pair_enable_irq(vsi, queue_pair);
12823
12824         i40e_exit_busy_conf(vsi);
12825
12826         return err;
12827 }
12828
12829 /**
12830  * i40e_xdp - implements ndo_bpf for i40e
12831  * @dev: netdevice
12832  * @xdp: XDP command
12833  **/
12834 static int i40e_xdp(struct net_device *dev,
12835                     struct netdev_bpf *xdp)
12836 {
12837         struct i40e_netdev_priv *np = netdev_priv(dev);
12838         struct i40e_vsi *vsi = np->vsi;
12839
12840         if (vsi->type != I40E_VSI_MAIN)
12841                 return -EINVAL;
12842
12843         switch (xdp->command) {
12844         case XDP_SETUP_PROG:
12845                 return i40e_xdp_setup(vsi, xdp->prog);
12846         case XDP_QUERY_PROG:
12847                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
12848                 return 0;
12849         case XDP_SETUP_XSK_UMEM:
12850                 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem,
12851                                            xdp->xsk.queue_id);
12852         default:
12853                 return -EINVAL;
12854         }
12855 }
12856
12857 static const struct net_device_ops i40e_netdev_ops = {
12858         .ndo_open               = i40e_open,
12859         .ndo_stop               = i40e_close,
12860         .ndo_start_xmit         = i40e_lan_xmit_frame,
12861         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
12862         .ndo_set_rx_mode        = i40e_set_rx_mode,
12863         .ndo_validate_addr      = eth_validate_addr,
12864         .ndo_set_mac_address    = i40e_set_mac,
12865         .ndo_change_mtu         = i40e_change_mtu,
12866         .ndo_do_ioctl           = i40e_ioctl,
12867         .ndo_tx_timeout         = i40e_tx_timeout,
12868         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
12869         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
12870 #ifdef CONFIG_NET_POLL_CONTROLLER
12871         .ndo_poll_controller    = i40e_netpoll,
12872 #endif
12873         .ndo_setup_tc           = __i40e_setup_tc,
12874         .ndo_set_features       = i40e_set_features,
12875         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
12876         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
12877         .ndo_get_vf_stats       = i40e_get_vf_stats,
12878         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
12879         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
12880         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
12881         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
12882         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
12883         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
12884         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
12885         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
12886         .ndo_fdb_add            = i40e_ndo_fdb_add,
12887         .ndo_features_check     = i40e_features_check,
12888         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
12889         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
12890         .ndo_bpf                = i40e_xdp,
12891         .ndo_xdp_xmit           = i40e_xdp_xmit,
12892         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
12893         .ndo_dfwd_add_station   = i40e_fwd_add,
12894         .ndo_dfwd_del_station   = i40e_fwd_del,
12895 };
12896
12897 /**
12898  * i40e_config_netdev - Setup the netdev flags
12899  * @vsi: the VSI being configured
12900  *
12901  * Returns 0 on success, negative value on failure
12902  **/
12903 static int i40e_config_netdev(struct i40e_vsi *vsi)
12904 {
12905         struct i40e_pf *pf = vsi->back;
12906         struct i40e_hw *hw = &pf->hw;
12907         struct i40e_netdev_priv *np;
12908         struct net_device *netdev;
12909         u8 broadcast[ETH_ALEN];
12910         u8 mac_addr[ETH_ALEN];
12911         int etherdev_size;
12912         netdev_features_t hw_enc_features;
12913         netdev_features_t hw_features;
12914
12915         etherdev_size = sizeof(struct i40e_netdev_priv);
12916         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
12917         if (!netdev)
12918                 return -ENOMEM;
12919
12920         vsi->netdev = netdev;
12921         np = netdev_priv(netdev);
12922         np->vsi = vsi;
12923
12924         hw_enc_features = NETIF_F_SG                    |
12925                           NETIF_F_IP_CSUM               |
12926                           NETIF_F_IPV6_CSUM             |
12927                           NETIF_F_HIGHDMA               |
12928                           NETIF_F_SOFT_FEATURES         |
12929                           NETIF_F_TSO                   |
12930                           NETIF_F_TSO_ECN               |
12931                           NETIF_F_TSO6                  |
12932                           NETIF_F_GSO_GRE               |
12933                           NETIF_F_GSO_GRE_CSUM          |
12934                           NETIF_F_GSO_PARTIAL           |
12935                           NETIF_F_GSO_IPXIP4            |
12936                           NETIF_F_GSO_IPXIP6            |
12937                           NETIF_F_GSO_UDP_TUNNEL        |
12938                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
12939                           NETIF_F_GSO_UDP_L4            |
12940                           NETIF_F_SCTP_CRC              |
12941                           NETIF_F_RXHASH                |
12942                           NETIF_F_RXCSUM                |
12943                           0;
12944
12945         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
12946                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
12947
12948         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
12949
12950         netdev->hw_enc_features |= hw_enc_features;
12951
12952         /* record features VLANs can make use of */
12953         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
12954
12955         /* enable macvlan offloads */
12956         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
12957
12958         hw_features = hw_enc_features           |
12959                       NETIF_F_HW_VLAN_CTAG_TX   |
12960                       NETIF_F_HW_VLAN_CTAG_RX;
12961
12962         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
12963                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
12964
12965         netdev->hw_features |= hw_features;
12966
12967         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
12968         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
12969
12970         if (vsi->type == I40E_VSI_MAIN) {
12971                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
12972                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
12973                 /* The following steps are necessary for two reasons. First,
12974                  * some older NVM configurations load a default MAC-VLAN
12975                  * filter that will accept any tagged packet, and we want to
12976                  * replace this with a normal filter. Additionally, it is
12977                  * possible our MAC address was provided by the platform using
12978                  * Open Firmware or similar.
12979                  *
12980                  * Thus, we need to remove the default filter and install one
12981                  * specific to the MAC address.
12982                  */
12983                 i40e_rm_default_mac_filter(vsi, mac_addr);
12984                 spin_lock_bh(&vsi->mac_filter_hash_lock);
12985                 i40e_add_mac_filter(vsi, mac_addr);
12986                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
12987         } else {
12988                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
12989                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
12990                  * the end, which is 4 bytes long, so force truncation of the
12991                  * original name by IFNAMSIZ - 4
12992                  */
12993                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
12994                          IFNAMSIZ - 4,
12995                          pf->vsi[pf->lan_vsi]->netdev->name);
12996                 eth_random_addr(mac_addr);
12997
12998                 spin_lock_bh(&vsi->mac_filter_hash_lock);
12999                 i40e_add_mac_filter(vsi, mac_addr);
13000                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13001         }
13002
13003         /* Add the broadcast filter so that we initially will receive
13004          * broadcast packets. Note that when a new VLAN is first added the
13005          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13006          * specific filters as part of transitioning into "vlan" operation.
13007          * When more VLANs are added, the driver will copy each existing MAC
13008          * filter and add it for the new VLAN.
13009          *
13010          * Broadcast filters are handled specially by
13011          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13012          * promiscuous bit instead of adding this directly as a MAC/VLAN
13013          * filter. The subtask will update the correct broadcast promiscuous
13014          * bits as VLANs become active or inactive.
13015          */
13016         eth_broadcast_addr(broadcast);
13017         spin_lock_bh(&vsi->mac_filter_hash_lock);
13018         i40e_add_mac_filter(vsi, broadcast);
13019         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13020
13021         ether_addr_copy(netdev->dev_addr, mac_addr);
13022         ether_addr_copy(netdev->perm_addr, mac_addr);
13023
13024         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13025         netdev->neigh_priv_len = sizeof(u32) * 4;
13026
13027         netdev->priv_flags |= IFF_UNICAST_FLT;
13028         netdev->priv_flags |= IFF_SUPP_NOFCS;
13029         /* Setup netdev TC information */
13030         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13031
13032         netdev->netdev_ops = &i40e_netdev_ops;
13033         netdev->watchdog_timeo = 5 * HZ;
13034         i40e_set_ethtool_ops(netdev);
13035
13036         /* MTU range: 68 - 9706 */
13037         netdev->min_mtu = ETH_MIN_MTU;
13038         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13039
13040         return 0;
13041 }
13042
13043 /**
13044  * i40e_vsi_delete - Delete a VSI from the switch
13045  * @vsi: the VSI being removed
13046  *
13047  * Returns 0 on success, negative value on failure
13048  **/
13049 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13050 {
13051         /* remove default VSI is not allowed */
13052         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13053                 return;
13054
13055         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13056 }
13057
13058 /**
13059  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13060  * @vsi: the VSI being queried
13061  *
13062  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13063  **/
13064 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13065 {
13066         struct i40e_veb *veb;
13067         struct i40e_pf *pf = vsi->back;
13068
13069         /* Uplink is not a bridge so default to VEB */
13070         if (vsi->veb_idx >= I40E_MAX_VEB)
13071                 return 1;
13072
13073         veb = pf->veb[vsi->veb_idx];
13074         if (!veb) {
13075                 dev_info(&pf->pdev->dev,
13076                          "There is no veb associated with the bridge\n");
13077                 return -ENOENT;
13078         }
13079
13080         /* Uplink is a bridge in VEPA mode */
13081         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13082                 return 0;
13083         } else {
13084                 /* Uplink is a bridge in VEB mode */
13085                 return 1;
13086         }
13087
13088         /* VEPA is now default bridge, so return 0 */
13089         return 0;
13090 }
13091
13092 /**
13093  * i40e_add_vsi - Add a VSI to the switch
13094  * @vsi: the VSI being configured
13095  *
13096  * This initializes a VSI context depending on the VSI type to be added and
13097  * passes it down to the add_vsi aq command.
13098  **/
13099 static int i40e_add_vsi(struct i40e_vsi *vsi)
13100 {
13101         int ret = -ENODEV;
13102         struct i40e_pf *pf = vsi->back;
13103         struct i40e_hw *hw = &pf->hw;
13104         struct i40e_vsi_context ctxt;
13105         struct i40e_mac_filter *f;
13106         struct hlist_node *h;
13107         int bkt;
13108
13109         u8 enabled_tc = 0x1; /* TC0 enabled */
13110         int f_count = 0;
13111
13112         memset(&ctxt, 0, sizeof(ctxt));
13113         switch (vsi->type) {
13114         case I40E_VSI_MAIN:
13115                 /* The PF's main VSI is already setup as part of the
13116                  * device initialization, so we'll not bother with
13117                  * the add_vsi call, but we will retrieve the current
13118                  * VSI context.
13119                  */
13120                 ctxt.seid = pf->main_vsi_seid;
13121                 ctxt.pf_num = pf->hw.pf_id;
13122                 ctxt.vf_num = 0;
13123                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13124                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13125                 if (ret) {
13126                         dev_info(&pf->pdev->dev,
13127                                  "couldn't get PF vsi config, err %s aq_err %s\n",
13128                                  i40e_stat_str(&pf->hw, ret),
13129                                  i40e_aq_str(&pf->hw,
13130                                              pf->hw.aq.asq_last_status));
13131                         return -ENOENT;
13132                 }
13133                 vsi->info = ctxt.info;
13134                 vsi->info.valid_sections = 0;
13135
13136                 vsi->seid = ctxt.seid;
13137                 vsi->id = ctxt.vsi_number;
13138
13139                 enabled_tc = i40e_pf_get_tc_map(pf);
13140
13141                 /* Source pruning is enabled by default, so the flag is
13142                  * negative logic - if it's set, we need to fiddle with
13143                  * the VSI to disable source pruning.
13144                  */
13145                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13146                         memset(&ctxt, 0, sizeof(ctxt));
13147                         ctxt.seid = pf->main_vsi_seid;
13148                         ctxt.pf_num = pf->hw.pf_id;
13149                         ctxt.vf_num = 0;
13150                         ctxt.info.valid_sections |=
13151                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13152                         ctxt.info.switch_id =
13153                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13154                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13155                         if (ret) {
13156                                 dev_info(&pf->pdev->dev,
13157                                          "update vsi failed, err %s aq_err %s\n",
13158                                          i40e_stat_str(&pf->hw, ret),
13159                                          i40e_aq_str(&pf->hw,
13160                                                      pf->hw.aq.asq_last_status));
13161                                 ret = -ENOENT;
13162                                 goto err;
13163                         }
13164                 }
13165
13166                 /* MFP mode setup queue map and update VSI */
13167                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13168                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13169                         memset(&ctxt, 0, sizeof(ctxt));
13170                         ctxt.seid = pf->main_vsi_seid;
13171                         ctxt.pf_num = pf->hw.pf_id;
13172                         ctxt.vf_num = 0;
13173                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13174                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13175                         if (ret) {
13176                                 dev_info(&pf->pdev->dev,
13177                                          "update vsi failed, err %s aq_err %s\n",
13178                                          i40e_stat_str(&pf->hw, ret),
13179                                          i40e_aq_str(&pf->hw,
13180                                                     pf->hw.aq.asq_last_status));
13181                                 ret = -ENOENT;
13182                                 goto err;
13183                         }
13184                         /* update the local VSI info queue map */
13185                         i40e_vsi_update_queue_map(vsi, &ctxt);
13186                         vsi->info.valid_sections = 0;
13187                 } else {
13188                         /* Default/Main VSI is only enabled for TC0
13189                          * reconfigure it to enable all TCs that are
13190                          * available on the port in SFP mode.
13191                          * For MFP case the iSCSI PF would use this
13192                          * flow to enable LAN+iSCSI TC.
13193                          */
13194                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13195                         if (ret) {
13196                                 /* Single TC condition is not fatal,
13197                                  * message and continue
13198                                  */
13199                                 dev_info(&pf->pdev->dev,
13200                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13201                                          enabled_tc,
13202                                          i40e_stat_str(&pf->hw, ret),
13203                                          i40e_aq_str(&pf->hw,
13204                                                     pf->hw.aq.asq_last_status));
13205                         }
13206                 }
13207                 break;
13208
13209         case I40E_VSI_FDIR:
13210                 ctxt.pf_num = hw->pf_id;
13211                 ctxt.vf_num = 0;
13212                 ctxt.uplink_seid = vsi->uplink_seid;
13213                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13214                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13215                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13216                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13217                         ctxt.info.valid_sections |=
13218                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13219                         ctxt.info.switch_id =
13220                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13221                 }
13222                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13223                 break;
13224
13225         case I40E_VSI_VMDQ2:
13226                 ctxt.pf_num = hw->pf_id;
13227                 ctxt.vf_num = 0;
13228                 ctxt.uplink_seid = vsi->uplink_seid;
13229                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13230                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13231
13232                 /* This VSI is connected to VEB so the switch_id
13233                  * should be set to zero by default.
13234                  */
13235                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13236                         ctxt.info.valid_sections |=
13237                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13238                         ctxt.info.switch_id =
13239                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13240                 }
13241
13242                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13243                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13244                 break;
13245
13246         case I40E_VSI_SRIOV:
13247                 ctxt.pf_num = hw->pf_id;
13248                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13249                 ctxt.uplink_seid = vsi->uplink_seid;
13250                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13251                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13252
13253                 /* This VSI is connected to VEB so the switch_id
13254                  * should be set to zero by default.
13255                  */
13256                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13257                         ctxt.info.valid_sections |=
13258                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13259                         ctxt.info.switch_id =
13260                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13261                 }
13262
13263                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13264                         ctxt.info.valid_sections |=
13265                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13266                         ctxt.info.queueing_opt_flags |=
13267                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13268                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13269                 }
13270
13271                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13272                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13273                 if (pf->vf[vsi->vf_id].spoofchk) {
13274                         ctxt.info.valid_sections |=
13275                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13276                         ctxt.info.sec_flags |=
13277                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13278                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13279                 }
13280                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13281                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13282                 break;
13283
13284         case I40E_VSI_IWARP:
13285                 /* send down message to iWARP */
13286                 break;
13287
13288         default:
13289                 return -ENODEV;
13290         }
13291
13292         if (vsi->type != I40E_VSI_MAIN) {
13293                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13294                 if (ret) {
13295                         dev_info(&vsi->back->pdev->dev,
13296                                  "add vsi failed, err %s aq_err %s\n",
13297                                  i40e_stat_str(&pf->hw, ret),
13298                                  i40e_aq_str(&pf->hw,
13299                                              pf->hw.aq.asq_last_status));
13300                         ret = -ENOENT;
13301                         goto err;
13302                 }
13303                 vsi->info = ctxt.info;
13304                 vsi->info.valid_sections = 0;
13305                 vsi->seid = ctxt.seid;
13306                 vsi->id = ctxt.vsi_number;
13307         }
13308
13309         vsi->active_filters = 0;
13310         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13311         spin_lock_bh(&vsi->mac_filter_hash_lock);
13312         /* If macvlan filters already exist, force them to get loaded */
13313         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13314                 f->state = I40E_FILTER_NEW;
13315                 f_count++;
13316         }
13317         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13318
13319         if (f_count) {
13320                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13321                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13322         }
13323
13324         /* Update VSI BW information */
13325         ret = i40e_vsi_get_bw_info(vsi);
13326         if (ret) {
13327                 dev_info(&pf->pdev->dev,
13328                          "couldn't get vsi bw info, err %s aq_err %s\n",
13329                          i40e_stat_str(&pf->hw, ret),
13330                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13331                 /* VSI is already added so not tearing that up */
13332                 ret = 0;
13333         }
13334
13335 err:
13336         return ret;
13337 }
13338
13339 /**
13340  * i40e_vsi_release - Delete a VSI and free its resources
13341  * @vsi: the VSI being removed
13342  *
13343  * Returns 0 on success or < 0 on error
13344  **/
13345 int i40e_vsi_release(struct i40e_vsi *vsi)
13346 {
13347         struct i40e_mac_filter *f;
13348         struct hlist_node *h;
13349         struct i40e_veb *veb = NULL;
13350         struct i40e_pf *pf;
13351         u16 uplink_seid;
13352         int i, n, bkt;
13353
13354         pf = vsi->back;
13355
13356         /* release of a VEB-owner or last VSI is not allowed */
13357         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13358                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13359                          vsi->seid, vsi->uplink_seid);
13360                 return -ENODEV;
13361         }
13362         if (vsi == pf->vsi[pf->lan_vsi] &&
13363             !test_bit(__I40E_DOWN, pf->state)) {
13364                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13365                 return -ENODEV;
13366         }
13367
13368         uplink_seid = vsi->uplink_seid;
13369         if (vsi->type != I40E_VSI_SRIOV) {
13370                 if (vsi->netdev_registered) {
13371                         vsi->netdev_registered = false;
13372                         if (vsi->netdev) {
13373                                 /* results in a call to i40e_close() */
13374                                 unregister_netdev(vsi->netdev);
13375                         }
13376                 } else {
13377                         i40e_vsi_close(vsi);
13378                 }
13379                 i40e_vsi_disable_irq(vsi);
13380         }
13381
13382         spin_lock_bh(&vsi->mac_filter_hash_lock);
13383
13384         /* clear the sync flag on all filters */
13385         if (vsi->netdev) {
13386                 __dev_uc_unsync(vsi->netdev, NULL);
13387                 __dev_mc_unsync(vsi->netdev, NULL);
13388         }
13389
13390         /* make sure any remaining filters are marked for deletion */
13391         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13392                 __i40e_del_filter(vsi, f);
13393
13394         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13395
13396         i40e_sync_vsi_filters(vsi);
13397
13398         i40e_vsi_delete(vsi);
13399         i40e_vsi_free_q_vectors(vsi);
13400         if (vsi->netdev) {
13401                 free_netdev(vsi->netdev);
13402                 vsi->netdev = NULL;
13403         }
13404         i40e_vsi_clear_rings(vsi);
13405         i40e_vsi_clear(vsi);
13406
13407         /* If this was the last thing on the VEB, except for the
13408          * controlling VSI, remove the VEB, which puts the controlling
13409          * VSI onto the next level down in the switch.
13410          *
13411          * Well, okay, there's one more exception here: don't remove
13412          * the orphan VEBs yet.  We'll wait for an explicit remove request
13413          * from up the network stack.
13414          */
13415         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13416                 if (pf->vsi[i] &&
13417                     pf->vsi[i]->uplink_seid == uplink_seid &&
13418                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13419                         n++;      /* count the VSIs */
13420                 }
13421         }
13422         for (i = 0; i < I40E_MAX_VEB; i++) {
13423                 if (!pf->veb[i])
13424                         continue;
13425                 if (pf->veb[i]->uplink_seid == uplink_seid)
13426                         n++;     /* count the VEBs */
13427                 if (pf->veb[i]->seid == uplink_seid)
13428                         veb = pf->veb[i];
13429         }
13430         if (n == 0 && veb && veb->uplink_seid != 0)
13431                 i40e_veb_release(veb);
13432
13433         return 0;
13434 }
13435
13436 /**
13437  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
13438  * @vsi: ptr to the VSI
13439  *
13440  * This should only be called after i40e_vsi_mem_alloc() which allocates the
13441  * corresponding SW VSI structure and initializes num_queue_pairs for the
13442  * newly allocated VSI.
13443  *
13444  * Returns 0 on success or negative on failure
13445  **/
13446 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13447 {
13448         int ret = -ENOENT;
13449         struct i40e_pf *pf = vsi->back;
13450
13451         if (vsi->q_vectors[0]) {
13452                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13453                          vsi->seid);
13454                 return -EEXIST;
13455         }
13456
13457         if (vsi->base_vector) {
13458                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13459                          vsi->seid, vsi->base_vector);
13460                 return -EEXIST;
13461         }
13462
13463         ret = i40e_vsi_alloc_q_vectors(vsi);
13464         if (ret) {
13465                 dev_info(&pf->pdev->dev,
13466                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13467                          vsi->num_q_vectors, vsi->seid, ret);
13468                 vsi->num_q_vectors = 0;
13469                 goto vector_setup_out;
13470         }
13471
13472         /* In Legacy mode, we do not have to get any other vector since we
13473          * piggyback on the misc/ICR0 for queue interrupts.
13474         */
13475         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13476                 return ret;
13477         if (vsi->num_q_vectors)
13478                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13479                                                  vsi->num_q_vectors, vsi->idx);
13480         if (vsi->base_vector < 0) {
13481                 dev_info(&pf->pdev->dev,
13482                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13483                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13484                 i40e_vsi_free_q_vectors(vsi);
13485                 ret = -ENOENT;
13486                 goto vector_setup_out;
13487         }
13488
13489 vector_setup_out:
13490         return ret;
13491 }
13492
13493 /**
13494  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
13495  * @vsi: pointer to the vsi.
13496  *
13497  * This re-allocates a vsi's queue resources.
13498  *
13499  * Returns pointer to the successfully allocated and configured VSI sw struct
13500  * on success, otherwise returns NULL on failure.
13501  **/
13502 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13503 {
13504         u16 alloc_queue_pairs;
13505         struct i40e_pf *pf;
13506         u8 enabled_tc;
13507         int ret;
13508
13509         if (!vsi)
13510                 return NULL;
13511
13512         pf = vsi->back;
13513
13514         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
13515         i40e_vsi_clear_rings(vsi);
13516
13517         i40e_vsi_free_arrays(vsi, false);
13518         i40e_set_num_rings_in_vsi(vsi);
13519         ret = i40e_vsi_alloc_arrays(vsi, false);
13520         if (ret)
13521                 goto err_vsi;
13522
13523         alloc_queue_pairs = vsi->alloc_queue_pairs *
13524                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13525
13526         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13527         if (ret < 0) {
13528                 dev_info(&pf->pdev->dev,
13529                          "failed to get tracking for %d queues for VSI %d err %d\n",
13530                          alloc_queue_pairs, vsi->seid, ret);
13531                 goto err_vsi;
13532         }
13533         vsi->base_queue = ret;
13534
13535         /* Update the FW view of the VSI. Force a reset of TC and queue
13536          * layout configurations.
13537          */
13538         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13539         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13540         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13541         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13542         if (vsi->type == I40E_VSI_MAIN)
13543                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
13544
13545         /* assign it some queues */
13546         ret = i40e_alloc_rings(vsi);
13547         if (ret)
13548                 goto err_rings;
13549
13550         /* map all of the rings to the q_vectors */
13551         i40e_vsi_map_rings_to_vectors(vsi);
13552         return vsi;
13553
13554 err_rings:
13555         i40e_vsi_free_q_vectors(vsi);
13556         if (vsi->netdev_registered) {
13557                 vsi->netdev_registered = false;
13558                 unregister_netdev(vsi->netdev);
13559                 free_netdev(vsi->netdev);
13560                 vsi->netdev = NULL;
13561         }
13562         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13563 err_vsi:
13564         i40e_vsi_clear(vsi);
13565         return NULL;
13566 }
13567
13568 /**
13569  * i40e_vsi_setup - Set up a VSI by a given type
13570  * @pf: board private structure
13571  * @type: VSI type
13572  * @uplink_seid: the switch element to link to
13573  * @param1: usage depends upon VSI type. For VF types, indicates VF id
13574  *
13575  * This allocates the sw VSI structure and its queue resources, then add a VSI
13576  * to the identified VEB.
13577  *
13578  * Returns pointer to the successfully allocated and configure VSI sw struct on
13579  * success, otherwise returns NULL on failure.
13580  **/
13581 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
13582                                 u16 uplink_seid, u32 param1)
13583 {
13584         struct i40e_vsi *vsi = NULL;
13585         struct i40e_veb *veb = NULL;
13586         u16 alloc_queue_pairs;
13587         int ret, i;
13588         int v_idx;
13589
13590         /* The requested uplink_seid must be either
13591          *     - the PF's port seid
13592          *              no VEB is needed because this is the PF
13593          *              or this is a Flow Director special case VSI
13594          *     - seid of an existing VEB
13595          *     - seid of a VSI that owns an existing VEB
13596          *     - seid of a VSI that doesn't own a VEB
13597          *              a new VEB is created and the VSI becomes the owner
13598          *     - seid of the PF VSI, which is what creates the first VEB
13599          *              this is a special case of the previous
13600          *
13601          * Find which uplink_seid we were given and create a new VEB if needed
13602          */
13603         for (i = 0; i < I40E_MAX_VEB; i++) {
13604                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
13605                         veb = pf->veb[i];
13606                         break;
13607                 }
13608         }
13609
13610         if (!veb && uplink_seid != pf->mac_seid) {
13611
13612                 for (i = 0; i < pf->num_alloc_vsi; i++) {
13613                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
13614                                 vsi = pf->vsi[i];
13615                                 break;
13616                         }
13617                 }
13618                 if (!vsi) {
13619                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
13620                                  uplink_seid);
13621                         return NULL;
13622                 }
13623
13624                 if (vsi->uplink_seid == pf->mac_seid)
13625                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
13626                                              vsi->tc_config.enabled_tc);
13627                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
13628                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13629                                              vsi->tc_config.enabled_tc);
13630                 if (veb) {
13631                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
13632                                 dev_info(&vsi->back->pdev->dev,
13633                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
13634                                 return NULL;
13635                         }
13636                         /* We come up by default in VEPA mode if SRIOV is not
13637                          * already enabled, in which case we can't force VEPA
13638                          * mode.
13639                          */
13640                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
13641                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
13642                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
13643                         }
13644                         i40e_config_bridge_mode(veb);
13645                 }
13646                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13647                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13648                                 veb = pf->veb[i];
13649                 }
13650                 if (!veb) {
13651                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
13652                         return NULL;
13653                 }
13654
13655                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
13656                 uplink_seid = veb->seid;
13657         }
13658
13659         /* get vsi sw struct */
13660         v_idx = i40e_vsi_mem_alloc(pf, type);
13661         if (v_idx < 0)
13662                 goto err_alloc;
13663         vsi = pf->vsi[v_idx];
13664         if (!vsi)
13665                 goto err_alloc;
13666         vsi->type = type;
13667         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
13668
13669         if (type == I40E_VSI_MAIN)
13670                 pf->lan_vsi = v_idx;
13671         else if (type == I40E_VSI_SRIOV)
13672                 vsi->vf_id = param1;
13673         /* assign it some queues */
13674         alloc_queue_pairs = vsi->alloc_queue_pairs *
13675                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13676
13677         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13678         if (ret < 0) {
13679                 dev_info(&pf->pdev->dev,
13680                          "failed to get tracking for %d queues for VSI %d err=%d\n",
13681                          alloc_queue_pairs, vsi->seid, ret);
13682                 goto err_vsi;
13683         }
13684         vsi->base_queue = ret;
13685
13686         /* get a VSI from the hardware */
13687         vsi->uplink_seid = uplink_seid;
13688         ret = i40e_add_vsi(vsi);
13689         if (ret)
13690                 goto err_vsi;
13691
13692         switch (vsi->type) {
13693         /* setup the netdev if needed */
13694         case I40E_VSI_MAIN:
13695         case I40E_VSI_VMDQ2:
13696                 ret = i40e_config_netdev(vsi);
13697                 if (ret)
13698                         goto err_netdev;
13699                 ret = register_netdev(vsi->netdev);
13700                 if (ret)
13701                         goto err_netdev;
13702                 vsi->netdev_registered = true;
13703                 netif_carrier_off(vsi->netdev);
13704 #ifdef CONFIG_I40E_DCB
13705                 /* Setup DCB netlink interface */
13706                 i40e_dcbnl_setup(vsi);
13707 #endif /* CONFIG_I40E_DCB */
13708                 /* fall through */
13709
13710         case I40E_VSI_FDIR:
13711                 /* set up vectors and rings if needed */
13712                 ret = i40e_vsi_setup_vectors(vsi);
13713                 if (ret)
13714                         goto err_msix;
13715
13716                 ret = i40e_alloc_rings(vsi);
13717                 if (ret)
13718                         goto err_rings;
13719
13720                 /* map all of the rings to the q_vectors */
13721                 i40e_vsi_map_rings_to_vectors(vsi);
13722
13723                 i40e_vsi_reset_stats(vsi);
13724                 break;
13725
13726         default:
13727                 /* no netdev or rings for the other VSI types */
13728                 break;
13729         }
13730
13731         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
13732             (vsi->type == I40E_VSI_VMDQ2)) {
13733                 ret = i40e_vsi_config_rss(vsi);
13734         }
13735         return vsi;
13736
13737 err_rings:
13738         i40e_vsi_free_q_vectors(vsi);
13739 err_msix:
13740         if (vsi->netdev_registered) {
13741                 vsi->netdev_registered = false;
13742                 unregister_netdev(vsi->netdev);
13743                 free_netdev(vsi->netdev);
13744                 vsi->netdev = NULL;
13745         }
13746 err_netdev:
13747         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13748 err_vsi:
13749         i40e_vsi_clear(vsi);
13750 err_alloc:
13751         return NULL;
13752 }
13753
13754 /**
13755  * i40e_veb_get_bw_info - Query VEB BW information
13756  * @veb: the veb to query
13757  *
13758  * Query the Tx scheduler BW configuration data for given VEB
13759  **/
13760 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
13761 {
13762         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
13763         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
13764         struct i40e_pf *pf = veb->pf;
13765         struct i40e_hw *hw = &pf->hw;
13766         u32 tc_bw_max;
13767         int ret = 0;
13768         int i;
13769
13770         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
13771                                                   &bw_data, NULL);
13772         if (ret) {
13773                 dev_info(&pf->pdev->dev,
13774                          "query veb bw config failed, err %s aq_err %s\n",
13775                          i40e_stat_str(&pf->hw, ret),
13776                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13777                 goto out;
13778         }
13779
13780         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
13781                                                    &ets_data, NULL);
13782         if (ret) {
13783                 dev_info(&pf->pdev->dev,
13784                          "query veb bw ets config failed, err %s aq_err %s\n",
13785                          i40e_stat_str(&pf->hw, ret),
13786                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13787                 goto out;
13788         }
13789
13790         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
13791         veb->bw_max_quanta = ets_data.tc_bw_max;
13792         veb->is_abs_credits = bw_data.absolute_credits_enable;
13793         veb->enabled_tc = ets_data.tc_valid_bits;
13794         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
13795                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
13796         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
13797                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
13798                 veb->bw_tc_limit_credits[i] =
13799                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
13800                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
13801         }
13802
13803 out:
13804         return ret;
13805 }
13806
13807 /**
13808  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
13809  * @pf: board private structure
13810  *
13811  * On error: returns error code (negative)
13812  * On success: returns vsi index in PF (positive)
13813  **/
13814 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
13815 {
13816         int ret = -ENOENT;
13817         struct i40e_veb *veb;
13818         int i;
13819
13820         /* Need to protect the allocation of switch elements at the PF level */
13821         mutex_lock(&pf->switch_mutex);
13822
13823         /* VEB list may be fragmented if VEB creation/destruction has
13824          * been happening.  We can afford to do a quick scan to look
13825          * for any free slots in the list.
13826          *
13827          * find next empty veb slot, looping back around if necessary
13828          */
13829         i = 0;
13830         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
13831                 i++;
13832         if (i >= I40E_MAX_VEB) {
13833                 ret = -ENOMEM;
13834                 goto err_alloc_veb;  /* out of VEB slots! */
13835         }
13836
13837         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
13838         if (!veb) {
13839                 ret = -ENOMEM;
13840                 goto err_alloc_veb;
13841         }
13842         veb->pf = pf;
13843         veb->idx = i;
13844         veb->enabled_tc = 1;
13845
13846         pf->veb[i] = veb;
13847         ret = i;
13848 err_alloc_veb:
13849         mutex_unlock(&pf->switch_mutex);
13850         return ret;
13851 }
13852
13853 /**
13854  * i40e_switch_branch_release - Delete a branch of the switch tree
13855  * @branch: where to start deleting
13856  *
13857  * This uses recursion to find the tips of the branch to be
13858  * removed, deleting until we get back to and can delete this VEB.
13859  **/
13860 static void i40e_switch_branch_release(struct i40e_veb *branch)
13861 {
13862         struct i40e_pf *pf = branch->pf;
13863         u16 branch_seid = branch->seid;
13864         u16 veb_idx = branch->idx;
13865         int i;
13866
13867         /* release any VEBs on this VEB - RECURSION */
13868         for (i = 0; i < I40E_MAX_VEB; i++) {
13869                 if (!pf->veb[i])
13870                         continue;
13871                 if (pf->veb[i]->uplink_seid == branch->seid)
13872                         i40e_switch_branch_release(pf->veb[i]);
13873         }
13874
13875         /* Release the VSIs on this VEB, but not the owner VSI.
13876          *
13877          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
13878          *       the VEB itself, so don't use (*branch) after this loop.
13879          */
13880         for (i = 0; i < pf->num_alloc_vsi; i++) {
13881                 if (!pf->vsi[i])
13882                         continue;
13883                 if (pf->vsi[i]->uplink_seid == branch_seid &&
13884                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13885                         i40e_vsi_release(pf->vsi[i]);
13886                 }
13887         }
13888
13889         /* There's one corner case where the VEB might not have been
13890          * removed, so double check it here and remove it if needed.
13891          * This case happens if the veb was created from the debugfs
13892          * commands and no VSIs were added to it.
13893          */
13894         if (pf->veb[veb_idx])
13895                 i40e_veb_release(pf->veb[veb_idx]);
13896 }
13897
13898 /**
13899  * i40e_veb_clear - remove veb struct
13900  * @veb: the veb to remove
13901  **/
13902 static void i40e_veb_clear(struct i40e_veb *veb)
13903 {
13904         if (!veb)
13905                 return;
13906
13907         if (veb->pf) {
13908                 struct i40e_pf *pf = veb->pf;
13909
13910                 mutex_lock(&pf->switch_mutex);
13911                 if (pf->veb[veb->idx] == veb)
13912                         pf->veb[veb->idx] = NULL;
13913                 mutex_unlock(&pf->switch_mutex);
13914         }
13915
13916         kfree(veb);
13917 }
13918
13919 /**
13920  * i40e_veb_release - Delete a VEB and free its resources
13921  * @veb: the VEB being removed
13922  **/
13923 void i40e_veb_release(struct i40e_veb *veb)
13924 {
13925         struct i40e_vsi *vsi = NULL;
13926         struct i40e_pf *pf;
13927         int i, n = 0;
13928
13929         pf = veb->pf;
13930
13931         /* find the remaining VSI and check for extras */
13932         for (i = 0; i < pf->num_alloc_vsi; i++) {
13933                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
13934                         n++;
13935                         vsi = pf->vsi[i];
13936                 }
13937         }
13938         if (n != 1) {
13939                 dev_info(&pf->pdev->dev,
13940                          "can't remove VEB %d with %d VSIs left\n",
13941                          veb->seid, n);
13942                 return;
13943         }
13944
13945         /* move the remaining VSI to uplink veb */
13946         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
13947         if (veb->uplink_seid) {
13948                 vsi->uplink_seid = veb->uplink_seid;
13949                 if (veb->uplink_seid == pf->mac_seid)
13950                         vsi->veb_idx = I40E_NO_VEB;
13951                 else
13952                         vsi->veb_idx = veb->veb_idx;
13953         } else {
13954                 /* floating VEB */
13955                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
13956                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
13957         }
13958
13959         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
13960         i40e_veb_clear(veb);
13961 }
13962
13963 /**
13964  * i40e_add_veb - create the VEB in the switch
13965  * @veb: the VEB to be instantiated
13966  * @vsi: the controlling VSI
13967  **/
13968 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
13969 {
13970         struct i40e_pf *pf = veb->pf;
13971         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
13972         int ret;
13973
13974         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
13975                               veb->enabled_tc, false,
13976                               &veb->seid, enable_stats, NULL);
13977
13978         /* get a VEB from the hardware */
13979         if (ret) {
13980                 dev_info(&pf->pdev->dev,
13981                          "couldn't add VEB, err %s aq_err %s\n",
13982                          i40e_stat_str(&pf->hw, ret),
13983                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13984                 return -EPERM;
13985         }
13986
13987         /* get statistics counter */
13988         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
13989                                          &veb->stats_idx, NULL, NULL, NULL);
13990         if (ret) {
13991                 dev_info(&pf->pdev->dev,
13992                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
13993                          i40e_stat_str(&pf->hw, ret),
13994                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13995                 return -EPERM;
13996         }
13997         ret = i40e_veb_get_bw_info(veb);
13998         if (ret) {
13999                 dev_info(&pf->pdev->dev,
14000                          "couldn't get VEB bw info, err %s aq_err %s\n",
14001                          i40e_stat_str(&pf->hw, ret),
14002                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14003                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14004                 return -ENOENT;
14005         }
14006
14007         vsi->uplink_seid = veb->seid;
14008         vsi->veb_idx = veb->idx;
14009         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14010
14011         return 0;
14012 }
14013
14014 /**
14015  * i40e_veb_setup - Set up a VEB
14016  * @pf: board private structure
14017  * @flags: VEB setup flags
14018  * @uplink_seid: the switch element to link to
14019  * @vsi_seid: the initial VSI seid
14020  * @enabled_tc: Enabled TC bit-map
14021  *
14022  * This allocates the sw VEB structure and links it into the switch
14023  * It is possible and legal for this to be a duplicate of an already
14024  * existing VEB.  It is also possible for both uplink and vsi seids
14025  * to be zero, in order to create a floating VEB.
14026  *
14027  * Returns pointer to the successfully allocated VEB sw struct on
14028  * success, otherwise returns NULL on failure.
14029  **/
14030 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14031                                 u16 uplink_seid, u16 vsi_seid,
14032                                 u8 enabled_tc)
14033 {
14034         struct i40e_veb *veb, *uplink_veb = NULL;
14035         int vsi_idx, veb_idx;
14036         int ret;
14037
14038         /* if one seid is 0, the other must be 0 to create a floating relay */
14039         if ((uplink_seid == 0 || vsi_seid == 0) &&
14040             (uplink_seid + vsi_seid != 0)) {
14041                 dev_info(&pf->pdev->dev,
14042                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14043                          uplink_seid, vsi_seid);
14044                 return NULL;
14045         }
14046
14047         /* make sure there is such a vsi and uplink */
14048         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14049                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14050                         break;
14051         if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14052                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14053                          vsi_seid);
14054                 return NULL;
14055         }
14056
14057         if (uplink_seid && uplink_seid != pf->mac_seid) {
14058                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14059                         if (pf->veb[veb_idx] &&
14060                             pf->veb[veb_idx]->seid == uplink_seid) {
14061                                 uplink_veb = pf->veb[veb_idx];
14062                                 break;
14063                         }
14064                 }
14065                 if (!uplink_veb) {
14066                         dev_info(&pf->pdev->dev,
14067                                  "uplink seid %d not found\n", uplink_seid);
14068                         return NULL;
14069                 }
14070         }
14071
14072         /* get veb sw struct */
14073         veb_idx = i40e_veb_mem_alloc(pf);
14074         if (veb_idx < 0)
14075                 goto err_alloc;
14076         veb = pf->veb[veb_idx];
14077         veb->flags = flags;
14078         veb->uplink_seid = uplink_seid;
14079         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14080         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14081
14082         /* create the VEB in the switch */
14083         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14084         if (ret)
14085                 goto err_veb;
14086         if (vsi_idx == pf->lan_vsi)
14087                 pf->lan_veb = veb->idx;
14088
14089         return veb;
14090
14091 err_veb:
14092         i40e_veb_clear(veb);
14093 err_alloc:
14094         return NULL;
14095 }
14096
14097 /**
14098  * i40e_setup_pf_switch_element - set PF vars based on switch type
14099  * @pf: board private structure
14100  * @ele: element we are building info from
14101  * @num_reported: total number of elements
14102  * @printconfig: should we print the contents
14103  *
14104  * helper function to assist in extracting a few useful SEID values.
14105  **/
14106 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14107                                 struct i40e_aqc_switch_config_element_resp *ele,
14108                                 u16 num_reported, bool printconfig)
14109 {
14110         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14111         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14112         u8 element_type = ele->element_type;
14113         u16 seid = le16_to_cpu(ele->seid);
14114
14115         if (printconfig)
14116                 dev_info(&pf->pdev->dev,
14117                          "type=%d seid=%d uplink=%d downlink=%d\n",
14118                          element_type, seid, uplink_seid, downlink_seid);
14119
14120         switch (element_type) {
14121         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14122                 pf->mac_seid = seid;
14123                 break;
14124         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14125                 /* Main VEB? */
14126                 if (uplink_seid != pf->mac_seid)
14127                         break;
14128                 if (pf->lan_veb >= I40E_MAX_VEB) {
14129                         int v;
14130
14131                         /* find existing or else empty VEB */
14132                         for (v = 0; v < I40E_MAX_VEB; v++) {
14133                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14134                                         pf->lan_veb = v;
14135                                         break;
14136                                 }
14137                         }
14138                         if (pf->lan_veb >= I40E_MAX_VEB) {
14139                                 v = i40e_veb_mem_alloc(pf);
14140                                 if (v < 0)
14141                                         break;
14142                                 pf->lan_veb = v;
14143                         }
14144                 }
14145                 if (pf->lan_veb >= I40E_MAX_VEB)
14146                         break;
14147
14148                 pf->veb[pf->lan_veb]->seid = seid;
14149                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14150                 pf->veb[pf->lan_veb]->pf = pf;
14151                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14152                 break;
14153         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14154                 if (num_reported != 1)
14155                         break;
14156                 /* This is immediately after a reset so we can assume this is
14157                  * the PF's VSI
14158                  */
14159                 pf->mac_seid = uplink_seid;
14160                 pf->pf_seid = downlink_seid;
14161                 pf->main_vsi_seid = seid;
14162                 if (printconfig)
14163                         dev_info(&pf->pdev->dev,
14164                                  "pf_seid=%d main_vsi_seid=%d\n",
14165                                  pf->pf_seid, pf->main_vsi_seid);
14166                 break;
14167         case I40E_SWITCH_ELEMENT_TYPE_PF:
14168         case I40E_SWITCH_ELEMENT_TYPE_VF:
14169         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14170         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14171         case I40E_SWITCH_ELEMENT_TYPE_PE:
14172         case I40E_SWITCH_ELEMENT_TYPE_PA:
14173                 /* ignore these for now */
14174                 break;
14175         default:
14176                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14177                          element_type, seid);
14178                 break;
14179         }
14180 }
14181
14182 /**
14183  * i40e_fetch_switch_configuration - Get switch config from firmware
14184  * @pf: board private structure
14185  * @printconfig: should we print the contents
14186  *
14187  * Get the current switch configuration from the device and
14188  * extract a few useful SEID values.
14189  **/
14190 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14191 {
14192         struct i40e_aqc_get_switch_config_resp *sw_config;
14193         u16 next_seid = 0;
14194         int ret = 0;
14195         u8 *aq_buf;
14196         int i;
14197
14198         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14199         if (!aq_buf)
14200                 return -ENOMEM;
14201
14202         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14203         do {
14204                 u16 num_reported, num_total;
14205
14206                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14207                                                 I40E_AQ_LARGE_BUF,
14208                                                 &next_seid, NULL);
14209                 if (ret) {
14210                         dev_info(&pf->pdev->dev,
14211                                  "get switch config failed err %s aq_err %s\n",
14212                                  i40e_stat_str(&pf->hw, ret),
14213                                  i40e_aq_str(&pf->hw,
14214                                              pf->hw.aq.asq_last_status));
14215                         kfree(aq_buf);
14216                         return -ENOENT;
14217                 }
14218
14219                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14220                 num_total = le16_to_cpu(sw_config->header.num_total);
14221
14222                 if (printconfig)
14223                         dev_info(&pf->pdev->dev,
14224                                  "header: %d reported %d total\n",
14225                                  num_reported, num_total);
14226
14227                 for (i = 0; i < num_reported; i++) {
14228                         struct i40e_aqc_switch_config_element_resp *ele =
14229                                 &sw_config->element[i];
14230
14231                         i40e_setup_pf_switch_element(pf, ele, num_reported,
14232                                                      printconfig);
14233                 }
14234         } while (next_seid != 0);
14235
14236         kfree(aq_buf);
14237         return ret;
14238 }
14239
14240 /**
14241  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14242  * @pf: board private structure
14243  * @reinit: if the Main VSI needs to re-initialized.
14244  *
14245  * Returns 0 on success, negative value on failure
14246  **/
14247 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
14248 {
14249         u16 flags = 0;
14250         int ret;
14251
14252         /* find out what's out there already */
14253         ret = i40e_fetch_switch_configuration(pf, false);
14254         if (ret) {
14255                 dev_info(&pf->pdev->dev,
14256                          "couldn't fetch switch config, err %s aq_err %s\n",
14257                          i40e_stat_str(&pf->hw, ret),
14258                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14259                 return ret;
14260         }
14261         i40e_pf_reset_stats(pf);
14262
14263         /* set the switch config bit for the whole device to
14264          * support limited promisc or true promisc
14265          * when user requests promisc. The default is limited
14266          * promisc.
14267         */
14268
14269         if ((pf->hw.pf_id == 0) &&
14270             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14271                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14272                 pf->last_sw_conf_flags = flags;
14273         }
14274
14275         if (pf->hw.pf_id == 0) {
14276                 u16 valid_flags;
14277
14278                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14279                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14280                                                 NULL);
14281                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14282                         dev_info(&pf->pdev->dev,
14283                                  "couldn't set switch config bits, err %s aq_err %s\n",
14284                                  i40e_stat_str(&pf->hw, ret),
14285                                  i40e_aq_str(&pf->hw,
14286                                              pf->hw.aq.asq_last_status));
14287                         /* not a fatal problem, just keep going */
14288                 }
14289                 pf->last_sw_conf_valid_flags = valid_flags;
14290         }
14291
14292         /* first time setup */
14293         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14294                 struct i40e_vsi *vsi = NULL;
14295                 u16 uplink_seid;
14296
14297                 /* Set up the PF VSI associated with the PF's main VSI
14298                  * that is already in the HW switch
14299                  */
14300                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14301                         uplink_seid = pf->veb[pf->lan_veb]->seid;
14302                 else
14303                         uplink_seid = pf->mac_seid;
14304                 if (pf->lan_vsi == I40E_NO_VSI)
14305                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14306                 else if (reinit)
14307                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14308                 if (!vsi) {
14309                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14310                         i40e_cloud_filter_exit(pf);
14311                         i40e_fdir_teardown(pf);
14312                         return -EAGAIN;
14313                 }
14314         } else {
14315                 /* force a reset of TC and queue layout configurations */
14316                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14317
14318                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14319                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14320                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14321         }
14322         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14323
14324         i40e_fdir_sb_setup(pf);
14325
14326         /* Setup static PF queue filter control settings */
14327         ret = i40e_setup_pf_filter_control(pf);
14328         if (ret) {
14329                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14330                          ret);
14331                 /* Failure here should not stop continuing other steps */
14332         }
14333
14334         /* enable RSS in the HW, even for only one queue, as the stack can use
14335          * the hash
14336          */
14337         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14338                 i40e_pf_config_rss(pf);
14339
14340         /* fill in link information and enable LSE reporting */
14341         i40e_link_event(pf);
14342
14343         /* Initialize user-specific link properties */
14344         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14345                                   I40E_AQ_AN_COMPLETED) ? true : false);
14346
14347         i40e_ptp_init(pf);
14348
14349         /* repopulate tunnel port filters */
14350         i40e_sync_udp_filters(pf);
14351
14352         return ret;
14353 }
14354
14355 /**
14356  * i40e_determine_queue_usage - Work out queue distribution
14357  * @pf: board private structure
14358  **/
14359 static void i40e_determine_queue_usage(struct i40e_pf *pf)
14360 {
14361         int queues_left;
14362         int q_max;
14363
14364         pf->num_lan_qps = 0;
14365
14366         /* Find the max queues to be put into basic use.  We'll always be
14367          * using TC0, whether or not DCB is running, and TC0 will get the
14368          * big RSS set.
14369          */
14370         queues_left = pf->hw.func_caps.num_tx_qp;
14371
14372         if ((queues_left == 1) ||
14373             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14374                 /* one qp for PF, no queues for anything else */
14375                 queues_left = 0;
14376                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14377
14378                 /* make sure all the fancies are disabled */
14379                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14380                                I40E_FLAG_IWARP_ENABLED  |
14381                                I40E_FLAG_FD_SB_ENABLED  |
14382                                I40E_FLAG_FD_ATR_ENABLED |
14383                                I40E_FLAG_DCB_CAPABLE    |
14384                                I40E_FLAG_DCB_ENABLED    |
14385                                I40E_FLAG_SRIOV_ENABLED  |
14386                                I40E_FLAG_VMDQ_ENABLED);
14387                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14388         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14389                                   I40E_FLAG_FD_SB_ENABLED |
14390                                   I40E_FLAG_FD_ATR_ENABLED |
14391                                   I40E_FLAG_DCB_CAPABLE))) {
14392                 /* one qp for PF */
14393                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14394                 queues_left -= pf->num_lan_qps;
14395
14396                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14397                                I40E_FLAG_IWARP_ENABLED  |
14398                                I40E_FLAG_FD_SB_ENABLED  |
14399                                I40E_FLAG_FD_ATR_ENABLED |
14400                                I40E_FLAG_DCB_ENABLED    |
14401                                I40E_FLAG_VMDQ_ENABLED);
14402                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14403         } else {
14404                 /* Not enough queues for all TCs */
14405                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14406                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14407                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14408                                         I40E_FLAG_DCB_ENABLED);
14409                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14410                 }
14411
14412                 /* limit lan qps to the smaller of qps, cpus or msix */
14413                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14414                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14415                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14416                 pf->num_lan_qps = q_max;
14417
14418                 queues_left -= pf->num_lan_qps;
14419         }
14420
14421         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14422                 if (queues_left > 1) {
14423                         queues_left -= 1; /* save 1 queue for FD */
14424                 } else {
14425                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14426                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14427                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14428                 }
14429         }
14430
14431         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14432             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14433                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14434                                         (queues_left / pf->num_vf_qps));
14435                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14436         }
14437
14438         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14439             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14440                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14441                                           (queues_left / pf->num_vmdq_qps));
14442                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14443         }
14444
14445         pf->queues_left = queues_left;
14446         dev_dbg(&pf->pdev->dev,
14447                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14448                 pf->hw.func_caps.num_tx_qp,
14449                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14450                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14451                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14452                 queues_left);
14453 }
14454
14455 /**
14456  * i40e_setup_pf_filter_control - Setup PF static filter control
14457  * @pf: PF to be setup
14458  *
14459  * i40e_setup_pf_filter_control sets up a PF's initial filter control
14460  * settings. If PE/FCoE are enabled then it will also set the per PF
14461  * based filter sizes required for them. It also enables Flow director,
14462  * ethertype and macvlan type filter settings for the pf.
14463  *
14464  * Returns 0 on success, negative on failure
14465  **/
14466 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14467 {
14468         struct i40e_filter_control_settings *settings = &pf->filter_settings;
14469
14470         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14471
14472         /* Flow Director is enabled */
14473         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14474                 settings->enable_fdir = true;
14475
14476         /* Ethtype and MACVLAN filters enabled for PF */
14477         settings->enable_ethtype = true;
14478         settings->enable_macvlan = true;
14479
14480         if (i40e_set_filter_control(&pf->hw, settings))
14481                 return -ENOENT;
14482
14483         return 0;
14484 }
14485
14486 #define INFO_STRING_LEN 255
14487 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
14488 static void i40e_print_features(struct i40e_pf *pf)
14489 {
14490         struct i40e_hw *hw = &pf->hw;
14491         char *buf;
14492         int i;
14493
14494         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14495         if (!buf)
14496                 return;
14497
14498         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
14499 #ifdef CONFIG_PCI_IOV
14500         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
14501 #endif
14502         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
14503                       pf->hw.func_caps.num_vsis,
14504                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
14505         if (pf->flags & I40E_FLAG_RSS_ENABLED)
14506                 i += snprintf(&buf[i], REMAIN(i), " RSS");
14507         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
14508                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
14509         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14510                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
14511                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
14512         }
14513         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
14514                 i += snprintf(&buf[i], REMAIN(i), " DCB");
14515         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
14516         i += snprintf(&buf[i], REMAIN(i), " Geneve");
14517         if (pf->flags & I40E_FLAG_PTP)
14518                 i += snprintf(&buf[i], REMAIN(i), " PTP");
14519         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
14520                 i += snprintf(&buf[i], REMAIN(i), " VEB");
14521         else
14522                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
14523
14524         dev_info(&pf->pdev->dev, "%s\n", buf);
14525         kfree(buf);
14526         WARN_ON(i > INFO_STRING_LEN);
14527 }
14528
14529 /**
14530  * i40e_get_platform_mac_addr - get platform-specific MAC address
14531  * @pdev: PCI device information struct
14532  * @pf: board private structure
14533  *
14534  * Look up the MAC address for the device. First we'll try
14535  * eth_platform_get_mac_address, which will check Open Firmware, or arch
14536  * specific fallback. Otherwise, we'll default to the stored value in
14537  * firmware.
14538  **/
14539 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
14540 {
14541         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
14542                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
14543 }
14544
14545 /**
14546  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
14547  * @fec_cfg: FEC option to set in flags
14548  * @flags: ptr to flags in which we set FEC option
14549  **/
14550 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
14551 {
14552         if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
14553                 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
14554         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
14555             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
14556                 *flags |= I40E_FLAG_RS_FEC;
14557                 *flags &= ~I40E_FLAG_BASE_R_FEC;
14558         }
14559         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
14560             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
14561                 *flags |= I40E_FLAG_BASE_R_FEC;
14562                 *flags &= ~I40E_FLAG_RS_FEC;
14563         }
14564         if (fec_cfg == 0)
14565                 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
14566 }
14567
14568 /**
14569  * i40e_check_recovery_mode - check if we are running transition firmware
14570  * @pf: board private structure
14571  *
14572  * Check registers indicating the firmware runs in recovery mode. Sets the
14573  * appropriate driver state.
14574  *
14575  * Returns true if the recovery mode was detected, false otherwise
14576  **/
14577 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
14578 {
14579         u32 val = rd32(&pf->hw, I40E_GL_FWSTS) & I40E_GL_FWSTS_FWS1B_MASK;
14580         bool is_recovery_mode = false;
14581
14582         if (pf->hw.mac.type == I40E_MAC_XL710)
14583                 is_recovery_mode =
14584                 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_CORER_MASK ||
14585                 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_GLOBR_MASK ||
14586                 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_TRANSITION_MASK ||
14587                 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_NVM_MASK;
14588         if (pf->hw.mac.type == I40E_MAC_X722)
14589                 is_recovery_mode =
14590                 val == I40E_X722_GL_FWSTS_FWS1B_REC_MOD_CORER_MASK ||
14591                 val == I40E_X722_GL_FWSTS_FWS1B_REC_MOD_GLOBR_MASK;
14592         if (is_recovery_mode) {
14593                 dev_notice(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
14594                 dev_notice(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
14595                 set_bit(__I40E_RECOVERY_MODE, pf->state);
14596
14597                 return true;
14598         }
14599         if (test_and_clear_bit(__I40E_RECOVERY_MODE, pf->state))
14600                 dev_info(&pf->pdev->dev, "Reinitializing in normal mode with full functionality.\n");
14601
14602         return false;
14603 }
14604
14605 /**
14606  * i40e_pf_loop_reset - perform reset in a loop.
14607  * @pf: board private structure
14608  *
14609  * This function is useful when a NIC is about to enter recovery mode.
14610  * When a NIC's internal data structures are corrupted the NIC's
14611  * firmware is going to enter recovery mode.
14612  * Right after a POR it takes about 7 minutes for firmware to enter
14613  * recovery mode. Until that time a NIC is in some kind of intermediate
14614  * state. After that time period the NIC almost surely enters
14615  * recovery mode. The only way for a driver to detect intermediate
14616  * state is to issue a series of pf-resets and check a return value.
14617  * If a PF reset returns success then the firmware could be in recovery
14618  * mode so the caller of this code needs to check for recovery mode
14619  * if this function returns success. There is a little chance that
14620  * firmware will hang in intermediate state forever.
14621  * Since waiting 7 minutes is quite a lot of time this function waits
14622  * 10 seconds and then gives up by returning an error.
14623  *
14624  * Return 0 on success, negative on failure.
14625  **/
14626 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf)
14627 {
14628         const unsigned short MAX_CNT = 1000;
14629         const unsigned short MSECS = 10;
14630         struct i40e_hw *hw = &pf->hw;
14631         i40e_status ret;
14632         int cnt;
14633
14634         for (cnt = 0; cnt < MAX_CNT; ++cnt) {
14635                 ret = i40e_pf_reset(hw);
14636                 if (!ret)
14637                         break;
14638                 msleep(MSECS);
14639         }
14640
14641         if (cnt == MAX_CNT) {
14642                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
14643                 return ret;
14644         }
14645
14646         pf->pfr_count++;
14647         return ret;
14648 }
14649
14650 /**
14651  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
14652  * @pf: board private structure
14653  * @hw: ptr to the hardware info
14654  *
14655  * This function does a minimal setup of all subsystems needed for running
14656  * recovery mode.
14657  *
14658  * Returns 0 on success, negative on failure
14659  **/
14660 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
14661 {
14662         struct i40e_vsi *vsi;
14663         int err;
14664         int v_idx;
14665
14666         pci_save_state(pf->pdev);
14667
14668         /* set up periodic task facility */
14669         timer_setup(&pf->service_timer, i40e_service_timer, 0);
14670         pf->service_timer_period = HZ;
14671
14672         INIT_WORK(&pf->service_task, i40e_service_task);
14673         clear_bit(__I40E_SERVICE_SCHED, pf->state);
14674
14675         err = i40e_init_interrupt_scheme(pf);
14676         if (err)
14677                 goto err_switch_setup;
14678
14679         /* The number of VSIs reported by the FW is the minimum guaranteed
14680          * to us; HW supports far more and we share the remaining pool with
14681          * the other PFs. We allocate space for more than the guarantee with
14682          * the understanding that we might not get them all later.
14683          */
14684         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
14685                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
14686         else
14687                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
14688
14689         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
14690         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
14691                           GFP_KERNEL);
14692         if (!pf->vsi) {
14693                 err = -ENOMEM;
14694                 goto err_switch_setup;
14695         }
14696
14697         /* We allocate one VSI which is needed as absolute minimum
14698          * in order to register the netdev
14699          */
14700         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
14701         if (v_idx < 0)
14702                 goto err_switch_setup;
14703         pf->lan_vsi = v_idx;
14704         vsi = pf->vsi[v_idx];
14705         if (!vsi)
14706                 goto err_switch_setup;
14707         vsi->alloc_queue_pairs = 1;
14708         err = i40e_config_netdev(vsi);
14709         if (err)
14710                 goto err_switch_setup;
14711         err = register_netdev(vsi->netdev);
14712         if (err)
14713                 goto err_switch_setup;
14714         vsi->netdev_registered = true;
14715         i40e_dbg_pf_init(pf);
14716
14717         err = i40e_setup_misc_vector_for_recovery_mode(pf);
14718         if (err)
14719                 goto err_switch_setup;
14720
14721         /* tell the firmware that we're starting */
14722         i40e_send_version(pf);
14723
14724         /* since everything's happy, start the service_task timer */
14725         mod_timer(&pf->service_timer,
14726                   round_jiffies(jiffies + pf->service_timer_period));
14727
14728         return 0;
14729
14730 err_switch_setup:
14731         i40e_reset_interrupt_capability(pf);
14732         del_timer_sync(&pf->service_timer);
14733         i40e_shutdown_adminq(hw);
14734         iounmap(hw->hw_addr);
14735         pci_disable_pcie_error_reporting(pf->pdev);
14736         pci_release_mem_regions(pf->pdev);
14737         pci_disable_device(pf->pdev);
14738         kfree(pf);
14739
14740         return err;
14741 }
14742
14743 /**
14744  * i40e_probe - Device initialization routine
14745  * @pdev: PCI device information struct
14746  * @ent: entry in i40e_pci_tbl
14747  *
14748  * i40e_probe initializes a PF identified by a pci_dev structure.
14749  * The OS initialization, configuring of the PF private structure,
14750  * and a hardware reset occur.
14751  *
14752  * Returns 0 on success, negative on failure
14753  **/
14754 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
14755 {
14756         struct i40e_aq_get_phy_abilities_resp abilities;
14757         struct i40e_pf *pf;
14758         struct i40e_hw *hw;
14759         static u16 pfs_found;
14760         u16 wol_nvm_bits;
14761         u16 link_status;
14762         int err;
14763         u32 val;
14764         u32 i;
14765         u8 set_fc_aq_fail;
14766
14767         err = pci_enable_device_mem(pdev);
14768         if (err)
14769                 return err;
14770
14771         /* set up for high or low dma */
14772         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
14773         if (err) {
14774                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
14775                 if (err) {
14776                         dev_err(&pdev->dev,
14777                                 "DMA configuration failed: 0x%x\n", err);
14778                         goto err_dma;
14779                 }
14780         }
14781
14782         /* set up pci connections */
14783         err = pci_request_mem_regions(pdev, i40e_driver_name);
14784         if (err) {
14785                 dev_info(&pdev->dev,
14786                          "pci_request_selected_regions failed %d\n", err);
14787                 goto err_pci_reg;
14788         }
14789
14790         pci_enable_pcie_error_reporting(pdev);
14791         pci_set_master(pdev);
14792
14793         /* Now that we have a PCI connection, we need to do the
14794          * low level device setup.  This is primarily setting up
14795          * the Admin Queue structures and then querying for the
14796          * device's current profile information.
14797          */
14798         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
14799         if (!pf) {
14800                 err = -ENOMEM;
14801                 goto err_pf_alloc;
14802         }
14803         pf->next_vsi = 0;
14804         pf->pdev = pdev;
14805         set_bit(__I40E_DOWN, pf->state);
14806
14807         hw = &pf->hw;
14808         hw->back = pf;
14809
14810         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
14811                                 I40E_MAX_CSR_SPACE);
14812         /* We believe that the highest register to read is
14813          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
14814          * is not less than that before mapping to prevent a
14815          * kernel panic.
14816          */
14817         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
14818                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
14819                         pf->ioremap_len);
14820                 err = -ENOMEM;
14821                 goto err_ioremap;
14822         }
14823         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
14824         if (!hw->hw_addr) {
14825                 err = -EIO;
14826                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
14827                          (unsigned int)pci_resource_start(pdev, 0),
14828                          pf->ioremap_len, err);
14829                 goto err_ioremap;
14830         }
14831         hw->vendor_id = pdev->vendor;
14832         hw->device_id = pdev->device;
14833         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
14834         hw->subsystem_vendor_id = pdev->subsystem_vendor;
14835         hw->subsystem_device_id = pdev->subsystem_device;
14836         hw->bus.device = PCI_SLOT(pdev->devfn);
14837         hw->bus.func = PCI_FUNC(pdev->devfn);
14838         hw->bus.bus_id = pdev->bus->number;
14839         pf->instance = pfs_found;
14840
14841         /* Select something other than the 802.1ad ethertype for the
14842          * switch to use internally and drop on ingress.
14843          */
14844         hw->switch_tag = 0xffff;
14845         hw->first_tag = ETH_P_8021AD;
14846         hw->second_tag = ETH_P_8021Q;
14847
14848         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
14849         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
14850         INIT_LIST_HEAD(&pf->ddp_old_prof);
14851
14852         /* set up the locks for the AQ, do this only once in probe
14853          * and destroy them only once in remove
14854          */
14855         mutex_init(&hw->aq.asq_mutex);
14856         mutex_init(&hw->aq.arq_mutex);
14857
14858         pf->msg_enable = netif_msg_init(debug,
14859                                         NETIF_MSG_DRV |
14860                                         NETIF_MSG_PROBE |
14861                                         NETIF_MSG_LINK);
14862         if (debug < -1)
14863                 pf->hw.debug_mask = debug;
14864
14865         /* do a special CORER for clearing PXE mode once at init */
14866         if (hw->revision_id == 0 &&
14867             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
14868                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
14869                 i40e_flush(hw);
14870                 msleep(200);
14871                 pf->corer_count++;
14872
14873                 i40e_clear_pxe_mode(hw);
14874         }
14875
14876         /* Reset here to make sure all is clean and to define PF 'n' */
14877         i40e_clear_hw(hw);
14878
14879         err = i40e_set_mac_type(hw);
14880         if (err) {
14881                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
14882                          err);
14883                 goto err_pf_reset;
14884         }
14885
14886         err = i40e_pf_loop_reset(pf);
14887         if (err) {
14888                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
14889                 goto err_pf_reset;
14890         }
14891
14892         i40e_check_recovery_mode(pf);
14893
14894         hw->aq.num_arq_entries = I40E_AQ_LEN;
14895         hw->aq.num_asq_entries = I40E_AQ_LEN;
14896         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
14897         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
14898         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
14899
14900         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
14901                  "%s-%s:misc",
14902                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
14903
14904         err = i40e_init_shared_code(hw);
14905         if (err) {
14906                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
14907                          err);
14908                 goto err_pf_reset;
14909         }
14910
14911         /* set up a default setting for link flow control */
14912         pf->hw.fc.requested_mode = I40E_FC_NONE;
14913
14914         err = i40e_init_adminq(hw);
14915         if (err) {
14916                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
14917                         dev_info(&pdev->dev,
14918                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
14919                                  hw->aq.api_maj_ver,
14920                                  hw->aq.api_min_ver,
14921                                  I40E_FW_API_VERSION_MAJOR,
14922                                  I40E_FW_MINOR_VERSION(hw));
14923                 else
14924                         dev_info(&pdev->dev,
14925                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
14926
14927                 goto err_pf_reset;
14928         }
14929         i40e_get_oem_version(hw);
14930
14931         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
14932         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
14933                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
14934                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
14935                  i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
14936                  hw->subsystem_vendor_id, hw->subsystem_device_id);
14937
14938         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
14939             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
14940                 dev_info(&pdev->dev,
14941                          "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n",
14942                          hw->aq.api_maj_ver,
14943                          hw->aq.api_min_ver,
14944                          I40E_FW_API_VERSION_MAJOR,
14945                          I40E_FW_MINOR_VERSION(hw));
14946         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
14947                 dev_info(&pdev->dev,
14948                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
14949                          hw->aq.api_maj_ver,
14950                          hw->aq.api_min_ver,
14951                          I40E_FW_API_VERSION_MAJOR,
14952                          I40E_FW_MINOR_VERSION(hw));
14953
14954         i40e_verify_eeprom(pf);
14955
14956         /* Rev 0 hardware was never productized */
14957         if (hw->revision_id < 1)
14958                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
14959
14960         i40e_clear_pxe_mode(hw);
14961
14962         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
14963         if (err)
14964                 goto err_adminq_setup;
14965
14966         err = i40e_sw_init(pf);
14967         if (err) {
14968                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
14969                 goto err_sw_init;
14970         }
14971
14972         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
14973                 return i40e_init_recovery_mode(pf, hw);
14974
14975         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
14976                                 hw->func_caps.num_rx_qp, 0, 0);
14977         if (err) {
14978                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
14979                 goto err_init_lan_hmc;
14980         }
14981
14982         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
14983         if (err) {
14984                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
14985                 err = -ENOENT;
14986                 goto err_configure_lan_hmc;
14987         }
14988
14989         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
14990          * Ignore error return codes because if it was already disabled via
14991          * hardware settings this will fail
14992          */
14993         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
14994                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
14995                 i40e_aq_stop_lldp(hw, true, false, NULL);
14996         }
14997
14998         /* allow a platform config to override the HW addr */
14999         i40e_get_platform_mac_addr(pdev, pf);
15000
15001         if (!is_valid_ether_addr(hw->mac.addr)) {
15002                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15003                 err = -EIO;
15004                 goto err_mac_addr;
15005         }
15006         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15007         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15008         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15009         if (is_valid_ether_addr(hw->mac.port_addr))
15010                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15011
15012         pci_set_drvdata(pdev, pf);
15013         pci_save_state(pdev);
15014
15015         dev_info(&pdev->dev,
15016                  (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15017                         "FW LLDP is disabled\n" :
15018                         "FW LLDP is enabled\n");
15019
15020         /* Enable FW to write default DCB config on link-up */
15021         i40e_aq_set_dcb_parameters(hw, true, NULL);
15022
15023 #ifdef CONFIG_I40E_DCB
15024         err = i40e_init_pf_dcb(pf);
15025         if (err) {
15026                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15027                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15028                 /* Continue without DCB enabled */
15029         }
15030 #endif /* CONFIG_I40E_DCB */
15031
15032         /* set up periodic task facility */
15033         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15034         pf->service_timer_period = HZ;
15035
15036         INIT_WORK(&pf->service_task, i40e_service_task);
15037         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15038
15039         /* NVM bit on means WoL disabled for the port */
15040         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15041         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15042                 pf->wol_en = false;
15043         else
15044                 pf->wol_en = true;
15045         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15046
15047         /* set up the main switch operations */
15048         i40e_determine_queue_usage(pf);
15049         err = i40e_init_interrupt_scheme(pf);
15050         if (err)
15051                 goto err_switch_setup;
15052
15053         /* The number of VSIs reported by the FW is the minimum guaranteed
15054          * to us; HW supports far more and we share the remaining pool with
15055          * the other PFs. We allocate space for more than the guarantee with
15056          * the understanding that we might not get them all later.
15057          */
15058         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15059                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15060         else
15061                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15062
15063         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15064         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15065                           GFP_KERNEL);
15066         if (!pf->vsi) {
15067                 err = -ENOMEM;
15068                 goto err_switch_setup;
15069         }
15070
15071 #ifdef CONFIG_PCI_IOV
15072         /* prep for VF support */
15073         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15074             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15075             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15076                 if (pci_num_vf(pdev))
15077                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15078         }
15079 #endif
15080         err = i40e_setup_pf_switch(pf, false);
15081         if (err) {
15082                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15083                 goto err_vsis;
15084         }
15085         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15086
15087         /* Make sure flow control is set according to current settings */
15088         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
15089         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
15090                 dev_dbg(&pf->pdev->dev,
15091                         "Set fc with err %s aq_err %s on get_phy_cap\n",
15092                         i40e_stat_str(hw, err),
15093                         i40e_aq_str(hw, hw->aq.asq_last_status));
15094         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
15095                 dev_dbg(&pf->pdev->dev,
15096                         "Set fc with err %s aq_err %s on set_phy_config\n",
15097                         i40e_stat_str(hw, err),
15098                         i40e_aq_str(hw, hw->aq.asq_last_status));
15099         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
15100                 dev_dbg(&pf->pdev->dev,
15101                         "Set fc with err %s aq_err %s on get_link_info\n",
15102                         i40e_stat_str(hw, err),
15103                         i40e_aq_str(hw, hw->aq.asq_last_status));
15104
15105         /* if FDIR VSI was set up, start it now */
15106         for (i = 0; i < pf->num_alloc_vsi; i++) {
15107                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15108                         i40e_vsi_open(pf->vsi[i]);
15109                         break;
15110                 }
15111         }
15112
15113         /* The driver only wants link up/down and module qualification
15114          * reports from firmware.  Note the negative logic.
15115          */
15116         err = i40e_aq_set_phy_int_mask(&pf->hw,
15117                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15118                                          I40E_AQ_EVENT_MEDIA_NA |
15119                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15120         if (err)
15121                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15122                          i40e_stat_str(&pf->hw, err),
15123                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15124
15125         /* Reconfigure hardware for allowing smaller MSS in the case
15126          * of TSO, so that we avoid the MDD being fired and causing
15127          * a reset in the case of small MSS+TSO.
15128          */
15129         val = rd32(hw, I40E_REG_MSS);
15130         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15131                 val &= ~I40E_REG_MSS_MIN_MASK;
15132                 val |= I40E_64BYTE_MSS;
15133                 wr32(hw, I40E_REG_MSS, val);
15134         }
15135
15136         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15137                 msleep(75);
15138                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15139                 if (err)
15140                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15141                                  i40e_stat_str(&pf->hw, err),
15142                                  i40e_aq_str(&pf->hw,
15143                                              pf->hw.aq.asq_last_status));
15144         }
15145         /* The main driver is (mostly) up and happy. We need to set this state
15146          * before setting up the misc vector or we get a race and the vector
15147          * ends up disabled forever.
15148          */
15149         clear_bit(__I40E_DOWN, pf->state);
15150
15151         /* In case of MSIX we are going to setup the misc vector right here
15152          * to handle admin queue events etc. In case of legacy and MSI
15153          * the misc functionality and queue processing is combined in
15154          * the same vector and that gets setup at open.
15155          */
15156         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15157                 err = i40e_setup_misc_vector(pf);
15158                 if (err) {
15159                         dev_info(&pdev->dev,
15160                                  "setup of misc vector failed: %d\n", err);
15161                         goto err_vsis;
15162                 }
15163         }
15164
15165 #ifdef CONFIG_PCI_IOV
15166         /* prep for VF support */
15167         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15168             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15169             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15170                 /* disable link interrupts for VFs */
15171                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15172                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15173                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15174                 i40e_flush(hw);
15175
15176                 if (pci_num_vf(pdev)) {
15177                         dev_info(&pdev->dev,
15178                                  "Active VFs found, allocating resources.\n");
15179                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15180                         if (err)
15181                                 dev_info(&pdev->dev,
15182                                          "Error %d allocating resources for existing VFs\n",
15183                                          err);
15184                 }
15185         }
15186 #endif /* CONFIG_PCI_IOV */
15187
15188         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15189                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15190                                                       pf->num_iwarp_msix,
15191                                                       I40E_IWARP_IRQ_PILE_ID);
15192                 if (pf->iwarp_base_vector < 0) {
15193                         dev_info(&pdev->dev,
15194                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
15195                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
15196                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15197                 }
15198         }
15199
15200         i40e_dbg_pf_init(pf);
15201
15202         /* tell the firmware that we're starting */
15203         i40e_send_version(pf);
15204
15205         /* since everything's happy, start the service_task timer */
15206         mod_timer(&pf->service_timer,
15207                   round_jiffies(jiffies + pf->service_timer_period));
15208
15209         /* add this PF to client device list and launch a client service task */
15210         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15211                 err = i40e_lan_add_device(pf);
15212                 if (err)
15213                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15214                                  err);
15215         }
15216
15217 #define PCI_SPEED_SIZE 8
15218 #define PCI_WIDTH_SIZE 8
15219         /* Devices on the IOSF bus do not have this information
15220          * and will report PCI Gen 1 x 1 by default so don't bother
15221          * checking them.
15222          */
15223         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15224                 char speed[PCI_SPEED_SIZE] = "Unknown";
15225                 char width[PCI_WIDTH_SIZE] = "Unknown";
15226
15227                 /* Get the negotiated link width and speed from PCI config
15228                  * space
15229                  */
15230                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15231                                           &link_status);
15232
15233                 i40e_set_pci_config_data(hw, link_status);
15234
15235                 switch (hw->bus.speed) {
15236                 case i40e_bus_speed_8000:
15237                         strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15238                 case i40e_bus_speed_5000:
15239                         strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15240                 case i40e_bus_speed_2500:
15241                         strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15242                 default:
15243                         break;
15244                 }
15245                 switch (hw->bus.width) {
15246                 case i40e_bus_width_pcie_x8:
15247                         strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15248                 case i40e_bus_width_pcie_x4:
15249                         strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15250                 case i40e_bus_width_pcie_x2:
15251                         strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15252                 case i40e_bus_width_pcie_x1:
15253                         strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15254                 default:
15255                         break;
15256                 }
15257
15258                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15259                          speed, width);
15260
15261                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15262                     hw->bus.speed < i40e_bus_speed_8000) {
15263                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15264                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15265                 }
15266         }
15267
15268         /* get the requested speeds from the fw */
15269         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15270         if (err)
15271                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
15272                         i40e_stat_str(&pf->hw, err),
15273                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15274         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15275
15276         /* set the FEC config due to the board capabilities */
15277         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15278
15279         /* get the supported phy types from the fw */
15280         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15281         if (err)
15282                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
15283                         i40e_stat_str(&pf->hw, err),
15284                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15285
15286         /* Add a filter to drop all Flow control frames from any VSI from being
15287          * transmitted. By doing so we stop a malicious VF from sending out
15288          * PAUSE or PFC frames and potentially controlling traffic for other
15289          * PF/VF VSIs.
15290          * The FW can still send Flow control frames if enabled.
15291          */
15292         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15293                                                        pf->main_vsi_seid);
15294
15295         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15296                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15297                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15298         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15299                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15300         /* print a string summarizing features */
15301         i40e_print_features(pf);
15302
15303         return 0;
15304
15305         /* Unwind what we've done if something failed in the setup */
15306 err_vsis:
15307         set_bit(__I40E_DOWN, pf->state);
15308         i40e_clear_interrupt_scheme(pf);
15309         kfree(pf->vsi);
15310 err_switch_setup:
15311         i40e_reset_interrupt_capability(pf);
15312         del_timer_sync(&pf->service_timer);
15313 err_mac_addr:
15314 err_configure_lan_hmc:
15315         (void)i40e_shutdown_lan_hmc(hw);
15316 err_init_lan_hmc:
15317         kfree(pf->qp_pile);
15318 err_sw_init:
15319 err_adminq_setup:
15320 err_pf_reset:
15321         iounmap(hw->hw_addr);
15322 err_ioremap:
15323         kfree(pf);
15324 err_pf_alloc:
15325         pci_disable_pcie_error_reporting(pdev);
15326         pci_release_mem_regions(pdev);
15327 err_pci_reg:
15328 err_dma:
15329         pci_disable_device(pdev);
15330         return err;
15331 }
15332
15333 /**
15334  * i40e_remove - Device removal routine
15335  * @pdev: PCI device information struct
15336  *
15337  * i40e_remove is called by the PCI subsystem to alert the driver
15338  * that is should release a PCI device.  This could be caused by a
15339  * Hot-Plug event, or because the driver is going to be removed from
15340  * memory.
15341  **/
15342 static void i40e_remove(struct pci_dev *pdev)
15343 {
15344         struct i40e_pf *pf = pci_get_drvdata(pdev);
15345         struct i40e_hw *hw = &pf->hw;
15346         i40e_status ret_code;
15347         int i;
15348
15349         i40e_dbg_pf_exit(pf);
15350
15351         i40e_ptp_stop(pf);
15352
15353         /* Disable RSS in hw */
15354         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15355         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15356
15357         /* no more scheduling of any task */
15358         set_bit(__I40E_SUSPENDED, pf->state);
15359         set_bit(__I40E_DOWN, pf->state);
15360         if (pf->service_timer.function)
15361                 del_timer_sync(&pf->service_timer);
15362         if (pf->service_task.func)
15363                 cancel_work_sync(&pf->service_task);
15364
15365         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15366                 struct i40e_vsi *vsi = pf->vsi[0];
15367
15368                 /* We know that we have allocated only one vsi for this PF,
15369                  * it was just for registering netdevice, so the interface
15370                  * could be visible in the 'ifconfig' output
15371                  */
15372                 unregister_netdev(vsi->netdev);
15373                 free_netdev(vsi->netdev);
15374
15375                 goto unmap;
15376         }
15377
15378         /* Client close must be called explicitly here because the timer
15379          * has been stopped.
15380          */
15381         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15382
15383         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15384                 i40e_free_vfs(pf);
15385                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15386         }
15387
15388         i40e_fdir_teardown(pf);
15389
15390         /* If there is a switch structure or any orphans, remove them.
15391          * This will leave only the PF's VSI remaining.
15392          */
15393         for (i = 0; i < I40E_MAX_VEB; i++) {
15394                 if (!pf->veb[i])
15395                         continue;
15396
15397                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15398                     pf->veb[i]->uplink_seid == 0)
15399                         i40e_switch_branch_release(pf->veb[i]);
15400         }
15401
15402         /* Now we can shutdown the PF's VSI, just before we kill
15403          * adminq and hmc.
15404          */
15405         if (pf->vsi[pf->lan_vsi])
15406                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
15407
15408         i40e_cloud_filter_exit(pf);
15409
15410         /* remove attached clients */
15411         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15412                 ret_code = i40e_lan_del_device(pf);
15413                 if (ret_code)
15414                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15415                                  ret_code);
15416         }
15417
15418         /* shutdown and destroy the HMC */
15419         if (hw->hmc.hmc_obj) {
15420                 ret_code = i40e_shutdown_lan_hmc(hw);
15421                 if (ret_code)
15422                         dev_warn(&pdev->dev,
15423                                  "Failed to destroy the HMC resources: %d\n",
15424                                  ret_code);
15425         }
15426
15427 unmap:
15428         /* Free MSI/legacy interrupt 0 when in recovery mode. */
15429         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15430             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15431                 free_irq(pf->pdev->irq, pf);
15432
15433         /* shutdown the adminq */
15434         i40e_shutdown_adminq(hw);
15435
15436         /* destroy the locks only once, here */
15437         mutex_destroy(&hw->aq.arq_mutex);
15438         mutex_destroy(&hw->aq.asq_mutex);
15439
15440         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
15441         rtnl_lock();
15442         i40e_clear_interrupt_scheme(pf);
15443         for (i = 0; i < pf->num_alloc_vsi; i++) {
15444                 if (pf->vsi[i]) {
15445                         if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
15446                                 i40e_vsi_clear_rings(pf->vsi[i]);
15447                         i40e_vsi_clear(pf->vsi[i]);
15448                         pf->vsi[i] = NULL;
15449                 }
15450         }
15451         rtnl_unlock();
15452
15453         for (i = 0; i < I40E_MAX_VEB; i++) {
15454                 kfree(pf->veb[i]);
15455                 pf->veb[i] = NULL;
15456         }
15457
15458         kfree(pf->qp_pile);
15459         kfree(pf->vsi);
15460
15461         iounmap(hw->hw_addr);
15462         kfree(pf);
15463         pci_release_mem_regions(pdev);
15464
15465         pci_disable_pcie_error_reporting(pdev);
15466         pci_disable_device(pdev);
15467 }
15468
15469 /**
15470  * i40e_pci_error_detected - warning that something funky happened in PCI land
15471  * @pdev: PCI device information struct
15472  * @error: the type of PCI error
15473  *
15474  * Called to warn that something happened and the error handling steps
15475  * are in progress.  Allows the driver to quiesce things, be ready for
15476  * remediation.
15477  **/
15478 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
15479                                                 enum pci_channel_state error)
15480 {
15481         struct i40e_pf *pf = pci_get_drvdata(pdev);
15482
15483         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
15484
15485         if (!pf) {
15486                 dev_info(&pdev->dev,
15487                          "Cannot recover - error happened during device probe\n");
15488                 return PCI_ERS_RESULT_DISCONNECT;
15489         }
15490
15491         /* shutdown all operations */
15492         if (!test_bit(__I40E_SUSPENDED, pf->state))
15493                 i40e_prep_for_reset(pf, false);
15494
15495         /* Request a slot reset */
15496         return PCI_ERS_RESULT_NEED_RESET;
15497 }
15498
15499 /**
15500  * i40e_pci_error_slot_reset - a PCI slot reset just happened
15501  * @pdev: PCI device information struct
15502  *
15503  * Called to find if the driver can work with the device now that
15504  * the pci slot has been reset.  If a basic connection seems good
15505  * (registers are readable and have sane content) then return a
15506  * happy little PCI_ERS_RESULT_xxx.
15507  **/
15508 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
15509 {
15510         struct i40e_pf *pf = pci_get_drvdata(pdev);
15511         pci_ers_result_t result;
15512         u32 reg;
15513
15514         dev_dbg(&pdev->dev, "%s\n", __func__);
15515         if (pci_enable_device_mem(pdev)) {
15516                 dev_info(&pdev->dev,
15517                          "Cannot re-enable PCI device after reset.\n");
15518                 result = PCI_ERS_RESULT_DISCONNECT;
15519         } else {
15520                 pci_set_master(pdev);
15521                 pci_restore_state(pdev);
15522                 pci_save_state(pdev);
15523                 pci_wake_from_d3(pdev, false);
15524
15525                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
15526                 if (reg == 0)
15527                         result = PCI_ERS_RESULT_RECOVERED;
15528                 else
15529                         result = PCI_ERS_RESULT_DISCONNECT;
15530         }
15531
15532         return result;
15533 }
15534
15535 /**
15536  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
15537  * @pdev: PCI device information struct
15538  */
15539 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
15540 {
15541         struct i40e_pf *pf = pci_get_drvdata(pdev);
15542
15543         i40e_prep_for_reset(pf, false);
15544 }
15545
15546 /**
15547  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
15548  * @pdev: PCI device information struct
15549  */
15550 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
15551 {
15552         struct i40e_pf *pf = pci_get_drvdata(pdev);
15553
15554         i40e_reset_and_rebuild(pf, false, false);
15555 }
15556
15557 /**
15558  * i40e_pci_error_resume - restart operations after PCI error recovery
15559  * @pdev: PCI device information struct
15560  *
15561  * Called to allow the driver to bring things back up after PCI error
15562  * and/or reset recovery has finished.
15563  **/
15564 static void i40e_pci_error_resume(struct pci_dev *pdev)
15565 {
15566         struct i40e_pf *pf = pci_get_drvdata(pdev);
15567
15568         dev_dbg(&pdev->dev, "%s\n", __func__);
15569         if (test_bit(__I40E_SUSPENDED, pf->state))
15570                 return;
15571
15572         i40e_handle_reset_warning(pf, false);
15573 }
15574
15575 /**
15576  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
15577  * using the mac_address_write admin q function
15578  * @pf: pointer to i40e_pf struct
15579  **/
15580 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
15581 {
15582         struct i40e_hw *hw = &pf->hw;
15583         i40e_status ret;
15584         u8 mac_addr[6];
15585         u16 flags = 0;
15586
15587         /* Get current MAC address in case it's an LAA */
15588         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
15589                 ether_addr_copy(mac_addr,
15590                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
15591         } else {
15592                 dev_err(&pf->pdev->dev,
15593                         "Failed to retrieve MAC address; using default\n");
15594                 ether_addr_copy(mac_addr, hw->mac.addr);
15595         }
15596
15597         /* The FW expects the mac address write cmd to first be called with
15598          * one of these flags before calling it again with the multicast
15599          * enable flags.
15600          */
15601         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
15602
15603         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
15604                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
15605
15606         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15607         if (ret) {
15608                 dev_err(&pf->pdev->dev,
15609                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
15610                 return;
15611         }
15612
15613         flags = I40E_AQC_MC_MAG_EN
15614                         | I40E_AQC_WOL_PRESERVE_ON_PFR
15615                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
15616         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15617         if (ret)
15618                 dev_err(&pf->pdev->dev,
15619                         "Failed to enable Multicast Magic Packet wake up\n");
15620 }
15621
15622 /**
15623  * i40e_shutdown - PCI callback for shutting down
15624  * @pdev: PCI device information struct
15625  **/
15626 static void i40e_shutdown(struct pci_dev *pdev)
15627 {
15628         struct i40e_pf *pf = pci_get_drvdata(pdev);
15629         struct i40e_hw *hw = &pf->hw;
15630
15631         set_bit(__I40E_SUSPENDED, pf->state);
15632         set_bit(__I40E_DOWN, pf->state);
15633
15634         del_timer_sync(&pf->service_timer);
15635         cancel_work_sync(&pf->service_task);
15636         i40e_cloud_filter_exit(pf);
15637         i40e_fdir_teardown(pf);
15638
15639         /* Client close must be called explicitly here because the timer
15640          * has been stopped.
15641          */
15642         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15643
15644         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15645                 i40e_enable_mc_magic_wake(pf);
15646
15647         i40e_prep_for_reset(pf, false);
15648
15649         wr32(hw, I40E_PFPM_APM,
15650              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15651         wr32(hw, I40E_PFPM_WUFC,
15652              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15653
15654         /* Free MSI/legacy interrupt 0 when in recovery mode. */
15655         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15656             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15657                 free_irq(pf->pdev->irq, pf);
15658
15659         /* Since we're going to destroy queues during the
15660          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
15661          * whole section
15662          */
15663         rtnl_lock();
15664         i40e_clear_interrupt_scheme(pf);
15665         rtnl_unlock();
15666
15667         if (system_state == SYSTEM_POWER_OFF) {
15668                 pci_wake_from_d3(pdev, pf->wol_en);
15669                 pci_set_power_state(pdev, PCI_D3hot);
15670         }
15671 }
15672
15673 /**
15674  * i40e_suspend - PM callback for moving to D3
15675  * @dev: generic device information structure
15676  **/
15677 static int __maybe_unused i40e_suspend(struct device *dev)
15678 {
15679         struct i40e_pf *pf = dev_get_drvdata(dev);
15680         struct i40e_hw *hw = &pf->hw;
15681
15682         /* If we're already suspended, then there is nothing to do */
15683         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
15684                 return 0;
15685
15686         set_bit(__I40E_DOWN, pf->state);
15687
15688         /* Ensure service task will not be running */
15689         del_timer_sync(&pf->service_timer);
15690         cancel_work_sync(&pf->service_task);
15691
15692         /* Client close must be called explicitly here because the timer
15693          * has been stopped.
15694          */
15695         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15696
15697         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15698                 i40e_enable_mc_magic_wake(pf);
15699
15700         /* Since we're going to destroy queues during the
15701          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
15702          * whole section
15703          */
15704         rtnl_lock();
15705
15706         i40e_prep_for_reset(pf, true);
15707
15708         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15709         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15710
15711         /* Clear the interrupt scheme and release our IRQs so that the system
15712          * can safely hibernate even when there are a large number of CPUs.
15713          * Otherwise hibernation might fail when mapping all the vectors back
15714          * to CPU0.
15715          */
15716         i40e_clear_interrupt_scheme(pf);
15717
15718         rtnl_unlock();
15719
15720         return 0;
15721 }
15722
15723 /**
15724  * i40e_resume - PM callback for waking up from D3
15725  * @dev: generic device information structure
15726  **/
15727 static int __maybe_unused i40e_resume(struct device *dev)
15728 {
15729         struct i40e_pf *pf = dev_get_drvdata(dev);
15730         int err;
15731
15732         /* If we're not suspended, then there is nothing to do */
15733         if (!test_bit(__I40E_SUSPENDED, pf->state))
15734                 return 0;
15735
15736         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
15737          * since we're going to be restoring queues
15738          */
15739         rtnl_lock();
15740
15741         /* We cleared the interrupt scheme when we suspended, so we need to
15742          * restore it now to resume device functionality.
15743          */
15744         err = i40e_restore_interrupt_scheme(pf);
15745         if (err) {
15746                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
15747                         err);
15748         }
15749
15750         clear_bit(__I40E_DOWN, pf->state);
15751         i40e_reset_and_rebuild(pf, false, true);
15752
15753         rtnl_unlock();
15754
15755         /* Clear suspended state last after everything is recovered */
15756         clear_bit(__I40E_SUSPENDED, pf->state);
15757
15758         /* Restart the service task */
15759         mod_timer(&pf->service_timer,
15760                   round_jiffies(jiffies + pf->service_timer_period));
15761
15762         return 0;
15763 }
15764
15765 static const struct pci_error_handlers i40e_err_handler = {
15766         .error_detected = i40e_pci_error_detected,
15767         .slot_reset = i40e_pci_error_slot_reset,
15768         .reset_prepare = i40e_pci_error_reset_prepare,
15769         .reset_done = i40e_pci_error_reset_done,
15770         .resume = i40e_pci_error_resume,
15771 };
15772
15773 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
15774
15775 static struct pci_driver i40e_driver = {
15776         .name     = i40e_driver_name,
15777         .id_table = i40e_pci_tbl,
15778         .probe    = i40e_probe,
15779         .remove   = i40e_remove,
15780         .driver   = {
15781                 .pm = &i40e_pm_ops,
15782         },
15783         .shutdown = i40e_shutdown,
15784         .err_handler = &i40e_err_handler,
15785         .sriov_configure = i40e_pci_sriov_configure,
15786 };
15787
15788 /**
15789  * i40e_init_module - Driver registration routine
15790  *
15791  * i40e_init_module is the first routine called when the driver is
15792  * loaded. All it does is register with the PCI subsystem.
15793  **/
15794 static int __init i40e_init_module(void)
15795 {
15796         pr_info("%s: %s - version %s\n", i40e_driver_name,
15797                 i40e_driver_string, i40e_driver_version_str);
15798         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
15799
15800         /* There is no need to throttle the number of active tasks because
15801          * each device limits its own task using a state bit for scheduling
15802          * the service task, and the device tasks do not interfere with each
15803          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
15804          * since we need to be able to guarantee forward progress even under
15805          * memory pressure.
15806          */
15807         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
15808         if (!i40e_wq) {
15809                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
15810                 return -ENOMEM;
15811         }
15812
15813         i40e_dbg_init();
15814         return pci_register_driver(&i40e_driver);
15815 }
15816 module_init(i40e_init_module);
15817
15818 /**
15819  * i40e_exit_module - Driver exit cleanup routine
15820  *
15821  * i40e_exit_module is called just before the driver is removed
15822  * from memory.
15823  **/
15824 static void __exit i40e_exit_module(void)
15825 {
15826         pci_unregister_driver(&i40e_driver);
15827         destroy_workqueue(i40e_wq);
15828         i40e_dbg_exit();
15829 }
15830 module_exit(i40e_exit_module);