drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2018 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/if.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/if_bridge.h>
  37 #include <linux/rtc.h>
  38 #include <linux/bpf.h>
  39 #include <net/ip.h>
  40 #include <net/tcp.h>
  41 #include <net/udp.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <net/udp_tunnel.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/prefetch.h>
  47 #include <linux/cache.h>
  48 #include <linux/log2.h>
  49 #include <linux/aer.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/cpu_rmap.h>
  52 #include <linux/cpumask.h>
  53 #include <net/pkt_cls.h>
  54
  55 #include "bnxt_hsi.h"
  56 #include "bnxt.h"
  57 #include "bnxt_ulp.h"
  58 #include "bnxt_sriov.h"
  59 #include "bnxt_ethtool.h"
  60 #include "bnxt_dcb.h"
  61 #include "bnxt_xdp.h"
  62 #include "bnxt_vfr.h"
  63 #include "bnxt_tc.h"
  64 #include "bnxt_devlink.h"
  65
  66 #define BNXT_TX_TIMEOUT         (5 * HZ)
  67
  68 static const char version[] =
  69         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  70
  71 MODULE_LICENSE("GPL");
  72 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  73 MODULE_VERSION(DRV_MODULE_VERSION);
  74
  75 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  76 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  77 #define BNXT_RX_COPY_THRESH 256
  78
  79 #define BNXT_TX_PUSH_THRESH 164
  80
  81 enum board_idx {
  82         BCM57301,
  83         BCM57302,
  84         BCM57304,
  85         BCM57417_NPAR,
  86         BCM58700,
  87         BCM57311,
  88         BCM57312,
  89         BCM57402,
  90         BCM57404,
  91         BCM57406,
  92         BCM57402_NPAR,
  93         BCM57407,
  94         BCM57412,
  95         BCM57414,
  96         BCM57416,
  97         BCM57417,
  98         BCM57412_NPAR,
  99         BCM57314,
 100         BCM57417_SFP,
 101         BCM57416_SFP,
 102         BCM57404_NPAR,
 103         BCM57406_NPAR,
 104         BCM57407_SFP,
 105         BCM57407_NPAR,
 106         BCM57414_NPAR,
 107         BCM57416_NPAR,
 108         BCM57452,
 109         BCM57454,
 110         BCM5745x_NPAR,
 111         BCM58802,
 112         BCM58804,
 113         BCM58808,
 114         NETXTREME_E_VF,
 115         NETXTREME_C_VF,
 116         NETXTREME_S_VF,
 117 };
 118
 119 /* indexed by enum above */
 120 static const struct {
 121         char *name;
 122 } board_info[] = {
 123         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 124         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 125         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 126         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 127         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 128         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 129         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 130         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 131         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 132         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 133         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 134         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 135         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 136         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 137         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 138         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 139         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 140         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 141         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 142         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 143         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 144         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 145         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 146         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 147         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 148         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 149         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 150         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 151         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 152         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 153         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 154         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 155         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 156         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 157         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 158 };
 159
 160 static const struct pci_device_id bnxt_pci_tbl[] = {
 161         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 162         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 163         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 164         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 165         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 166         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 167         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 168         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 170         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 171         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 172         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 173         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 174         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 175         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 176         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 177         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 178         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 179         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 180         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 181         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 183         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 184         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 185         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 186         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 187         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 188         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 189         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 190         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 191         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 193         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 194         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 195         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 196         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 197         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 198 #ifdef CONFIG_BNXT_SRIOV
 199         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 201         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 202         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 203         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 204         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 205         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 206         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 207         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 208 #endif
 209         { 0 }
 210 };
 211
 212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 213
 214 static const u16 bnxt_vf_req_snif[] = {
 215         HWRM_FUNC_CFG,
 216         HWRM_FUNC_VF_CFG,
 217         HWRM_PORT_PHY_QCFG,
 218         HWRM_CFA_L2_FILTER_ALLOC,
 219 };
 220
 221 static const u16 bnxt_async_events_arr[] = {
 222         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 223         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 224         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 225         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 226         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 227 };
 228
 229 static struct workqueue_struct *bnxt_pf_wq;
 230
 231 static bool bnxt_vf_pciid(enum board_idx idx)
 232 {
 233         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 234                 idx == NETXTREME_S_VF);
 235 }
 236
 237 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 238 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 239 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 240
 241 #define BNXT_CP_DB_REARM(db, raw_cons)                                  \
 242                 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
 243
 244 #define BNXT_CP_DB(db, raw_cons)                                        \
 245                 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
 246
 247 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 248                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 249
 250 const u16 bnxt_lhint_arr[] = {
 251         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 252         TX_BD_FLAGS_LHINT_512_TO_1023,
 253         TX_BD_FLAGS_LHINT_1024_TO_2047,
 254         TX_BD_FLAGS_LHINT_1024_TO_2047,
 255         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 256         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 257         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 258         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 259         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 260         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 261         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 262         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 263         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 264         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 265         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 266         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 267         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 268         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 269         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 270 };
 271
 272 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 273 {
 274         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 275
 276         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 277                 return 0;
 278
 279         return md_dst->u.port_info.port_id;
 280 }
 281
 282 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 283 {
 284         struct bnxt *bp = netdev_priv(dev);
 285         struct tx_bd *txbd;
 286         struct tx_bd_ext *txbd1;
 287         struct netdev_queue *txq;
 288         int i;
 289         dma_addr_t mapping;
 290         unsigned int length, pad = 0;
 291         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 292         u16 prod, last_frag;
 293         struct pci_dev *pdev = bp->pdev;
 294         struct bnxt_tx_ring_info *txr;
 295         struct bnxt_sw_tx_bd *tx_buf;
 296
 297         i = skb_get_queue_mapping(skb);
 298         if (unlikely(i >= bp->tx_nr_rings)) {
 299                 dev_kfree_skb_any(skb);
 300                 return NETDEV_TX_OK;
 301         }
 302
 303         txq = netdev_get_tx_queue(dev, i);
 304         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 305         prod = txr->tx_prod;
 306
 307         free_size = bnxt_tx_avail(bp, txr);
 308         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 309                 netif_tx_stop_queue(txq);
 310                 return NETDEV_TX_BUSY;
 311         }
 312
 313         length = skb->len;
 314         len = skb_headlen(skb);
 315         last_frag = skb_shinfo(skb)->nr_frags;
 316
 317         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 318
 319         txbd->tx_bd_opaque = prod;
 320
 321         tx_buf = &txr->tx_buf_ring[prod];
 322         tx_buf->skb = skb;
 323         tx_buf->nr_frags = last_frag;
 324
 325         vlan_tag_flags = 0;
 326         cfa_action = bnxt_xmit_get_cfa_action(skb);
 327         if (skb_vlan_tag_present(skb)) {
 328                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 329                                  skb_vlan_tag_get(skb);
 330                 /* Currently supports 8021Q, 8021AD vlan offloads
 331                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 332                  */
 333                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 334                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 335         }
 336
 337         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 338                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 339                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 340                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 341                 void *pdata = tx_push_buf->data;
 342                 u64 *end;
 343                 int j, push_len;
 344
 345                 /* Set COAL_NOW to be ready quickly for the next push */
 346                 tx_push->tx_bd_len_flags_type =
 347                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 348                                         TX_BD_TYPE_LONG_TX_BD |
 349                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 350                                         TX_BD_FLAGS_COAL_NOW |
 351                                         TX_BD_FLAGS_PACKET_END |
 352                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 353
 354                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 355                         tx_push1->tx_bd_hsize_lflags =
 356                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 357                 else
 358                         tx_push1->tx_bd_hsize_lflags = 0;
 359
 360                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 361                 tx_push1->tx_bd_cfa_action =
 362                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 363
 364                 end = pdata + length;
 365                 end = PTR_ALIGN(end, 8) - 1;
 366                 *end = 0;
 367
 368                 skb_copy_from_linear_data(skb, pdata, len);
 369                 pdata += len;
 370                 for (j = 0; j < last_frag; j++) {
 371                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 372                         void *fptr;
 373
 374                         fptr = skb_frag_address_safe(frag);
 375                         if (!fptr)
 376                                 goto normal_tx;
 377
 378                         memcpy(pdata, fptr, skb_frag_size(frag));
 379                         pdata += skb_frag_size(frag);
 380                 }
 381
 382                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 383                 txbd->tx_bd_haddr = txr->data_mapping;
 384                 prod = NEXT_TX(prod);
 385                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 386                 memcpy(txbd, tx_push1, sizeof(*txbd));
 387                 prod = NEXT_TX(prod);
 388                 tx_push->doorbell =
 389                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 390                 txr->tx_prod = prod;
 391
 392                 tx_buf->is_push = 1;
 393                 netdev_tx_sent_queue(txq, skb->len);
 394                 wmb();  /* Sync is_push and byte queue before pushing data */
 395
 396                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 397                 if (push_len > 16) {
 398                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
 399                         __iowrite32_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
 400                                          (push_len - 16) << 1);
 401                 } else {
 402                         __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
 403                                          push_len);
 404                 }
 405
 406                 goto tx_done;
 407         }
 408
 409 normal_tx:
 410         if (length < BNXT_MIN_PKT_SIZE) {
 411                 pad = BNXT_MIN_PKT_SIZE - length;
 412                 if (skb_pad(skb, pad)) {
 413                         /* SKB already freed. */
 414                         tx_buf->skb = NULL;
 415                         return NETDEV_TX_OK;
 416                 }
 417                 length = BNXT_MIN_PKT_SIZE;
 418         }
 419
 420         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 421
 422         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 423                 dev_kfree_skb_any(skb);
 424                 tx_buf->skb = NULL;
 425                 return NETDEV_TX_OK;
 426         }
 427
 428         dma_unmap_addr_set(tx_buf, mapping, mapping);
 429         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 430                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 431
 432         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 433
 434         prod = NEXT_TX(prod);
 435         txbd1 = (struct tx_bd_ext *)
 436                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 437
 438         txbd1->tx_bd_hsize_lflags = 0;
 439         if (skb_is_gso(skb)) {
 440                 u32 hdr_len;
 441
 442                 if (skb->encapsulation)
 443                         hdr_len = skb_inner_network_offset(skb) +
 444                                 skb_inner_network_header_len(skb) +
 445                                 inner_tcp_hdrlen(skb);
 446                 else
 447                         hdr_len = skb_transport_offset(skb) +
 448                                 tcp_hdrlen(skb);
 449
 450                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 451                                         TX_BD_FLAGS_T_IPID |
 452                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 453                 length = skb_shinfo(skb)->gso_size;
 454                 txbd1->tx_bd_mss = cpu_to_le32(length);
 455                 length += hdr_len;
 456         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 457                 txbd1->tx_bd_hsize_lflags =
 458                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 459                 txbd1->tx_bd_mss = 0;
 460         }
 461
 462         length >>= 9;
 463         flags |= bnxt_lhint_arr[length];
 464         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 465
 466         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 467         txbd1->tx_bd_cfa_action =
 468                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 469         for (i = 0; i < last_frag; i++) {
 470                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 471
 472                 prod = NEXT_TX(prod);
 473                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 474
 475                 len = skb_frag_size(frag);
 476                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 477                                            DMA_TO_DEVICE);
 478
 479                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 480                         goto tx_dma_error;
 481
 482                 tx_buf = &txr->tx_buf_ring[prod];
 483                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 484
 485                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 486
 487                 flags = len << TX_BD_LEN_SHIFT;
 488                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 489         }
 490
 491         flags &= ~TX_BD_LEN;
 492         txbd->tx_bd_len_flags_type =
 493                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 494                             TX_BD_FLAGS_PACKET_END);
 495
 496         netdev_tx_sent_queue(txq, skb->len);
 497
 498         /* Sync BD data before updating doorbell */
 499         wmb();
 500
 501         prod = NEXT_TX(prod);
 502         txr->tx_prod = prod;
 503
 504         if (!skb->xmit_more || netif_xmit_stopped(txq))
 505                 bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 506
 507 tx_done:
 508
 509         mmiowb();
 510
 511         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 512                 if (skb->xmit_more && !tx_buf->is_push)
 513                         bnxt_db_write(bp, txr->tx_doorbell, DB_KEY_TX | prod);
 514
 515                 netif_tx_stop_queue(txq);
 516
 517                 /* netif_tx_stop_queue() must be done before checking
 518                  * tx index in bnxt_tx_avail() below, because in
 519                  * bnxt_tx_int(), we update tx index before checking for
 520                  * netif_tx_queue_stopped().
 521                  */
 522                 smp_mb();
 523                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 524                         netif_tx_wake_queue(txq);
 525         }
 526         return NETDEV_TX_OK;
 527
 528 tx_dma_error:
 529         last_frag = i;
 530
 531         /* start back at beginning and unmap skb */
 532         prod = txr->tx_prod;
 533         tx_buf = &txr->tx_buf_ring[prod];
 534         tx_buf->skb = NULL;
 535         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 536                          skb_headlen(skb), PCI_DMA_TODEVICE);
 537         prod = NEXT_TX(prod);
 538
 539         /* unmap remaining mapped pages */
 540         for (i = 0; i < last_frag; i++) {
 541                 prod = NEXT_TX(prod);
 542                 tx_buf = &txr->tx_buf_ring[prod];
 543                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 544                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 545                                PCI_DMA_TODEVICE);
 546         }
 547
 548         dev_kfree_skb_any(skb);
 549         return NETDEV_TX_OK;
 550 }
 551
 552 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 553 {
 554         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 555         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 556         u16 cons = txr->tx_cons;
 557         struct pci_dev *pdev = bp->pdev;
 558         int i;
 559         unsigned int tx_bytes = 0;
 560
 561         for (i = 0; i < nr_pkts; i++) {
 562                 struct bnxt_sw_tx_bd *tx_buf;
 563                 struct sk_buff *skb;
 564                 int j, last;
 565
 566                 tx_buf = &txr->tx_buf_ring[cons];
 567                 cons = NEXT_TX(cons);
 568                 skb = tx_buf->skb;
 569                 tx_buf->skb = NULL;
 570
 571                 if (tx_buf->is_push) {
 572                         tx_buf->is_push = 0;
 573                         goto next_tx_int;
 574                 }
 575
 576                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 577                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 578                 last = tx_buf->nr_frags;
 579
 580                 for (j = 0; j < last; j++) {
 581                         cons = NEXT_TX(cons);
 582                         tx_buf = &txr->tx_buf_ring[cons];
 583                         dma_unmap_page(
 584                                 &pdev->dev,
 585                                 dma_unmap_addr(tx_buf, mapping),
 586                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 587                                 PCI_DMA_TODEVICE);
 588                 }
 589
 590 next_tx_int:
 591                 cons = NEXT_TX(cons);
 592
 593                 tx_bytes += skb->len;
 594                 dev_kfree_skb_any(skb);
 595         }
 596
 597         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 598         txr->tx_cons = cons;
 599
 600         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 601          * before checking for netif_tx_queue_stopped().  Without the
 602          * memory barrier, there is a small possibility that bnxt_start_xmit()
 603          * will miss it and cause the queue to be stopped forever.
 604          */
 605         smp_mb();
 606
 607         if (unlikely(netif_tx_queue_stopped(txq)) &&
 608             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 609                 __netif_tx_lock(txq, smp_processor_id());
 610                 if (netif_tx_queue_stopped(txq) &&
 611                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 612                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 613                         netif_tx_wake_queue(txq);
 614                 __netif_tx_unlock(txq);
 615         }
 616 }
 617
 618 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 619                                          gfp_t gfp)
 620 {
 621         struct device *dev = &bp->pdev->dev;
 622         struct page *page;
 623
 624         page = alloc_page(gfp);
 625         if (!page)
 626                 return NULL;
 627
 628         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 629                                       DMA_ATTR_WEAK_ORDERING);
 630         if (dma_mapping_error(dev, *mapping)) {
 631                 __free_page(page);
 632                 return NULL;
 633         }
 634         *mapping += bp->rx_dma_offset;
 635         return page;
 636 }
 637
 638 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 639                                        gfp_t gfp)
 640 {
 641         u8 *data;
 642         struct pci_dev *pdev = bp->pdev;
 643
 644         data = kmalloc(bp->rx_buf_size, gfp);
 645         if (!data)
 646                 return NULL;
 647
 648         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 649                                         bp->rx_buf_use_size, bp->rx_dir,
 650                                         DMA_ATTR_WEAK_ORDERING);
 651
 652         if (dma_mapping_error(&pdev->dev, *mapping)) {
 653                 kfree(data);
 654                 data = NULL;
 655         }
 656         return data;
 657 }
 658
 659 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 660                        u16 prod, gfp_t gfp)
 661 {
 662         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 663         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 664         dma_addr_t mapping;
 665
 666         if (BNXT_RX_PAGE_MODE(bp)) {
 667                 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
 668
 669                 if (!page)
 670                         return -ENOMEM;
 671
 672                 rx_buf->data = page;
 673                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 674         } else {
 675                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 676
 677                 if (!data)
 678                         return -ENOMEM;
 679
 680                 rx_buf->data = data;
 681                 rx_buf->data_ptr = data + bp->rx_offset;
 682         }
 683         rx_buf->mapping = mapping;
 684
 685         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 686         return 0;
 687 }
 688
 689 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 690 {
 691         u16 prod = rxr->rx_prod;
 692         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 693         struct rx_bd *cons_bd, *prod_bd;
 694
 695         prod_rx_buf = &rxr->rx_buf_ring[prod];
 696         cons_rx_buf = &rxr->rx_buf_ring[cons];
 697
 698         prod_rx_buf->data = data;
 699         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 700
 701         prod_rx_buf->mapping = cons_rx_buf->mapping;
 702
 703         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 704         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 705
 706         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 707 }
 708
 709 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 710 {
 711         u16 next, max = rxr->rx_agg_bmap_size;
 712
 713         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 714         if (next >= max)
 715                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 716         return next;
 717 }
 718
 719 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 720                                      struct bnxt_rx_ring_info *rxr,
 721                                      u16 prod, gfp_t gfp)
 722 {
 723         struct rx_bd *rxbd =
 724                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 725         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 726         struct pci_dev *pdev = bp->pdev;
 727         struct page *page;
 728         dma_addr_t mapping;
 729         u16 sw_prod = rxr->rx_sw_agg_prod;
 730         unsigned int offset = 0;
 731
 732         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 733                 page = rxr->rx_page;
 734                 if (!page) {
 735                         page = alloc_page(gfp);
 736                         if (!page)
 737                                 return -ENOMEM;
 738                         rxr->rx_page = page;
 739                         rxr->rx_page_offset = 0;
 740                 }
 741                 offset = rxr->rx_page_offset;
 742                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 743                 if (rxr->rx_page_offset == PAGE_SIZE)
 744                         rxr->rx_page = NULL;
 745                 else
 746                         get_page(page);
 747         } else {
 748                 page = alloc_page(gfp);
 749                 if (!page)
 750                         return -ENOMEM;
 751         }
 752
 753         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 754                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 755                                      DMA_ATTR_WEAK_ORDERING);
 756         if (dma_mapping_error(&pdev->dev, mapping)) {
 757                 __free_page(page);
 758                 return -EIO;
 759         }
 760
 761         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 762                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 763
 764         __set_bit(sw_prod, rxr->rx_agg_bmap);
 765         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 766         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 767
 768         rx_agg_buf->page = page;
 769         rx_agg_buf->offset = offset;
 770         rx_agg_buf->mapping = mapping;
 771         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 772         rxbd->rx_bd_opaque = sw_prod;
 773         return 0;
 774 }
 775
 776 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
 777                                    u32 agg_bufs)
 778 {
 779         struct bnxt *bp = bnapi->bp;
 780         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 781         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 782         u16 prod = rxr->rx_agg_prod;
 783         u16 sw_prod = rxr->rx_sw_agg_prod;
 784         u32 i;
 785
 786         for (i = 0; i < agg_bufs; i++) {
 787                 u16 cons;
 788                 struct rx_agg_cmp *agg;
 789                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 790                 struct rx_bd *prod_bd;
 791                 struct page *page;
 792
 793                 agg = (struct rx_agg_cmp *)
 794                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 795                 cons = agg->rx_agg_cmp_opaque;
 796                 __clear_bit(cons, rxr->rx_agg_bmap);
 797
 798                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 799                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 800
 801                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 802                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 803                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 804
 805                 /* It is possible for sw_prod to be equal to cons, so
 806                  * set cons_rx_buf->page to NULL first.
 807                  */
 808                 page = cons_rx_buf->page;
 809                 cons_rx_buf->page = NULL;
 810                 prod_rx_buf->page = page;
 811                 prod_rx_buf->offset = cons_rx_buf->offset;
 812
 813                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 814
 815                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 816
 817                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 818                 prod_bd->rx_bd_opaque = sw_prod;
 819
 820                 prod = NEXT_RX_AGG(prod);
 821                 sw_prod = NEXT_RX_AGG(sw_prod);
 822                 cp_cons = NEXT_CMP(cp_cons);
 823         }
 824         rxr->rx_agg_prod = prod;
 825         rxr->rx_sw_agg_prod = sw_prod;
 826 }
 827
 828 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 829                                         struct bnxt_rx_ring_info *rxr,
 830                                         u16 cons, void *data, u8 *data_ptr,
 831                                         dma_addr_t dma_addr,
 832                                         unsigned int offset_and_len)
 833 {
 834         unsigned int payload = offset_and_len >> 16;
 835         unsigned int len = offset_and_len & 0xffff;
 836         struct skb_frag_struct *frag;
 837         struct page *page = data;
 838         u16 prod = rxr->rx_prod;
 839         struct sk_buff *skb;
 840         int off, err;
 841
 842         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 843         if (unlikely(err)) {
 844                 bnxt_reuse_rx_data(rxr, cons, data);
 845                 return NULL;
 846         }
 847         dma_addr -= bp->rx_dma_offset;
 848         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 849                              DMA_ATTR_WEAK_ORDERING);
 850
 851         if (unlikely(!payload))
 852                 payload = eth_get_headlen(data_ptr, len);
 853
 854         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 855         if (!skb) {
 856                 __free_page(page);
 857                 return NULL;
 858         }
 859
 860         off = (void *)data_ptr - page_address(page);
 861         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 862         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 863                payload + NET_IP_ALIGN);
 864
 865         frag = &skb_shinfo(skb)->frags[0];
 866         skb_frag_size_sub(frag, payload);
 867         frag->page_offset += payload;
 868         skb->data_len -= payload;
 869         skb->tail += payload;
 870
 871         return skb;
 872 }
 873
 874 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 875                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 876                                    void *data, u8 *data_ptr,
 877                                    dma_addr_t dma_addr,
 878                                    unsigned int offset_and_len)
 879 {
 880         u16 prod = rxr->rx_prod;
 881         struct sk_buff *skb;
 882         int err;
 883
 884         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 885         if (unlikely(err)) {
 886                 bnxt_reuse_rx_data(rxr, cons, data);
 887                 return NULL;
 888         }
 889
 890         skb = build_skb(data, 0);
 891         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 892                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 893         if (!skb) {
 894                 kfree(data);
 895                 return NULL;
 896         }
 897
 898         skb_reserve(skb, bp->rx_offset);
 899         skb_put(skb, offset_and_len & 0xffff);
 900         return skb;
 901 }
 902
 903 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
 904                                      struct sk_buff *skb, u16 cp_cons,
 905                                      u32 agg_bufs)
 906 {
 907         struct pci_dev *pdev = bp->pdev;
 908         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 909         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 910         u16 prod = rxr->rx_agg_prod;
 911         u32 i;
 912
 913         for (i = 0; i < agg_bufs; i++) {
 914                 u16 cons, frag_len;
 915                 struct rx_agg_cmp *agg;
 916                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 917                 struct page *page;
 918                 dma_addr_t mapping;
 919
 920                 agg = (struct rx_agg_cmp *)
 921                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 922                 cons = agg->rx_agg_cmp_opaque;
 923                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 924                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 925
 926                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 927                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 928                                    cons_rx_buf->offset, frag_len);
 929                 __clear_bit(cons, rxr->rx_agg_bmap);
 930
 931                 /* It is possible for bnxt_alloc_rx_page() to allocate
 932                  * a sw_prod index that equals the cons index, so we
 933                  * need to clear the cons entry now.
 934                  */
 935                 mapping = cons_rx_buf->mapping;
 936                 page = cons_rx_buf->page;
 937                 cons_rx_buf->page = NULL;
 938
 939                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 940                         struct skb_shared_info *shinfo;
 941                         unsigned int nr_frags;
 942
 943                         shinfo = skb_shinfo(skb);
 944                         nr_frags = --shinfo->nr_frags;
 945                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 946
 947                         dev_kfree_skb(skb);
 948
 949                         cons_rx_buf->page = page;
 950
 951                         /* Update prod since possibly some pages have been
 952                          * allocated already.
 953                          */
 954                         rxr->rx_agg_prod = prod;
 955                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
 956                         return NULL;
 957                 }
 958
 959                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
 960                                      PCI_DMA_FROMDEVICE,
 961                                      DMA_ATTR_WEAK_ORDERING);
 962
 963                 skb->data_len += frag_len;
 964                 skb->len += frag_len;
 965                 skb->truesize += PAGE_SIZE;
 966
 967                 prod = NEXT_RX_AGG(prod);
 968                 cp_cons = NEXT_CMP(cp_cons);
 969         }
 970         rxr->rx_agg_prod = prod;
 971         return skb;
 972 }
 973
 974 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 975                                u8 agg_bufs, u32 *raw_cons)
 976 {
 977         u16 last;
 978         struct rx_agg_cmp *agg;
 979
 980         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
 981         last = RING_CMP(*raw_cons);
 982         agg = (struct rx_agg_cmp *)
 983                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
 984         return RX_AGG_CMP_VALID(agg, *raw_cons);
 985 }
 986
 987 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
 988                                             unsigned int len,
 989                                             dma_addr_t mapping)
 990 {
 991         struct bnxt *bp = bnapi->bp;
 992         struct pci_dev *pdev = bp->pdev;
 993         struct sk_buff *skb;
 994
 995         skb = napi_alloc_skb(&bnapi->napi, len);
 996         if (!skb)
 997                 return NULL;
 998
 999         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1000                                 bp->rx_dir);
1001
1002         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1003                len + NET_IP_ALIGN);
1004
1005         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1006                                    bp->rx_dir);
1007
1008         skb_put(skb, len);
1009         return skb;
1010 }
1011
1012 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
1013                            u32 *raw_cons, void *cmp)
1014 {
1015         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1016         struct rx_cmp *rxcmp = cmp;
1017         u32 tmp_raw_cons = *raw_cons;
1018         u8 cmp_type, agg_bufs = 0;
1019
1020         cmp_type = RX_CMP_TYPE(rxcmp);
1021
1022         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1023                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1024                             RX_CMP_AGG_BUFS) >>
1025                            RX_CMP_AGG_BUFS_SHIFT;
1026         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1027                 struct rx_tpa_end_cmp *tpa_end = cmp;
1028
1029                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1030                             RX_TPA_END_CMP_AGG_BUFS) >>
1031                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1032         }
1033
1034         if (agg_bufs) {
1035                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1036                         return -EBUSY;
1037         }
1038         *raw_cons = tmp_raw_cons;
1039         return 0;
1040 }
1041
1042 static void bnxt_queue_sp_work(struct bnxt *bp)
1043 {
1044         if (BNXT_PF(bp))
1045                 queue_work(bnxt_pf_wq, &bp->sp_task);
1046         else
1047                 schedule_work(&bp->sp_task);
1048 }
1049
1050 static void bnxt_cancel_sp_work(struct bnxt *bp)
1051 {
1052         if (BNXT_PF(bp))
1053                 flush_workqueue(bnxt_pf_wq);
1054         else
1055                 cancel_work_sync(&bp->sp_task);
1056 }
1057
1058 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1059 {
1060         if (!rxr->bnapi->in_reset) {
1061                 rxr->bnapi->in_reset = true;
1062                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1063                 bnxt_queue_sp_work(bp);
1064         }
1065         rxr->rx_next_cons = 0xffff;
1066 }
1067
1068 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1069                            struct rx_tpa_start_cmp *tpa_start,
1070                            struct rx_tpa_start_cmp_ext *tpa_start1)
1071 {
1072         u8 agg_id = TPA_START_AGG_ID(tpa_start);
1073         u16 cons, prod;
1074         struct bnxt_tpa_info *tpa_info;
1075         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1076         struct rx_bd *prod_bd;
1077         dma_addr_t mapping;
1078
1079         cons = tpa_start->rx_tpa_start_cmp_opaque;
1080         prod = rxr->rx_prod;
1081         cons_rx_buf = &rxr->rx_buf_ring[cons];
1082         prod_rx_buf = &rxr->rx_buf_ring[prod];
1083         tpa_info = &rxr->rx_tpa[agg_id];
1084
1085         if (unlikely(cons != rxr->rx_next_cons)) {
1086                 bnxt_sched_reset(bp, rxr);
1087                 return;
1088         }
1089         /* Store cfa_code in tpa_info to use in tpa_end
1090          * completion processing.
1091          */
1092         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1093         prod_rx_buf->data = tpa_info->data;
1094         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1095
1096         mapping = tpa_info->mapping;
1097         prod_rx_buf->mapping = mapping;
1098
1099         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1100
1101         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1102
1103         tpa_info->data = cons_rx_buf->data;
1104         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1105         cons_rx_buf->data = NULL;
1106         tpa_info->mapping = cons_rx_buf->mapping;
1107
1108         tpa_info->len =
1109                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1110                                 RX_TPA_START_CMP_LEN_SHIFT;
1111         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1112                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1113
1114                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1115                 tpa_info->gso_type = SKB_GSO_TCPV4;
1116                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1117                 if (hash_type == 3)
1118                         tpa_info->gso_type = SKB_GSO_TCPV6;
1119                 tpa_info->rss_hash =
1120                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1121         } else {
1122                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1123                 tpa_info->gso_type = 0;
1124                 if (netif_msg_rx_err(bp))
1125                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1126         }
1127         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1128         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1129         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1130
1131         rxr->rx_prod = NEXT_RX(prod);
1132         cons = NEXT_RX(cons);
1133         rxr->rx_next_cons = NEXT_RX(cons);
1134         cons_rx_buf = &rxr->rx_buf_ring[cons];
1135
1136         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1137         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1138         cons_rx_buf->data = NULL;
1139 }
1140
1141 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
1142                            u16 cp_cons, u32 agg_bufs)
1143 {
1144         if (agg_bufs)
1145                 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1146 }
1147
1148 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1149                                            int payload_off, int tcp_ts,
1150                                            struct sk_buff *skb)
1151 {
1152 #ifdef CONFIG_INET
1153         struct tcphdr *th;
1154         int len, nw_off;
1155         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1156         u32 hdr_info = tpa_info->hdr_info;
1157         bool loopback = false;
1158
1159         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1160         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1161         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1162
1163         /* If the packet is an internal loopback packet, the offsets will
1164          * have an extra 4 bytes.
1165          */
1166         if (inner_mac_off == 4) {
1167                 loopback = true;
1168         } else if (inner_mac_off > 4) {
1169                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1170                                             ETH_HLEN - 2));
1171
1172                 /* We only support inner iPv4/ipv6.  If we don't see the
1173                  * correct protocol ID, it must be a loopback packet where
1174                  * the offsets are off by 4.
1175                  */
1176                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1177                         loopback = true;
1178         }
1179         if (loopback) {
1180                 /* internal loopback packet, subtract all offsets by 4 */
1181                 inner_ip_off -= 4;
1182                 inner_mac_off -= 4;
1183                 outer_ip_off -= 4;
1184         }
1185
1186         nw_off = inner_ip_off - ETH_HLEN;
1187         skb_set_network_header(skb, nw_off);
1188         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1189                 struct ipv6hdr *iph = ipv6_hdr(skb);
1190
1191                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1192                 len = skb->len - skb_transport_offset(skb);
1193                 th = tcp_hdr(skb);
1194                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1195         } else {
1196                 struct iphdr *iph = ip_hdr(skb);
1197
1198                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1199                 len = skb->len - skb_transport_offset(skb);
1200                 th = tcp_hdr(skb);
1201                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1202         }
1203
1204         if (inner_mac_off) { /* tunnel */
1205                 struct udphdr *uh = NULL;
1206                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1207                                             ETH_HLEN - 2));
1208
1209                 if (proto == htons(ETH_P_IP)) {
1210                         struct iphdr *iph = (struct iphdr *)skb->data;
1211
1212                         if (iph->protocol == IPPROTO_UDP)
1213                                 uh = (struct udphdr *)(iph + 1);
1214                 } else {
1215                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1216
1217                         if (iph->nexthdr == IPPROTO_UDP)
1218                                 uh = (struct udphdr *)(iph + 1);
1219                 }
1220                 if (uh) {
1221                         if (uh->check)
1222                                 skb_shinfo(skb)->gso_type |=
1223                                         SKB_GSO_UDP_TUNNEL_CSUM;
1224                         else
1225                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1226                 }
1227         }
1228 #endif
1229         return skb;
1230 }
1231
1232 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1233 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1234
1235 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1236                                            int payload_off, int tcp_ts,
1237                                            struct sk_buff *skb)
1238 {
1239 #ifdef CONFIG_INET
1240         struct tcphdr *th;
1241         int len, nw_off, tcp_opt_len = 0;
1242
1243         if (tcp_ts)
1244                 tcp_opt_len = 12;
1245
1246         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1247                 struct iphdr *iph;
1248
1249                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1250                          ETH_HLEN;
1251                 skb_set_network_header(skb, nw_off);
1252                 iph = ip_hdr(skb);
1253                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1254                 len = skb->len - skb_transport_offset(skb);
1255                 th = tcp_hdr(skb);
1256                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1257         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1258                 struct ipv6hdr *iph;
1259
1260                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1261                          ETH_HLEN;
1262                 skb_set_network_header(skb, nw_off);
1263                 iph = ipv6_hdr(skb);
1264                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1265                 len = skb->len - skb_transport_offset(skb);
1266                 th = tcp_hdr(skb);
1267                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1268         } else {
1269                 dev_kfree_skb_any(skb);
1270                 return NULL;
1271         }
1272
1273         if (nw_off) { /* tunnel */
1274                 struct udphdr *uh = NULL;
1275
1276                 if (skb->protocol == htons(ETH_P_IP)) {
1277                         struct iphdr *iph = (struct iphdr *)skb->data;
1278
1279                         if (iph->protocol == IPPROTO_UDP)
1280                                 uh = (struct udphdr *)(iph + 1);
1281                 } else {
1282                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1283
1284                         if (iph->nexthdr == IPPROTO_UDP)
1285                                 uh = (struct udphdr *)(iph + 1);
1286                 }
1287                 if (uh) {
1288                         if (uh->check)
1289                                 skb_shinfo(skb)->gso_type |=
1290                                         SKB_GSO_UDP_TUNNEL_CSUM;
1291                         else
1292                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1293                 }
1294         }
1295 #endif
1296         return skb;
1297 }
1298
1299 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1300                                            struct bnxt_tpa_info *tpa_info,
1301                                            struct rx_tpa_end_cmp *tpa_end,
1302                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1303                                            struct sk_buff *skb)
1304 {
1305 #ifdef CONFIG_INET
1306         int payload_off;
1307         u16 segs;
1308
1309         segs = TPA_END_TPA_SEGS(tpa_end);
1310         if (segs == 1)
1311                 return skb;
1312
1313         NAPI_GRO_CB(skb)->count = segs;
1314         skb_shinfo(skb)->gso_size =
1315                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1316         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1317         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1318                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1319                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1320         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1321         if (likely(skb))
1322                 tcp_gro_complete(skb);
1323 #endif
1324         return skb;
1325 }
1326
1327 /* Given the cfa_code of a received packet determine which
1328  * netdev (vf-rep or PF) the packet is destined to.
1329  */
1330 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1331 {
1332         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1333
1334         /* if vf-rep dev is NULL, the must belongs to the PF */
1335         return dev ? dev : bp->dev;
1336 }
1337
1338 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1339                                            struct bnxt_napi *bnapi,
1340                                            u32 *raw_cons,
1341                                            struct rx_tpa_end_cmp *tpa_end,
1342                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1343                                            u8 *event)
1344 {
1345         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1346         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1347         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1348         u8 *data_ptr, agg_bufs;
1349         u16 cp_cons = RING_CMP(*raw_cons);
1350         unsigned int len;
1351         struct bnxt_tpa_info *tpa_info;
1352         dma_addr_t mapping;
1353         struct sk_buff *skb;
1354         void *data;
1355
1356         if (unlikely(bnapi->in_reset)) {
1357                 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1358
1359                 if (rc < 0)
1360                         return ERR_PTR(-EBUSY);
1361                 return NULL;
1362         }
1363
1364         tpa_info = &rxr->rx_tpa[agg_id];
1365         data = tpa_info->data;
1366         data_ptr = tpa_info->data_ptr;
1367         prefetch(data_ptr);
1368         len = tpa_info->len;
1369         mapping = tpa_info->mapping;
1370
1371         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1372                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1373
1374         if (agg_bufs) {
1375                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1376                         return ERR_PTR(-EBUSY);
1377
1378                 *event |= BNXT_AGG_EVENT;
1379                 cp_cons = NEXT_CMP(cp_cons);
1380         }
1381
1382         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1383                 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1384                 if (agg_bufs > MAX_SKB_FRAGS)
1385                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1386                                     agg_bufs, (int)MAX_SKB_FRAGS);
1387                 return NULL;
1388         }
1389
1390         if (len <= bp->rx_copy_thresh) {
1391                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1392                 if (!skb) {
1393                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1394                         return NULL;
1395                 }
1396         } else {
1397                 u8 *new_data;
1398                 dma_addr_t new_mapping;
1399
1400                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1401                 if (!new_data) {
1402                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1403                         return NULL;
1404                 }
1405
1406                 tpa_info->data = new_data;
1407                 tpa_info->data_ptr = new_data + bp->rx_offset;
1408                 tpa_info->mapping = new_mapping;
1409
1410                 skb = build_skb(data, 0);
1411                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1412                                        bp->rx_buf_use_size, bp->rx_dir,
1413                                        DMA_ATTR_WEAK_ORDERING);
1414
1415                 if (!skb) {
1416                         kfree(data);
1417                         bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1418                         return NULL;
1419                 }
1420                 skb_reserve(skb, bp->rx_offset);
1421                 skb_put(skb, len);
1422         }
1423
1424         if (agg_bufs) {
1425                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1426                 if (!skb) {
1427                         /* Page reuse already handled by bnxt_rx_pages(). */
1428                         return NULL;
1429                 }
1430         }
1431
1432         skb->protocol =
1433                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1434
1435         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1436                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1437
1438         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1439             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1440                 u16 vlan_proto = tpa_info->metadata >>
1441                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1442                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1443
1444                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1445         }
1446
1447         skb_checksum_none_assert(skb);
1448         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1449                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1450                 skb->csum_level =
1451                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1452         }
1453
1454         if (TPA_END_GRO(tpa_end))
1455                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1456
1457         return skb;
1458 }
1459
1460 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1461                              struct sk_buff *skb)
1462 {
1463         if (skb->dev != bp->dev) {
1464                 /* this packet belongs to a vf-rep */
1465                 bnxt_vf_rep_rx(bp, skb);
1466                 return;
1467         }
1468         skb_record_rx_queue(skb, bnapi->index);
1469         napi_gro_receive(&bnapi->napi, skb);
1470 }
1471
1472 /* returns the following:
1473  * 1       - 1 packet successfully received
1474  * 0       - successful TPA_START, packet not completed yet
1475  * -EBUSY  - completion ring does not have all the agg buffers yet
1476  * -ENOMEM - packet aborted due to out of memory
1477  * -EIO    - packet aborted due to hw error indicated in BD
1478  */
1479 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1480                        u8 *event)
1481 {
1482         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1483         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1484         struct net_device *dev = bp->dev;
1485         struct rx_cmp *rxcmp;
1486         struct rx_cmp_ext *rxcmp1;
1487         u32 tmp_raw_cons = *raw_cons;
1488         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1489         struct bnxt_sw_rx_bd *rx_buf;
1490         unsigned int len;
1491         u8 *data_ptr, agg_bufs, cmp_type;
1492         dma_addr_t dma_addr;
1493         struct sk_buff *skb;
1494         void *data;
1495         int rc = 0;
1496         u32 misc;
1497
1498         rxcmp = (struct rx_cmp *)
1499                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1500
1501         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1502         cp_cons = RING_CMP(tmp_raw_cons);
1503         rxcmp1 = (struct rx_cmp_ext *)
1504                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1505
1506         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1507                 return -EBUSY;
1508
1509         cmp_type = RX_CMP_TYPE(rxcmp);
1510
1511         prod = rxr->rx_prod;
1512
1513         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1514                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1515                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1516
1517                 *event |= BNXT_RX_EVENT;
1518                 goto next_rx_no_prod_no_len;
1519
1520         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1521                 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1522                                    (struct rx_tpa_end_cmp *)rxcmp,
1523                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1524
1525                 if (IS_ERR(skb))
1526                         return -EBUSY;
1527
1528                 rc = -ENOMEM;
1529                 if (likely(skb)) {
1530                         bnxt_deliver_skb(bp, bnapi, skb);
1531                         rc = 1;
1532                 }
1533                 *event |= BNXT_RX_EVENT;
1534                 goto next_rx_no_prod_no_len;
1535         }
1536
1537         cons = rxcmp->rx_cmp_opaque;
1538         rx_buf = &rxr->rx_buf_ring[cons];
1539         data = rx_buf->data;
1540         data_ptr = rx_buf->data_ptr;
1541         if (unlikely(cons != rxr->rx_next_cons)) {
1542                 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1543
1544                 bnxt_sched_reset(bp, rxr);
1545                 return rc1;
1546         }
1547         prefetch(data_ptr);
1548
1549         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1550         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1551
1552         if (agg_bufs) {
1553                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1554                         return -EBUSY;
1555
1556                 cp_cons = NEXT_CMP(cp_cons);
1557                 *event |= BNXT_AGG_EVENT;
1558         }
1559         *event |= BNXT_RX_EVENT;
1560
1561         rx_buf->data = NULL;
1562         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1563                 bnxt_reuse_rx_data(rxr, cons, data);
1564                 if (agg_bufs)
1565                         bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1566
1567                 rc = -EIO;
1568                 goto next_rx;
1569         }
1570
1571         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1572         dma_addr = rx_buf->mapping;
1573
1574         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1575                 rc = 1;
1576                 goto next_rx;
1577         }
1578
1579         if (len <= bp->rx_copy_thresh) {
1580                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1581                 bnxt_reuse_rx_data(rxr, cons, data);
1582                 if (!skb) {
1583                         rc = -ENOMEM;
1584                         goto next_rx;
1585                 }
1586         } else {
1587                 u32 payload;
1588
1589                 if (rx_buf->data_ptr == data_ptr)
1590                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1591                 else
1592                         payload = 0;
1593                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1594                                       payload | len);
1595                 if (!skb) {
1596                         rc = -ENOMEM;
1597                         goto next_rx;
1598                 }
1599         }
1600
1601         if (agg_bufs) {
1602                 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1603                 if (!skb) {
1604                         rc = -ENOMEM;
1605                         goto next_rx;
1606                 }
1607         }
1608
1609         if (RX_CMP_HASH_VALID(rxcmp)) {
1610                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1611                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1612
1613                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1614                 if (hash_type != 1 && hash_type != 3)
1615                         type = PKT_HASH_TYPE_L3;
1616                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1617         }
1618
1619         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1620         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1621
1622         if ((rxcmp1->rx_cmp_flags2 &
1623              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1624             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1625                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1626                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1627                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1628
1629                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1630         }
1631
1632         skb_checksum_none_assert(skb);
1633         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1634                 if (dev->features & NETIF_F_RXCSUM) {
1635                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1636                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1637                 }
1638         } else {
1639                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1640                         if (dev->features & NETIF_F_RXCSUM)
1641                                 cpr->rx_l4_csum_errors++;
1642                 }
1643         }
1644
1645         bnxt_deliver_skb(bp, bnapi, skb);
1646         rc = 1;
1647
1648 next_rx:
1649         rxr->rx_prod = NEXT_RX(prod);
1650         rxr->rx_next_cons = NEXT_RX(cons);
1651
1652         cpr->rx_packets += 1;
1653         cpr->rx_bytes += len;
1654
1655 next_rx_no_prod_no_len:
1656         *raw_cons = tmp_raw_cons;
1657
1658         return rc;
1659 }
1660
1661 /* In netpoll mode, if we are using a combined completion ring, we need to
1662  * discard the rx packets and recycle the buffers.
1663  */
1664 static int bnxt_force_rx_discard(struct bnxt *bp, struct bnxt_napi *bnapi,
1665                                  u32 *raw_cons, u8 *event)
1666 {
1667         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1668         u32 tmp_raw_cons = *raw_cons;
1669         struct rx_cmp_ext *rxcmp1;
1670         struct rx_cmp *rxcmp;
1671         u16 cp_cons;
1672         u8 cmp_type;
1673
1674         cp_cons = RING_CMP(tmp_raw_cons);
1675         rxcmp = (struct rx_cmp *)
1676                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1677
1678         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1679         cp_cons = RING_CMP(tmp_raw_cons);
1680         rxcmp1 = (struct rx_cmp_ext *)
1681                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1682
1683         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1684                 return -EBUSY;
1685
1686         cmp_type = RX_CMP_TYPE(rxcmp);
1687         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1688                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1689                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1690         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1691                 struct rx_tpa_end_cmp_ext *tpa_end1;
1692
1693                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1694                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1695                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1696         }
1697         return bnxt_rx_pkt(bp, bnapi, raw_cons, event);
1698 }
1699
1700 #define BNXT_GET_EVENT_PORT(data)       \
1701         ((data) &                       \
1702          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1703
1704 static int bnxt_async_event_process(struct bnxt *bp,
1705                                     struct hwrm_async_event_cmpl *cmpl)
1706 {
1707         u16 event_id = le16_to_cpu(cmpl->event_id);
1708
1709         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1710         switch (event_id) {
1711         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1712                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1713                 struct bnxt_link_info *link_info = &bp->link_info;
1714
1715                 if (BNXT_VF(bp))
1716                         goto async_event_process_exit;
1717
1718                 /* print unsupported speed warning in forced speed mode only */
1719                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1720                     (data1 & 0x20000)) {
1721                         u16 fw_speed = link_info->force_link_speed;
1722                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1723
1724                         if (speed != SPEED_UNKNOWN)
1725                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1726                                             speed);
1727                 }
1728                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1729                 /* fall thru */
1730         }
1731         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1732                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1733                 break;
1734         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1735                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1736                 break;
1737         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1738                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1739                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1740
1741                 if (BNXT_VF(bp))
1742                         break;
1743
1744                 if (bp->pf.port_id != port_id)
1745                         break;
1746
1747                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1748                 break;
1749         }
1750         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1751                 if (BNXT_PF(bp))
1752                         goto async_event_process_exit;
1753                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1754                 break;
1755         default:
1756                 goto async_event_process_exit;
1757         }
1758         bnxt_queue_sp_work(bp);
1759 async_event_process_exit:
1760         bnxt_ulp_async_events(bp, cmpl);
1761         return 0;
1762 }
1763
1764 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1765 {
1766         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1767         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1768         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1769                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1770
1771         switch (cmpl_type) {
1772         case CMPL_BASE_TYPE_HWRM_DONE:
1773                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1774                 if (seq_id == bp->hwrm_intr_seq_id)
1775                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1776                 else
1777                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1778                 break;
1779
1780         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1781                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1782
1783                 if ((vf_id < bp->pf.first_vf_id) ||
1784                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1785                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1786                                    vf_id);
1787                         return -EINVAL;
1788                 }
1789
1790                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1791                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1792                 bnxt_queue_sp_work(bp);
1793                 break;
1794
1795         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1796                 bnxt_async_event_process(bp,
1797                                          (struct hwrm_async_event_cmpl *)txcmp);
1798
1799         default:
1800                 break;
1801         }
1802
1803         return 0;
1804 }
1805
1806 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1807 {
1808         struct bnxt_napi *bnapi = dev_instance;
1809         struct bnxt *bp = bnapi->bp;
1810         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1811         u32 cons = RING_CMP(cpr->cp_raw_cons);
1812
1813         cpr->event_ctr++;
1814         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1815         napi_schedule(&bnapi->napi);
1816         return IRQ_HANDLED;
1817 }
1818
1819 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1820 {
1821         u32 raw_cons = cpr->cp_raw_cons;
1822         u16 cons = RING_CMP(raw_cons);
1823         struct tx_cmp *txcmp;
1824
1825         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1826
1827         return TX_CMP_VALID(txcmp, raw_cons);
1828 }
1829
1830 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1831 {
1832         struct bnxt_napi *bnapi = dev_instance;
1833         struct bnxt *bp = bnapi->bp;
1834         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1835         u32 cons = RING_CMP(cpr->cp_raw_cons);
1836         u32 int_status;
1837
1838         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1839
1840         if (!bnxt_has_work(bp, cpr)) {
1841                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1842                 /* return if erroneous interrupt */
1843                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1844                         return IRQ_NONE;
1845         }
1846
1847         /* disable ring IRQ */
1848         BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1849
1850         /* Return here if interrupt is shared and is disabled. */
1851         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1852                 return IRQ_HANDLED;
1853
1854         napi_schedule(&bnapi->napi);
1855         return IRQ_HANDLED;
1856 }
1857
1858 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1859 {
1860         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1861         u32 raw_cons = cpr->cp_raw_cons;
1862         u32 cons;
1863         int tx_pkts = 0;
1864         int rx_pkts = 0;
1865         u8 event = 0;
1866         struct tx_cmp *txcmp;
1867
1868         while (1) {
1869                 int rc;
1870
1871                 cons = RING_CMP(raw_cons);
1872                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1873
1874                 if (!TX_CMP_VALID(txcmp, raw_cons))
1875                         break;
1876
1877                 /* The valid test of the entry must be done first before
1878                  * reading any further.
1879                  */
1880                 dma_rmb();
1881                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1882                         tx_pkts++;
1883                         /* return full budget so NAPI will complete. */
1884                         if (unlikely(tx_pkts > bp->tx_wake_thresh))
1885                                 rx_pkts = budget;
1886                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1887                         if (likely(budget))
1888                                 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1889                         else
1890                                 rc = bnxt_force_rx_discard(bp, bnapi, &raw_cons,
1891                                                            &event);
1892                         if (likely(rc >= 0))
1893                                 rx_pkts += rc;
1894                         /* Increment rx_pkts when rc is -ENOMEM to count towards
1895                          * the NAPI budget.  Otherwise, we may potentially loop
1896                          * here forever if we consistently cannot allocate
1897                          * buffers.
1898                          */
1899                         else if (rc == -ENOMEM && budget)
1900                                 rx_pkts++;
1901                         else if (rc == -EBUSY)  /* partial completion */
1902                                 break;
1903                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1904                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1905                                     (TX_CMP_TYPE(txcmp) ==
1906                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1907                                     (TX_CMP_TYPE(txcmp) ==
1908                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1909                         bnxt_hwrm_handler(bp, txcmp);
1910                 }
1911                 raw_cons = NEXT_RAW_CMP(raw_cons);
1912
1913                 if (rx_pkts == budget)
1914                         break;
1915         }
1916
1917         if (event & BNXT_TX_EVENT) {
1918                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1919                 void __iomem *db = txr->tx_doorbell;
1920                 u16 prod = txr->tx_prod;
1921
1922                 /* Sync BD data before updating doorbell */
1923                 wmb();
1924
1925                 bnxt_db_write_relaxed(bp, db, DB_KEY_TX | prod);
1926         }
1927
1928         cpr->cp_raw_cons = raw_cons;
1929         /* ACK completion ring before freeing tx ring and producing new
1930          * buffers in rx/agg rings to prevent overflowing the completion
1931          * ring.
1932          */
1933         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1934
1935         if (tx_pkts)
1936                 bnapi->tx_int(bp, bnapi, tx_pkts);
1937
1938         if (event & BNXT_RX_EVENT) {
1939                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1940
1941                 bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
1942                 if (event & BNXT_AGG_EVENT)
1943                         bnxt_db_write(bp, rxr->rx_agg_doorbell,
1944                                       DB_KEY_RX | rxr->rx_agg_prod);
1945         }
1946         return rx_pkts;
1947 }
1948
1949 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
1950 {
1951         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1952         struct bnxt *bp = bnapi->bp;
1953         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1954         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1955         struct tx_cmp *txcmp;
1956         struct rx_cmp_ext *rxcmp1;
1957         u32 cp_cons, tmp_raw_cons;
1958         u32 raw_cons = cpr->cp_raw_cons;
1959         u32 rx_pkts = 0;
1960         u8 event = 0;
1961
1962         while (1) {
1963                 int rc;
1964
1965                 cp_cons = RING_CMP(raw_cons);
1966                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1967
1968                 if (!TX_CMP_VALID(txcmp, raw_cons))
1969                         break;
1970
1971                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1972                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
1973                         cp_cons = RING_CMP(tmp_raw_cons);
1974                         rxcmp1 = (struct rx_cmp_ext *)
1975                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1976
1977                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1978                                 break;
1979
1980                         /* force an error to recycle the buffer */
1981                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1982                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1983
1984                         rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &event);
1985                         if (likely(rc == -EIO) && budget)
1986                                 rx_pkts++;
1987                         else if (rc == -EBUSY)  /* partial completion */
1988                                 break;
1989                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
1990                                     CMPL_BASE_TYPE_HWRM_DONE)) {
1991                         bnxt_hwrm_handler(bp, txcmp);
1992                 } else {
1993                         netdev_err(bp->dev,
1994                                    "Invalid completion received on special ring\n");
1995                 }
1996                 raw_cons = NEXT_RAW_CMP(raw_cons);
1997
1998                 if (rx_pkts == budget)
1999                         break;
2000         }
2001
2002         cpr->cp_raw_cons = raw_cons;
2003         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
2004         bnxt_db_write(bp, rxr->rx_doorbell, DB_KEY_RX | rxr->rx_prod);
2005
2006         if (event & BNXT_AGG_EVENT)
2007                 bnxt_db_write(bp, rxr->rx_agg_doorbell,
2008                               DB_KEY_RX | rxr->rx_agg_prod);
2009
2010         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2011                 napi_complete_done(napi, rx_pkts);
2012                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
2013         }
2014         return rx_pkts;
2015 }
2016
2017 static int bnxt_poll(struct napi_struct *napi, int budget)
2018 {
2019         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2020         struct bnxt *bp = bnapi->bp;
2021         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2022         int work_done = 0;
2023
2024         while (1) {
2025                 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
2026
2027                 if (work_done >= budget)
2028                         break;
2029
2030                 if (!bnxt_has_work(bp, cpr)) {
2031                         if (napi_complete_done(napi, work_done))
2032                                 BNXT_CP_DB_REARM(cpr->cp_doorbell,
2033                                                  cpr->cp_raw_cons);
2034                         break;
2035                 }
2036         }
2037         if (bp->flags & BNXT_FLAG_DIM) {
2038                 struct net_dim_sample dim_sample;
2039
2040                 net_dim_sample(cpr->event_ctr,
2041                                cpr->rx_packets,
2042                                cpr->rx_bytes,
2043                                &dim_sample);
2044                 net_dim(&cpr->dim, dim_sample);
2045         }
2046         mmiowb();
2047         return work_done;
2048 }
2049
2050 static void bnxt_free_tx_skbs(struct bnxt *bp)
2051 {
2052         int i, max_idx;
2053         struct pci_dev *pdev = bp->pdev;
2054
2055         if (!bp->tx_ring)
2056                 return;
2057
2058         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2059         for (i = 0; i < bp->tx_nr_rings; i++) {
2060                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2061                 int j;
2062
2063                 for (j = 0; j < max_idx;) {
2064                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2065                         struct sk_buff *skb = tx_buf->skb;
2066                         int k, last;
2067
2068                         if (!skb) {
2069                                 j++;
2070                                 continue;
2071                         }
2072
2073                         tx_buf->skb = NULL;
2074
2075                         if (tx_buf->is_push) {
2076                                 dev_kfree_skb(skb);
2077                                 j += 2;
2078                                 continue;
2079                         }
2080
2081                         dma_unmap_single(&pdev->dev,
2082                                          dma_unmap_addr(tx_buf, mapping),
2083                                          skb_headlen(skb),
2084                                          PCI_DMA_TODEVICE);
2085
2086                         last = tx_buf->nr_frags;
2087                         j += 2;
2088                         for (k = 0; k < last; k++, j++) {
2089                                 int ring_idx = j & bp->tx_ring_mask;
2090                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2091
2092                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2093                                 dma_unmap_page(
2094                                         &pdev->dev,
2095                                         dma_unmap_addr(tx_buf, mapping),
2096                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2097                         }
2098                         dev_kfree_skb(skb);
2099                 }
2100                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2101         }
2102 }
2103
2104 static void bnxt_free_rx_skbs(struct bnxt *bp)
2105 {
2106         int i, max_idx, max_agg_idx;
2107         struct pci_dev *pdev = bp->pdev;
2108
2109         if (!bp->rx_ring)
2110                 return;
2111
2112         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2113         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2114         for (i = 0; i < bp->rx_nr_rings; i++) {
2115                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2116                 int j;
2117
2118                 if (rxr->rx_tpa) {
2119                         for (j = 0; j < MAX_TPA; j++) {
2120                                 struct bnxt_tpa_info *tpa_info =
2121                                                         &rxr->rx_tpa[j];
2122                                 u8 *data = tpa_info->data;
2123
2124                                 if (!data)
2125                                         continue;
2126
2127                                 dma_unmap_single_attrs(&pdev->dev,
2128                                                        tpa_info->mapping,
2129                                                        bp->rx_buf_use_size,
2130                                                        bp->rx_dir,
2131                                                        DMA_ATTR_WEAK_ORDERING);
2132
2133                                 tpa_info->data = NULL;
2134
2135                                 kfree(data);
2136                         }
2137                 }
2138
2139                 for (j = 0; j < max_idx; j++) {
2140                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2141                         dma_addr_t mapping = rx_buf->mapping;
2142                         void *data = rx_buf->data;
2143
2144                         if (!data)
2145                                 continue;
2146
2147                         rx_buf->data = NULL;
2148
2149                         if (BNXT_RX_PAGE_MODE(bp)) {
2150                                 mapping -= bp->rx_dma_offset;
2151                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2152                                                      PAGE_SIZE, bp->rx_dir,
2153                                                      DMA_ATTR_WEAK_ORDERING);
2154                                 __free_page(data);
2155                         } else {
2156                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2157                                                        bp->rx_buf_use_size,
2158                                                        bp->rx_dir,
2159                                                        DMA_ATTR_WEAK_ORDERING);
2160                                 kfree(data);
2161                         }
2162                 }
2163
2164                 for (j = 0; j < max_agg_idx; j++) {
2165                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2166                                 &rxr->rx_agg_ring[j];
2167                         struct page *page = rx_agg_buf->page;
2168
2169                         if (!page)
2170                                 continue;
2171
2172                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2173                                              BNXT_RX_PAGE_SIZE,
2174                                              PCI_DMA_FROMDEVICE,
2175                                              DMA_ATTR_WEAK_ORDERING);
2176
2177                         rx_agg_buf->page = NULL;
2178                         __clear_bit(j, rxr->rx_agg_bmap);
2179
2180                         __free_page(page);
2181                 }
2182                 if (rxr->rx_page) {
2183                         __free_page(rxr->rx_page);
2184                         rxr->rx_page = NULL;
2185                 }
2186         }
2187 }
2188
2189 static void bnxt_free_skbs(struct bnxt *bp)
2190 {
2191         bnxt_free_tx_skbs(bp);
2192         bnxt_free_rx_skbs(bp);
2193 }
2194
2195 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2196 {
2197         struct pci_dev *pdev = bp->pdev;
2198         int i;
2199
2200         for (i = 0; i < ring->nr_pages; i++) {
2201                 if (!ring->pg_arr[i])
2202                         continue;
2203
2204                 dma_free_coherent(&pdev->dev, ring->page_size,
2205                                   ring->pg_arr[i], ring->dma_arr[i]);
2206
2207                 ring->pg_arr[i] = NULL;
2208         }
2209         if (ring->pg_tbl) {
2210                 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
2211                                   ring->pg_tbl, ring->pg_tbl_map);
2212                 ring->pg_tbl = NULL;
2213         }
2214         if (ring->vmem_size && *ring->vmem) {
2215                 vfree(*ring->vmem);
2216                 *ring->vmem = NULL;
2217         }
2218 }
2219
2220 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
2221 {
2222         int i;
2223         struct pci_dev *pdev = bp->pdev;
2224
2225         if (ring->nr_pages > 1) {
2226                 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
2227                                                   ring->nr_pages * 8,
2228                                                   &ring->pg_tbl_map,
2229                                                   GFP_KERNEL);
2230                 if (!ring->pg_tbl)
2231                         return -ENOMEM;
2232         }
2233
2234         for (i = 0; i < ring->nr_pages; i++) {
2235                 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2236                                                      ring->page_size,
2237                                                      &ring->dma_arr[i],
2238                                                      GFP_KERNEL);
2239                 if (!ring->pg_arr[i])
2240                         return -ENOMEM;
2241
2242                 if (ring->nr_pages > 1)
2243                         ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
2244         }
2245
2246         if (ring->vmem_size) {
2247                 *ring->vmem = vzalloc(ring->vmem_size);
2248                 if (!(*ring->vmem))
2249                         return -ENOMEM;
2250         }
2251         return 0;
2252 }
2253
2254 static void bnxt_free_rx_rings(struct bnxt *bp)
2255 {
2256         int i;
2257
2258         if (!bp->rx_ring)
2259                 return;
2260
2261         for (i = 0; i < bp->rx_nr_rings; i++) {
2262                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2263                 struct bnxt_ring_struct *ring;
2264
2265                 if (rxr->xdp_prog)
2266                         bpf_prog_put(rxr->xdp_prog);
2267
2268                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2269                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2270
2271                 kfree(rxr->rx_tpa);
2272                 rxr->rx_tpa = NULL;
2273
2274                 kfree(rxr->rx_agg_bmap);
2275                 rxr->rx_agg_bmap = NULL;
2276
2277                 ring = &rxr->rx_ring_struct;
2278                 bnxt_free_ring(bp, ring);
2279
2280                 ring = &rxr->rx_agg_ring_struct;
2281                 bnxt_free_ring(bp, ring);
2282         }
2283 }
2284
2285 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2286 {
2287         int i, rc, agg_rings = 0, tpa_rings = 0;
2288
2289         if (!bp->rx_ring)
2290                 return -ENOMEM;
2291
2292         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2293                 agg_rings = 1;
2294
2295         if (bp->flags & BNXT_FLAG_TPA)
2296                 tpa_rings = 1;
2297
2298         for (i = 0; i < bp->rx_nr_rings; i++) {
2299                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2300                 struct bnxt_ring_struct *ring;
2301
2302                 ring = &rxr->rx_ring_struct;
2303
2304                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2305                 if (rc < 0)
2306                         return rc;
2307
2308                 rc = bnxt_alloc_ring(bp, ring);
2309                 if (rc)
2310                         return rc;
2311
2312                 if (agg_rings) {
2313                         u16 mem_size;
2314
2315                         ring = &rxr->rx_agg_ring_struct;
2316                         rc = bnxt_alloc_ring(bp, ring);
2317                         if (rc)
2318                                 return rc;
2319
2320                         ring->grp_idx = i;
2321                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2322                         mem_size = rxr->rx_agg_bmap_size / 8;
2323                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2324                         if (!rxr->rx_agg_bmap)
2325                                 return -ENOMEM;
2326
2327                         if (tpa_rings) {
2328                                 rxr->rx_tpa = kcalloc(MAX_TPA,
2329                                                 sizeof(struct bnxt_tpa_info),
2330                                                 GFP_KERNEL);
2331                                 if (!rxr->rx_tpa)
2332                                         return -ENOMEM;
2333                         }
2334                 }
2335         }
2336         return 0;
2337 }
2338
2339 static void bnxt_free_tx_rings(struct bnxt *bp)
2340 {
2341         int i;
2342         struct pci_dev *pdev = bp->pdev;
2343
2344         if (!bp->tx_ring)
2345                 return;
2346
2347         for (i = 0; i < bp->tx_nr_rings; i++) {
2348                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2349                 struct bnxt_ring_struct *ring;
2350
2351                 if (txr->tx_push) {
2352                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2353                                           txr->tx_push, txr->tx_push_mapping);
2354                         txr->tx_push = NULL;
2355                 }
2356
2357                 ring = &txr->tx_ring_struct;
2358
2359                 bnxt_free_ring(bp, ring);
2360         }
2361 }
2362
2363 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2364 {
2365         int i, j, rc;
2366         struct pci_dev *pdev = bp->pdev;
2367
2368         bp->tx_push_size = 0;
2369         if (bp->tx_push_thresh) {
2370                 int push_size;
2371
2372                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2373                                         bp->tx_push_thresh);
2374
2375                 if (push_size > 256) {
2376                         push_size = 0;
2377                         bp->tx_push_thresh = 0;
2378                 }
2379
2380                 bp->tx_push_size = push_size;
2381         }
2382
2383         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2384                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2385                 struct bnxt_ring_struct *ring;
2386
2387                 ring = &txr->tx_ring_struct;
2388
2389                 rc = bnxt_alloc_ring(bp, ring);
2390                 if (rc)
2391                         return rc;
2392
2393                 ring->grp_idx = txr->bnapi->index;
2394                 if (bp->tx_push_size) {
2395                         dma_addr_t mapping;
2396
2397                         /* One pre-allocated DMA buffer to backup
2398                          * TX push operation
2399                          */
2400                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2401                                                 bp->tx_push_size,
2402                                                 &txr->tx_push_mapping,
2403                                                 GFP_KERNEL);
2404
2405                         if (!txr->tx_push)
2406                                 return -ENOMEM;
2407
2408                         mapping = txr->tx_push_mapping +
2409                                 sizeof(struct tx_push_bd);
2410                         txr->data_mapping = cpu_to_le64(mapping);
2411
2412                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
2413                 }
2414                 ring->queue_id = bp->q_info[j].queue_id;
2415                 if (i < bp->tx_nr_rings_xdp)
2416                         continue;
2417                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2418                         j++;
2419         }
2420         return 0;
2421 }
2422
2423 static void bnxt_free_cp_rings(struct bnxt *bp)
2424 {
2425         int i;
2426
2427         if (!bp->bnapi)
2428                 return;
2429
2430         for (i = 0; i < bp->cp_nr_rings; i++) {
2431                 struct bnxt_napi *bnapi = bp->bnapi[i];
2432                 struct bnxt_cp_ring_info *cpr;
2433                 struct bnxt_ring_struct *ring;
2434
2435                 if (!bnapi)
2436                         continue;
2437
2438                 cpr = &bnapi->cp_ring;
2439                 ring = &cpr->cp_ring_struct;
2440
2441                 bnxt_free_ring(bp, ring);
2442         }
2443 }
2444
2445 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2446 {
2447         int i, rc, ulp_base_vec, ulp_msix;
2448
2449         ulp_msix = bnxt_get_ulp_msix_num(bp);
2450         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
2451         for (i = 0; i < bp->cp_nr_rings; i++) {
2452                 struct bnxt_napi *bnapi = bp->bnapi[i];
2453                 struct bnxt_cp_ring_info *cpr;
2454                 struct bnxt_ring_struct *ring;
2455
2456                 if (!bnapi)
2457                         continue;
2458
2459                 cpr = &bnapi->cp_ring;
2460                 ring = &cpr->cp_ring_struct;
2461
2462                 rc = bnxt_alloc_ring(bp, ring);
2463                 if (rc)
2464                         return rc;
2465
2466                 if (ulp_msix && i >= ulp_base_vec)
2467                         ring->map_idx = i + ulp_msix;
2468                 else
2469                         ring->map_idx = i;
2470         }
2471         return 0;
2472 }
2473
2474 static void bnxt_init_ring_struct(struct bnxt *bp)
2475 {
2476         int i;
2477
2478         for (i = 0; i < bp->cp_nr_rings; i++) {
2479                 struct bnxt_napi *bnapi = bp->bnapi[i];
2480                 struct bnxt_cp_ring_info *cpr;
2481                 struct bnxt_rx_ring_info *rxr;
2482                 struct bnxt_tx_ring_info *txr;
2483                 struct bnxt_ring_struct *ring;
2484
2485                 if (!bnapi)
2486                         continue;
2487
2488                 cpr = &bnapi->cp_ring;
2489                 ring = &cpr->cp_ring_struct;
2490                 ring->nr_pages = bp->cp_nr_pages;
2491                 ring->page_size = HW_CMPD_RING_SIZE;
2492                 ring->pg_arr = (void **)cpr->cp_desc_ring;
2493                 ring->dma_arr = cpr->cp_desc_mapping;
2494                 ring->vmem_size = 0;
2495
2496                 rxr = bnapi->rx_ring;
2497                 if (!rxr)
2498                         goto skip_rx;
2499
2500                 ring = &rxr->rx_ring_struct;
2501                 ring->nr_pages = bp->rx_nr_pages;
2502                 ring->page_size = HW_RXBD_RING_SIZE;
2503                 ring->pg_arr = (void **)rxr->rx_desc_ring;
2504                 ring->dma_arr = rxr->rx_desc_mapping;
2505                 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2506                 ring->vmem = (void **)&rxr->rx_buf_ring;
2507
2508                 ring = &rxr->rx_agg_ring_struct;
2509                 ring->nr_pages = bp->rx_agg_nr_pages;
2510                 ring->page_size = HW_RXBD_RING_SIZE;
2511                 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2512                 ring->dma_arr = rxr->rx_agg_desc_mapping;
2513                 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2514                 ring->vmem = (void **)&rxr->rx_agg_ring;
2515
2516 skip_rx:
2517                 txr = bnapi->tx_ring;
2518                 if (!txr)
2519                         continue;
2520
2521                 ring = &txr->tx_ring_struct;
2522                 ring->nr_pages = bp->tx_nr_pages;
2523                 ring->page_size = HW_RXBD_RING_SIZE;
2524                 ring->pg_arr = (void **)txr->tx_desc_ring;
2525                 ring->dma_arr = txr->tx_desc_mapping;
2526                 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2527                 ring->vmem = (void **)&txr->tx_buf_ring;
2528         }
2529 }
2530
2531 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2532 {
2533         int i;
2534         u32 prod;
2535         struct rx_bd **rx_buf_ring;
2536
2537         rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2538         for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2539                 int j;
2540                 struct rx_bd *rxbd;
2541
2542                 rxbd = rx_buf_ring[i];
2543                 if (!rxbd)
2544                         continue;
2545
2546                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2547                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2548                         rxbd->rx_bd_opaque = prod;
2549                 }
2550         }
2551 }
2552
2553 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2554 {
2555         struct net_device *dev = bp->dev;
2556         struct bnxt_rx_ring_info *rxr;
2557         struct bnxt_ring_struct *ring;
2558         u32 prod, type;
2559         int i;
2560
2561         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2562                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2563
2564         if (NET_IP_ALIGN == 2)
2565                 type |= RX_BD_FLAGS_SOP;
2566
2567         rxr = &bp->rx_ring[ring_nr];
2568         ring = &rxr->rx_ring_struct;
2569         bnxt_init_rxbd_pages(ring, type);
2570
2571         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2572                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2573                 if (IS_ERR(rxr->xdp_prog)) {
2574                         int rc = PTR_ERR(rxr->xdp_prog);
2575
2576                         rxr->xdp_prog = NULL;
2577                         return rc;
2578                 }
2579         }
2580         prod = rxr->rx_prod;
2581         for (i = 0; i < bp->rx_ring_size; i++) {
2582                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2583                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2584                                     ring_nr, i, bp->rx_ring_size);
2585                         break;
2586                 }
2587                 prod = NEXT_RX(prod);
2588         }
2589         rxr->rx_prod = prod;
2590         ring->fw_ring_id = INVALID_HW_RING_ID;
2591
2592         ring = &rxr->rx_agg_ring_struct;
2593         ring->fw_ring_id = INVALID_HW_RING_ID;
2594
2595         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2596                 return 0;
2597
2598         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2599                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2600
2601         bnxt_init_rxbd_pages(ring, type);
2602
2603         prod = rxr->rx_agg_prod;
2604         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2605                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2606                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2607                                     ring_nr, i, bp->rx_ring_size);
2608                         break;
2609                 }
2610                 prod = NEXT_RX_AGG(prod);
2611         }
2612         rxr->rx_agg_prod = prod;
2613
2614         if (bp->flags & BNXT_FLAG_TPA) {
2615                 if (rxr->rx_tpa) {
2616                         u8 *data;
2617                         dma_addr_t mapping;
2618
2619                         for (i = 0; i < MAX_TPA; i++) {
2620                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2621                                                             GFP_KERNEL);
2622                                 if (!data)
2623                                         return -ENOMEM;
2624
2625                                 rxr->rx_tpa[i].data = data;
2626                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
2627                                 rxr->rx_tpa[i].mapping = mapping;
2628                         }
2629                 } else {
2630                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2631                         return -ENOMEM;
2632                 }
2633         }
2634
2635         return 0;
2636 }
2637
2638 static void bnxt_init_cp_rings(struct bnxt *bp)
2639 {
2640         int i;
2641
2642         for (i = 0; i < bp->cp_nr_rings; i++) {
2643                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2644                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2645
2646                 ring->fw_ring_id = INVALID_HW_RING_ID;
2647                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
2648                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
2649         }
2650 }
2651
2652 static int bnxt_init_rx_rings(struct bnxt *bp)
2653 {
2654         int i, rc = 0;
2655
2656         if (BNXT_RX_PAGE_MODE(bp)) {
2657                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2658                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
2659         } else {
2660                 bp->rx_offset = BNXT_RX_OFFSET;
2661                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2662         }
2663
2664         for (i = 0; i < bp->rx_nr_rings; i++) {
2665                 rc = bnxt_init_one_rx_ring(bp, i);
2666                 if (rc)
2667                         break;
2668         }
2669
2670         return rc;
2671 }
2672
2673 static int bnxt_init_tx_rings(struct bnxt *bp)
2674 {
2675         u16 i;
2676
2677         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2678                                    MAX_SKB_FRAGS + 1);
2679
2680         for (i = 0; i < bp->tx_nr_rings; i++) {
2681                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2682                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2683
2684                 ring->fw_ring_id = INVALID_HW_RING_ID;
2685         }
2686
2687         return 0;
2688 }
2689
2690 static void bnxt_free_ring_grps(struct bnxt *bp)
2691 {
2692         kfree(bp->grp_info);
2693         bp->grp_info = NULL;
2694 }
2695
2696 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2697 {
2698         int i;
2699
2700         if (irq_re_init) {
2701                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2702                                        sizeof(struct bnxt_ring_grp_info),
2703                                        GFP_KERNEL);
2704                 if (!bp->grp_info)
2705                         return -ENOMEM;
2706         }
2707         for (i = 0; i < bp->cp_nr_rings; i++) {
2708                 if (irq_re_init)
2709                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2710                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2711                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2712                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2713                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2714         }
2715         return 0;
2716 }
2717
2718 static void bnxt_free_vnics(struct bnxt *bp)
2719 {
2720         kfree(bp->vnic_info);
2721         bp->vnic_info = NULL;
2722         bp->nr_vnics = 0;
2723 }
2724
2725 static int bnxt_alloc_vnics(struct bnxt *bp)
2726 {
2727         int num_vnics = 1;
2728
2729 #ifdef CONFIG_RFS_ACCEL
2730         if (bp->flags & BNXT_FLAG_RFS)
2731                 num_vnics += bp->rx_nr_rings;
2732 #endif
2733
2734         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
2735                 num_vnics++;
2736
2737         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2738                                 GFP_KERNEL);
2739         if (!bp->vnic_info)
2740                 return -ENOMEM;
2741
2742         bp->nr_vnics = num_vnics;
2743         return 0;
2744 }
2745
2746 static void bnxt_init_vnics(struct bnxt *bp)
2747 {
2748         int i;
2749
2750         for (i = 0; i < bp->nr_vnics; i++) {
2751                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2752
2753                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2754                 vnic->fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
2755                 vnic->fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
2756                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2757
2758                 if (bp->vnic_info[i].rss_hash_key) {
2759                         if (i == 0)
2760                                 prandom_bytes(vnic->rss_hash_key,
2761                                               HW_HASH_KEY_SIZE);
2762                         else
2763                                 memcpy(vnic->rss_hash_key,
2764                                        bp->vnic_info[0].rss_hash_key,
2765                                        HW_HASH_KEY_SIZE);
2766                 }
2767         }
2768 }
2769
2770 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2771 {
2772         int pages;
2773
2774         pages = ring_size / desc_per_pg;
2775
2776         if (!pages)
2777                 return 1;
2778
2779         pages++;
2780
2781         while (pages & (pages - 1))
2782                 pages++;
2783
2784         return pages;
2785 }
2786
2787 void bnxt_set_tpa_flags(struct bnxt *bp)
2788 {
2789         bp->flags &= ~BNXT_FLAG_TPA;
2790         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
2791                 return;
2792         if (bp->dev->features & NETIF_F_LRO)
2793                 bp->flags |= BNXT_FLAG_LRO;
2794         else if (bp->dev->features & NETIF_F_GRO_HW)
2795                 bp->flags |= BNXT_FLAG_GRO;
2796 }
2797
2798 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2799  * be set on entry.
2800  */
2801 void bnxt_set_ring_params(struct bnxt *bp)
2802 {
2803         u32 ring_size, rx_size, rx_space;
2804         u32 agg_factor = 0, agg_ring_size = 0;
2805
2806         /* 8 for CRC and VLAN */
2807         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2808
2809         rx_space = rx_size + NET_SKB_PAD +
2810                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2811
2812         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2813         ring_size = bp->rx_ring_size;
2814         bp->rx_agg_ring_size = 0;
2815         bp->rx_agg_nr_pages = 0;
2816
2817         if (bp->flags & BNXT_FLAG_TPA)
2818                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
2819
2820         bp->flags &= ~BNXT_FLAG_JUMBO;
2821         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
2822                 u32 jumbo_factor;
2823
2824                 bp->flags |= BNXT_FLAG_JUMBO;
2825                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2826                 if (jumbo_factor > agg_factor)
2827                         agg_factor = jumbo_factor;
2828         }
2829         agg_ring_size = ring_size * agg_factor;
2830
2831         if (agg_ring_size) {
2832                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2833                                                         RX_DESC_CNT);
2834                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2835                         u32 tmp = agg_ring_size;
2836
2837                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2838                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2839                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2840                                     tmp, agg_ring_size);
2841                 }
2842                 bp->rx_agg_ring_size = agg_ring_size;
2843                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2844                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2845                 rx_space = rx_size + NET_SKB_PAD +
2846                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2847         }
2848
2849         bp->rx_buf_use_size = rx_size;
2850         bp->rx_buf_size = rx_space;
2851
2852         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2853         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2854
2855         ring_size = bp->tx_ring_size;
2856         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2857         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2858
2859         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2860         bp->cp_ring_size = ring_size;
2861
2862         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2863         if (bp->cp_nr_pages > MAX_CP_PAGES) {
2864                 bp->cp_nr_pages = MAX_CP_PAGES;
2865                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2866                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2867                             ring_size, bp->cp_ring_size);
2868         }
2869         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2870         bp->cp_ring_mask = bp->cp_bit - 1;
2871 }
2872
2873 /* Changing allocation mode of RX rings.
2874  * TODO: Update when extending xdp_rxq_info to support allocation modes.
2875  */
2876 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
2877 {
2878         if (page_mode) {
2879                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
2880                         return -EOPNOTSUPP;
2881                 bp->dev->max_mtu =
2882                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
2883                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
2884                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
2885                 bp->rx_dir = DMA_BIDIRECTIONAL;
2886                 bp->rx_skb_func = bnxt_rx_page_skb;
2887                 /* Disable LRO or GRO_HW */
2888                 netdev_update_features(bp->dev);
2889         } else {
2890                 bp->dev->max_mtu = bp->max_mtu;
2891                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
2892                 bp->rx_dir = DMA_FROM_DEVICE;
2893                 bp->rx_skb_func = bnxt_rx_skb;
2894         }
2895         return 0;
2896 }
2897
2898 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2899 {
2900         int i;
2901         struct bnxt_vnic_info *vnic;
2902         struct pci_dev *pdev = bp->pdev;
2903
2904         if (!bp->vnic_info)
2905                 return;
2906
2907         for (i = 0; i < bp->nr_vnics; i++) {
2908                 vnic = &bp->vnic_info[i];
2909
2910                 kfree(vnic->fw_grp_ids);
2911                 vnic->fw_grp_ids = NULL;
2912
2913                 kfree(vnic->uc_list);
2914                 vnic->uc_list = NULL;
2915
2916                 if (vnic->mc_list) {
2917                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2918                                           vnic->mc_list, vnic->mc_list_mapping);
2919                         vnic->mc_list = NULL;
2920                 }
2921
2922                 if (vnic->rss_table) {
2923                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
2924                                           vnic->rss_table,
2925                                           vnic->rss_table_dma_addr);
2926                         vnic->rss_table = NULL;
2927                 }
2928
2929                 vnic->rss_hash_key = NULL;
2930                 vnic->flags = 0;
2931         }
2932 }
2933
2934 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2935 {
2936         int i, rc = 0, size;
2937         struct bnxt_vnic_info *vnic;
2938         struct pci_dev *pdev = bp->pdev;
2939         int max_rings;
2940
2941         for (i = 0; i < bp->nr_vnics; i++) {
2942                 vnic = &bp->vnic_info[i];
2943
2944                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2945                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2946
2947                         if (mem_size > 0) {
2948                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2949                                 if (!vnic->uc_list) {
2950                                         rc = -ENOMEM;
2951                                         goto out;
2952                                 }
2953                         }
2954                 }
2955
2956                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2957                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2958                         vnic->mc_list =
2959                                 dma_alloc_coherent(&pdev->dev,
2960                                                    vnic->mc_list_size,
2961                                                    &vnic->mc_list_mapping,
2962                                                    GFP_KERNEL);
2963                         if (!vnic->mc_list) {
2964                                 rc = -ENOMEM;
2965                                 goto out;
2966                         }
2967                 }
2968
2969                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2970                         max_rings = bp->rx_nr_rings;
2971                 else
2972                         max_rings = 1;
2973
2974                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2975                 if (!vnic->fw_grp_ids) {
2976                         rc = -ENOMEM;
2977                         goto out;
2978                 }
2979
2980                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
2981                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
2982                         continue;
2983
2984                 /* Allocate rss table and hash key */
2985                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2986                                                      &vnic->rss_table_dma_addr,
2987                                                      GFP_KERNEL);
2988                 if (!vnic->rss_table) {
2989                         rc = -ENOMEM;
2990                         goto out;
2991                 }
2992
2993                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2994
2995                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2996                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2997         }
2998         return 0;
2999
3000 out:
3001         return rc;
3002 }
3003
3004 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3005 {
3006         struct pci_dev *pdev = bp->pdev;
3007
3008         dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3009                           bp->hwrm_cmd_resp_dma_addr);
3010
3011         bp->hwrm_cmd_resp_addr = NULL;
3012         if (bp->hwrm_dbg_resp_addr) {
3013                 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
3014                                   bp->hwrm_dbg_resp_addr,
3015                                   bp->hwrm_dbg_resp_dma_addr);
3016
3017                 bp->hwrm_dbg_resp_addr = NULL;
3018         }
3019 }
3020
3021 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3022 {
3023         struct pci_dev *pdev = bp->pdev;
3024
3025         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3026                                                    &bp->hwrm_cmd_resp_dma_addr,
3027                                                    GFP_KERNEL);
3028         if (!bp->hwrm_cmd_resp_addr)
3029                 return -ENOMEM;
3030         bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
3031                                                     HWRM_DBG_REG_BUF_SIZE,
3032                                                     &bp->hwrm_dbg_resp_dma_addr,
3033                                                     GFP_KERNEL);
3034         if (!bp->hwrm_dbg_resp_addr)
3035                 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
3036
3037         return 0;
3038 }
3039
3040 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3041 {
3042         if (bp->hwrm_short_cmd_req_addr) {
3043                 struct pci_dev *pdev = bp->pdev;
3044
3045                 dma_free_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3046                                   bp->hwrm_short_cmd_req_addr,
3047                                   bp->hwrm_short_cmd_req_dma_addr);
3048                 bp->hwrm_short_cmd_req_addr = NULL;
3049         }
3050 }
3051
3052 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3053 {
3054         struct pci_dev *pdev = bp->pdev;
3055
3056         bp->hwrm_short_cmd_req_addr =
3057                 dma_alloc_coherent(&pdev->dev, BNXT_HWRM_MAX_REQ_LEN,
3058                                    &bp->hwrm_short_cmd_req_dma_addr,
3059                                    GFP_KERNEL);
3060         if (!bp->hwrm_short_cmd_req_addr)
3061                 return -ENOMEM;
3062
3063         return 0;
3064 }
3065
3066 static void bnxt_free_stats(struct bnxt *bp)
3067 {
3068         u32 size, i;
3069         struct pci_dev *pdev = bp->pdev;
3070
3071         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3072         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3073
3074         if (bp->hw_rx_port_stats) {
3075                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3076                                   bp->hw_rx_port_stats,
3077                                   bp->hw_rx_port_stats_map);
3078                 bp->hw_rx_port_stats = NULL;
3079         }
3080
3081         if (bp->hw_rx_port_stats_ext) {
3082                 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3083                                   bp->hw_rx_port_stats_ext,
3084                                   bp->hw_rx_port_stats_ext_map);
3085                 bp->hw_rx_port_stats_ext = NULL;
3086         }
3087
3088         if (!bp->bnapi)
3089                 return;
3090
3091         size = sizeof(struct ctx_hw_stats);
3092
3093         for (i = 0; i < bp->cp_nr_rings; i++) {
3094                 struct bnxt_napi *bnapi = bp->bnapi[i];
3095                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3096
3097                 if (cpr->hw_stats) {
3098                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3099                                           cpr->hw_stats_map);
3100                         cpr->hw_stats = NULL;
3101                 }
3102         }
3103 }
3104
3105 static int bnxt_alloc_stats(struct bnxt *bp)
3106 {
3107         u32 size, i;
3108         struct pci_dev *pdev = bp->pdev;
3109
3110         size = sizeof(struct ctx_hw_stats);
3111
3112         for (i = 0; i < bp->cp_nr_rings; i++) {
3113                 struct bnxt_napi *bnapi = bp->bnapi[i];
3114                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3115
3116                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3117                                                    &cpr->hw_stats_map,
3118                                                    GFP_KERNEL);
3119                 if (!cpr->hw_stats)
3120                         return -ENOMEM;
3121
3122                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3123         }
3124
3125         if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3126                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3127                                          sizeof(struct tx_port_stats) + 1024;
3128
3129                 bp->hw_rx_port_stats =
3130                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3131                                            &bp->hw_rx_port_stats_map,
3132                                            GFP_KERNEL);
3133                 if (!bp->hw_rx_port_stats)
3134                         return -ENOMEM;
3135
3136                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3137                                        512;
3138                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3139                                            sizeof(struct rx_port_stats) + 512;
3140                 bp->flags |= BNXT_FLAG_PORT_STATS;
3141
3142                 /* Display extended statistics only if FW supports it */
3143                 if (bp->hwrm_spec_code < 0x10804 ||
3144                     bp->hwrm_spec_code == 0x10900)
3145                         return 0;
3146
3147                 bp->hw_rx_port_stats_ext =
3148                         dma_zalloc_coherent(&pdev->dev,
3149                                             sizeof(struct rx_port_stats_ext),
3150                                             &bp->hw_rx_port_stats_ext_map,
3151                                             GFP_KERNEL);
3152                 if (!bp->hw_rx_port_stats_ext)
3153                         return 0;
3154
3155                 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3156         }
3157         return 0;
3158 }
3159
3160 static void bnxt_clear_ring_indices(struct bnxt *bp)
3161 {
3162         int i;
3163
3164         if (!bp->bnapi)
3165                 return;
3166
3167         for (i = 0; i < bp->cp_nr_rings; i++) {
3168                 struct bnxt_napi *bnapi = bp->bnapi[i];
3169                 struct bnxt_cp_ring_info *cpr;
3170                 struct bnxt_rx_ring_info *rxr;
3171                 struct bnxt_tx_ring_info *txr;
3172
3173                 if (!bnapi)
3174                         continue;
3175
3176                 cpr = &bnapi->cp_ring;
3177                 cpr->cp_raw_cons = 0;
3178
3179                 txr = bnapi->tx_ring;
3180                 if (txr) {
3181                         txr->tx_prod = 0;
3182                         txr->tx_cons = 0;
3183                 }
3184
3185                 rxr = bnapi->rx_ring;
3186                 if (rxr) {
3187                         rxr->rx_prod = 0;
3188                         rxr->rx_agg_prod = 0;
3189                         rxr->rx_sw_agg_prod = 0;
3190                         rxr->rx_next_cons = 0;
3191                 }
3192         }
3193 }
3194
3195 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3196 {
3197 #ifdef CONFIG_RFS_ACCEL
3198         int i;
3199
3200         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3201          * safe to delete the hash table.
3202          */
3203         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3204                 struct hlist_head *head;
3205                 struct hlist_node *tmp;
3206                 struct bnxt_ntuple_filter *fltr;
3207
3208                 head = &bp->ntp_fltr_hash_tbl[i];
3209                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3210                         hlist_del(&fltr->hash);
3211                         kfree(fltr);
3212                 }
3213         }
3214         if (irq_reinit) {
3215                 kfree(bp->ntp_fltr_bmap);
3216                 bp->ntp_fltr_bmap = NULL;
3217         }
3218         bp->ntp_fltr_count = 0;
3219 #endif
3220 }
3221
3222 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3223 {
3224 #ifdef CONFIG_RFS_ACCEL
3225         int i, rc = 0;
3226
3227         if (!(bp->flags & BNXT_FLAG_RFS))
3228                 return 0;
3229
3230         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3231                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3232
3233         bp->ntp_fltr_count = 0;
3234         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3235                                     sizeof(long),
3236                                     GFP_KERNEL);
3237
3238         if (!bp->ntp_fltr_bmap)
3239                 rc = -ENOMEM;
3240
3241         return rc;
3242 #else
3243         return 0;
3244 #endif
3245 }
3246
3247 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3248 {
3249         bnxt_free_vnic_attributes(bp);
3250         bnxt_free_tx_rings(bp);
3251         bnxt_free_rx_rings(bp);
3252         bnxt_free_cp_rings(bp);
3253         bnxt_free_ntp_fltrs(bp, irq_re_init);
3254         if (irq_re_init) {
3255                 bnxt_free_stats(bp);
3256                 bnxt_free_ring_grps(bp);
3257                 bnxt_free_vnics(bp);
3258                 kfree(bp->tx_ring_map);
3259                 bp->tx_ring_map = NULL;
3260                 kfree(bp->tx_ring);
3261                 bp->tx_ring = NULL;
3262                 kfree(bp->rx_ring);
3263                 bp->rx_ring = NULL;
3264                 kfree(bp->bnapi);
3265                 bp->bnapi = NULL;
3266         } else {
3267                 bnxt_clear_ring_indices(bp);
3268         }
3269 }
3270
3271 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3272 {
3273         int i, j, rc, size, arr_size;
3274         void *bnapi;
3275
3276         if (irq_re_init) {
3277                 /* Allocate bnapi mem pointer array and mem block for
3278                  * all queues
3279                  */
3280                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3281                                 bp->cp_nr_rings);
3282                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3283                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3284                 if (!bnapi)
3285                         return -ENOMEM;
3286
3287                 bp->bnapi = bnapi;
3288                 bnapi += arr_size;
3289                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3290                         bp->bnapi[i] = bnapi;
3291                         bp->bnapi[i]->index = i;
3292                         bp->bnapi[i]->bp = bp;
3293                 }
3294
3295                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3296                                       sizeof(struct bnxt_rx_ring_info),
3297                                       GFP_KERNEL);
3298                 if (!bp->rx_ring)
3299                         return -ENOMEM;
3300
3301                 for (i = 0; i < bp->rx_nr_rings; i++) {
3302                         bp->rx_ring[i].bnapi = bp->bnapi[i];
3303                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3304                 }
3305
3306                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3307                                       sizeof(struct bnxt_tx_ring_info),
3308                                       GFP_KERNEL);
3309                 if (!bp->tx_ring)
3310                         return -ENOMEM;
3311
3312                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3313                                           GFP_KERNEL);
3314
3315                 if (!bp->tx_ring_map)
3316                         return -ENOMEM;
3317
3318                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3319                         j = 0;
3320                 else
3321                         j = bp->rx_nr_rings;
3322
3323                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3324                         bp->tx_ring[i].bnapi = bp->bnapi[j];
3325                         bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
3326                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3327                         if (i >= bp->tx_nr_rings_xdp) {
3328                                 bp->tx_ring[i].txq_index = i -
3329                                         bp->tx_nr_rings_xdp;
3330                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3331                         } else {
3332                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3333                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3334                         }
3335                 }
3336
3337                 rc = bnxt_alloc_stats(bp);
3338                 if (rc)
3339                         goto alloc_mem_err;
3340
3341                 rc = bnxt_alloc_ntp_fltrs(bp);
3342                 if (rc)
3343                         goto alloc_mem_err;
3344
3345                 rc = bnxt_alloc_vnics(bp);
3346                 if (rc)
3347                         goto alloc_mem_err;
3348         }
3349
3350         bnxt_init_ring_struct(bp);
3351
3352         rc = bnxt_alloc_rx_rings(bp);
3353         if (rc)
3354                 goto alloc_mem_err;
3355
3356         rc = bnxt_alloc_tx_rings(bp);
3357         if (rc)
3358                 goto alloc_mem_err;
3359
3360         rc = bnxt_alloc_cp_rings(bp);
3361         if (rc)
3362                 goto alloc_mem_err;
3363
3364         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3365                                   BNXT_VNIC_UCAST_FLAG;
3366         rc = bnxt_alloc_vnic_attributes(bp);
3367         if (rc)
3368                 goto alloc_mem_err;
3369         return 0;
3370
3371 alloc_mem_err:
3372         bnxt_free_mem(bp, true);
3373         return rc;
3374 }
3375
3376 static void bnxt_disable_int(struct bnxt *bp)
3377 {
3378         int i;
3379
3380         if (!bp->bnapi)
3381                 return;
3382
3383         for (i = 0; i < bp->cp_nr_rings; i++) {
3384                 struct bnxt_napi *bnapi = bp->bnapi[i];
3385                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3386                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3387
3388                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3389                         BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3390         }
3391 }
3392
3393 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
3394 {
3395         struct bnxt_napi *bnapi = bp->bnapi[n];
3396         struct bnxt_cp_ring_info *cpr;
3397
3398         cpr = &bnapi->cp_ring;
3399         return cpr->cp_ring_struct.map_idx;
3400 }
3401
3402 static void bnxt_disable_int_sync(struct bnxt *bp)
3403 {
3404         int i;
3405
3406         atomic_inc(&bp->intr_sem);
3407
3408         bnxt_disable_int(bp);
3409         for (i = 0; i < bp->cp_nr_rings; i++) {
3410                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
3411
3412                 synchronize_irq(bp->irq_tbl[map_idx].vector);
3413         }
3414 }
3415
3416 static void bnxt_enable_int(struct bnxt *bp)
3417 {
3418         int i;
3419
3420         atomic_set(&bp->intr_sem, 0);
3421         for (i = 0; i < bp->cp_nr_rings; i++) {
3422                 struct bnxt_napi *bnapi = bp->bnapi[i];
3423                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3424
3425                 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
3426         }
3427 }
3428
3429 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3430                             u16 cmpl_ring, u16 target_id)
3431 {
3432         struct input *req = request;
3433
3434         req->req_type = cpu_to_le16(req_type);
3435         req->cmpl_ring = cpu_to_le16(cmpl_ring);
3436         req->target_id = cpu_to_le16(target_id);
3437         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3438 }
3439
3440 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3441                                  int timeout, bool silent)
3442 {
3443         int i, intr_process, rc, tmo_count;
3444         struct input *req = msg;
3445         u32 *data = msg;
3446         __le32 *resp_len;
3447         u8 *valid;
3448         u16 cp_ring_id, len = 0;
3449         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
3450         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
3451         struct hwrm_short_input short_input = {0};
3452
3453         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
3454         memset(resp, 0, PAGE_SIZE);
3455         cp_ring_id = le16_to_cpu(req->cmpl_ring);
3456         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3457
3458         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
3459                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3460
3461                 memcpy(short_cmd_req, req, msg_len);
3462                 memset(short_cmd_req + msg_len, 0, BNXT_HWRM_MAX_REQ_LEN -
3463                                                    msg_len);
3464
3465                 short_input.req_type = req->req_type;
3466                 short_input.signature =
3467                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3468                 short_input.size = cpu_to_le16(msg_len);
3469                 short_input.req_addr =
3470                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3471
3472                 data = (u32 *)&short_input;
3473                 msg_len = sizeof(short_input);
3474
3475                 /* Sync memory write before updating doorbell */
3476                 wmb();
3477
3478                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3479         }
3480
3481         /* Write request msg to hwrm channel */
3482         __iowrite32_copy(bp->bar0, data, msg_len / 4);
3483
3484         for (i = msg_len; i < max_req_len; i += 4)
3485                 writel(0, bp->bar0 + i);
3486
3487         /* currently supports only one outstanding message */
3488         if (intr_process)
3489                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
3490
3491         /* Ring channel doorbell */
3492         writel(1, bp->bar0 + 0x100);
3493
3494         if (!timeout)
3495                 timeout = DFLT_HWRM_CMD_TIMEOUT;
3496
3497         i = 0;
3498         tmo_count = timeout * 40;
3499         resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
3500         if (intr_process) {
3501                 /* Wait until hwrm response cmpl interrupt is processed */
3502                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
3503                        i++ < tmo_count) {
3504                         usleep_range(25, 40);
3505                 }
3506
3507                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
3508                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
3509                                    le16_to_cpu(req->req_type));
3510                         return -1;
3511                 }
3512                 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3513                       HWRM_RESP_LEN_SFT;
3514                 valid = bp->hwrm_cmd_resp_addr + len - 1;
3515         } else {
3516                 /* Check if response len is updated */
3517                 for (i = 0; i < tmo_count; i++) {
3518                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3519                               HWRM_RESP_LEN_SFT;
3520                         if (len)
3521                                 break;
3522                         usleep_range(25, 40);
3523                 }
3524
3525                 if (i >= tmo_count) {
3526                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
3527                                    timeout, le16_to_cpu(req->req_type),
3528                                    le16_to_cpu(req->seq_id), len);
3529                         return -1;
3530                 }
3531
3532                 /* Last byte of resp contains valid bit */
3533                 valid = bp->hwrm_cmd_resp_addr + len - 1;
3534                 for (i = 0; i < 5; i++) {
3535                         /* make sure we read from updated DMA memory */
3536                         dma_rmb();
3537                         if (*valid)
3538                                 break;
3539                         udelay(1);
3540                 }
3541
3542                 if (i >= 5) {
3543                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
3544                                    timeout, le16_to_cpu(req->req_type),
3545                                    le16_to_cpu(req->seq_id), len, *valid);
3546                         return -1;
3547                 }
3548         }
3549
3550         /* Zero valid bit for compatibility.  Valid bit in an older spec
3551          * may become a new field in a newer spec.  We must make sure that
3552          * a new field not implemented by old spec will read zero.
3553          */
3554         *valid = 0;
3555         rc = le16_to_cpu(resp->error_code);
3556         if (rc && !silent)
3557                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3558                            le16_to_cpu(resp->req_type),
3559                            le16_to_cpu(resp->seq_id), rc);
3560         return rc;
3561 }
3562
3563 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3564 {
3565         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
3566 }
3567
3568 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3569                               int timeout)
3570 {
3571         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3572 }
3573
3574 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3575 {
3576         int rc;
3577
3578         mutex_lock(&bp->hwrm_cmd_lock);
3579         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3580         mutex_unlock(&bp->hwrm_cmd_lock);
3581         return rc;
3582 }
3583
3584 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3585                              int timeout)
3586 {
3587         int rc;
3588
3589         mutex_lock(&bp->hwrm_cmd_lock);
3590         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3591         mutex_unlock(&bp->hwrm_cmd_lock);
3592         return rc;
3593 }
3594
3595 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3596                                      int bmap_size)
3597 {
3598         struct hwrm_func_drv_rgtr_input req = {0};
3599         DECLARE_BITMAP(async_events_bmap, 256);
3600         u32 *events = (u32 *)async_events_bmap;
3601         int i;
3602
3603         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3604
3605         req.enables =
3606                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
3607
3608         memset(async_events_bmap, 0, sizeof(async_events_bmap));
3609         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
3610                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
3611
3612         if (bmap && bmap_size) {
3613                 for (i = 0; i < bmap_size; i++) {
3614                         if (test_bit(i, bmap))
3615                                 __set_bit(i, async_events_bmap);
3616                 }
3617         }
3618
3619         for (i = 0; i < 8; i++)
3620                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
3621
3622         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3623 }
3624
3625 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
3626 {
3627         struct hwrm_func_drv_rgtr_input req = {0};
3628
3629         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3630
3631         req.enables =
3632                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
3633                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
3634
3635         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
3636         req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
3637         req.ver_maj_8b = DRV_VER_MAJ;
3638         req.ver_min_8b = DRV_VER_MIN;
3639         req.ver_upd_8b = DRV_VER_UPD;
3640         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
3641         req.ver_min = cpu_to_le16(DRV_VER_MIN);
3642         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
3643
3644         if (BNXT_PF(bp)) {
3645                 u32 data[8];
3646                 int i;
3647
3648                 memset(data, 0, sizeof(data));
3649                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
3650                         u16 cmd = bnxt_vf_req_snif[i];
3651                         unsigned int bit, idx;
3652
3653                         idx = cmd / 32;
3654                         bit = cmd % 32;
3655                         data[idx] |= 1 << bit;
3656                 }
3657
3658                 for (i = 0; i < 8; i++)
3659                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
3660
3661                 req.enables |=
3662                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
3663         }
3664
3665         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3666 }
3667
3668 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
3669 {
3670         struct hwrm_func_drv_unrgtr_input req = {0};
3671
3672         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
3673         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3674 }
3675
3676 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
3677 {
3678         u32 rc = 0;
3679         struct hwrm_tunnel_dst_port_free_input req = {0};
3680
3681         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
3682         req.tunnel_type = tunnel_type;
3683
3684         switch (tunnel_type) {
3685         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
3686                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
3687                 break;
3688         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
3689                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
3690                 break;
3691         default:
3692                 break;
3693         }
3694
3695         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3696         if (rc)
3697                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
3698                            rc);
3699         return rc;
3700 }
3701
3702 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
3703                                            u8 tunnel_type)
3704 {
3705         u32 rc = 0;
3706         struct hwrm_tunnel_dst_port_alloc_input req = {0};
3707         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3708
3709         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
3710
3711         req.tunnel_type = tunnel_type;
3712         req.tunnel_dst_port_val = port;
3713
3714         mutex_lock(&bp->hwrm_cmd_lock);
3715         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3716         if (rc) {
3717                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
3718                            rc);
3719                 goto err_out;
3720         }
3721
3722         switch (tunnel_type) {
3723         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
3724                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
3725                 break;
3726         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
3727                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
3728                 break;
3729         default:
3730                 break;
3731         }
3732
3733 err_out:
3734         mutex_unlock(&bp->hwrm_cmd_lock);
3735         return rc;
3736 }
3737
3738 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
3739 {
3740         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
3741         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3742
3743         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
3744         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3745
3746         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
3747         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
3748         req.mask = cpu_to_le32(vnic->rx_mask);
3749         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3750 }
3751
3752 #ifdef CONFIG_RFS_ACCEL
3753 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3754                                             struct bnxt_ntuple_filter *fltr)
3755 {
3756         struct hwrm_cfa_ntuple_filter_free_input req = {0};
3757
3758         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3759         req.ntuple_filter_id = fltr->filter_id;
3760         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3761 }
3762
3763 #define BNXT_NTP_FLTR_FLAGS                                     \
3764         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
3765          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
3766          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
3767          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
3768          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
3769          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
3770          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
3771          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
3772          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
3773          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
3774          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
3775          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
3776          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
3777          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
3778
3779 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
3780                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
3781
3782 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3783                                              struct bnxt_ntuple_filter *fltr)
3784 {
3785         int rc = 0;
3786         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3787         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3788                 bp->hwrm_cmd_resp_addr;
3789         struct flow_keys *keys = &fltr->fkeys;
3790         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3791
3792         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3793         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
3794
3795         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3796
3797         req.ethertype = htons(ETH_P_IP);
3798         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
3799         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
3800         req.ip_protocol = keys->basic.ip_proto;
3801
3802         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
3803                 int i;
3804
3805                 req.ethertype = htons(ETH_P_IPV6);
3806                 req.ip_addr_type =
3807                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
3808                 *(struct in6_addr *)&req.src_ipaddr[0] =
3809                         keys->addrs.v6addrs.src;
3810                 *(struct in6_addr *)&req.dst_ipaddr[0] =
3811                         keys->addrs.v6addrs.dst;
3812                 for (i = 0; i < 4; i++) {
3813                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3814                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
3815                 }
3816         } else {
3817                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3818                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3819                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3820                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3821         }
3822         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
3823                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
3824                 req.tunnel_type =
3825                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
3826         }
3827
3828         req.src_port = keys->ports.src;
3829         req.src_port_mask = cpu_to_be16(0xffff);
3830         req.dst_port = keys->ports.dst;
3831         req.dst_port_mask = cpu_to_be16(0xffff);
3832
3833         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
3834         mutex_lock(&bp->hwrm_cmd_lock);
3835         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3836         if (!rc)
3837                 fltr->filter_id = resp->ntuple_filter_id;
3838         mutex_unlock(&bp->hwrm_cmd_lock);
3839         return rc;
3840 }
3841 #endif
3842
3843 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3844                                      u8 *mac_addr)
3845 {
3846         u32 rc = 0;
3847         struct hwrm_cfa_l2_filter_alloc_input req = {0};
3848         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3849
3850         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3851         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
3852         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
3853                 req.flags |=
3854                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
3855         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
3856         req.enables =
3857                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
3858                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
3859                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3860         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3861         req.l2_addr_mask[0] = 0xff;
3862         req.l2_addr_mask[1] = 0xff;
3863         req.l2_addr_mask[2] = 0xff;
3864         req.l2_addr_mask[3] = 0xff;
3865         req.l2_addr_mask[4] = 0xff;
3866         req.l2_addr_mask[5] = 0xff;
3867
3868         mutex_lock(&bp->hwrm_cmd_lock);
3869         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3870         if (!rc)
3871                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3872                                                         resp->l2_filter_id;
3873         mutex_unlock(&bp->hwrm_cmd_lock);
3874         return rc;
3875 }
3876
3877 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3878 {
3879         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3880         int rc = 0;
3881
3882         /* Any associated ntuple filters will also be cleared by firmware. */
3883         mutex_lock(&bp->hwrm_cmd_lock);
3884         for (i = 0; i < num_of_vnics; i++) {
3885                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3886
3887                 for (j = 0; j < vnic->uc_filter_count; j++) {
3888                         struct hwrm_cfa_l2_filter_free_input req = {0};
3889
3890                         bnxt_hwrm_cmd_hdr_init(bp, &req,
3891                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
3892
3893                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
3894
3895                         rc = _hwrm_send_message(bp, &req, sizeof(req),
3896                                                 HWRM_CMD_TIMEOUT);
3897                 }
3898                 vnic->uc_filter_count = 0;
3899         }
3900         mutex_unlock(&bp->hwrm_cmd_lock);
3901
3902         return rc;
3903 }
3904
3905 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3906 {
3907         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3908         struct hwrm_vnic_tpa_cfg_input req = {0};
3909
3910         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
3911                 return 0;
3912
3913         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3914
3915         if (tpa_flags) {
3916                 u16 mss = bp->dev->mtu - 40;
3917                 u32 nsegs, n, segs = 0, flags;
3918
3919                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3920                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3921                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3922                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3923                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3924                 if (tpa_flags & BNXT_FLAG_GRO)
3925                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3926
3927                 req.flags = cpu_to_le32(flags);
3928
3929                 req.enables =
3930                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3931                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3932                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3933
3934                 /* Number of segs are log2 units, and first packet is not
3935                  * included as part of this units.
3936                  */
3937                 if (mss <= BNXT_RX_PAGE_SIZE) {
3938                         n = BNXT_RX_PAGE_SIZE / mss;
3939                         nsegs = (MAX_SKB_FRAGS - 1) * n;
3940                 } else {
3941                         n = mss / BNXT_RX_PAGE_SIZE;
3942                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
3943                                 n++;
3944                         nsegs = (MAX_SKB_FRAGS - n) / n;
3945                 }
3946
3947                 segs = ilog2(nsegs);
3948                 req.max_agg_segs = cpu_to_le16(segs);
3949                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3950
3951                 req.min_agg_len = cpu_to_le32(512);
3952         }
3953         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3954
3955         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3956 }
3957
3958 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3959 {
3960         u32 i, j, max_rings;
3961         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3962         struct hwrm_vnic_rss_cfg_input req = {0};
3963
3964         if (vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
3965                 return 0;
3966
3967         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3968         if (set_rss) {
3969                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
3970                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
3971                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3972                                 max_rings = bp->rx_nr_rings - 1;
3973                         else
3974                                 max_rings = bp->rx_nr_rings;
3975                 } else {
3976                         max_rings = 1;
3977                 }
3978
3979                 /* Fill the RSS indirection table with ring group ids */
3980                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3981                         if (j == max_rings)
3982                                 j = 0;
3983                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3984                 }
3985
3986                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3987                 req.hash_key_tbl_addr =
3988                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
3989         }
3990         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
3991         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3992 }
3993
3994 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3995 {
3996         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3997         struct hwrm_vnic_plcmodes_cfg_input req = {0};
3998
3999         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4000         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4001                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4002                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4003         req.enables =
4004                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4005                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4006         /* thresholds not implemented in firmware yet */
4007         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4008         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4009         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4010         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4011 }
4012
4013 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4014                                         u16 ctx_idx)
4015 {
4016         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4017
4018         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4019         req.rss_cos_lb_ctx_id =
4020                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4021
4022         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4023         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4024 }
4025
4026 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4027 {
4028         int i, j;
4029
4030         for (i = 0; i < bp->nr_vnics; i++) {
4031                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4032
4033                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4034                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4035                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4036                 }
4037         }
4038         bp->rsscos_nr_ctxs = 0;
4039 }
4040
4041 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4042 {
4043         int rc;
4044         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4045         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4046                                                 bp->hwrm_cmd_resp_addr;
4047
4048         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4049                                -1);
4050
4051         mutex_lock(&bp->hwrm_cmd_lock);
4052         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4053         if (!rc)
4054                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4055                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
4056         mutex_unlock(&bp->hwrm_cmd_lock);
4057
4058         return rc;
4059 }
4060
4061 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4062 {
4063         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4064                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4065         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4066 }
4067
4068 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
4069 {
4070         unsigned int ring = 0, grp_idx;
4071         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4072         struct hwrm_vnic_cfg_input req = {0};
4073         u16 def_vlan = 0;
4074
4075         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
4076
4077         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
4078         /* Only RSS support for now TBD: COS & LB */
4079         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4080                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4081                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4082                                            VNIC_CFG_REQ_ENABLES_MRU);
4083         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4084                 req.rss_rule =
4085                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4086                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4087                                            VNIC_CFG_REQ_ENABLES_MRU);
4088                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
4089         } else {
4090                 req.rss_rule = cpu_to_le16(0xffff);
4091         }
4092
4093         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4094             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
4095                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4096                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4097         } else {
4098                 req.cos_rule = cpu_to_le16(0xffff);
4099         }
4100
4101         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4102                 ring = 0;
4103         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
4104                 ring = vnic_id - 1;
4105         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4106                 ring = bp->rx_nr_rings - 1;
4107
4108         grp_idx = bp->rx_ring[ring].bnapi->index;
4109         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4110         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
4111
4112         req.lb_rule = cpu_to_le16(0xffff);
4113         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4114                               VLAN_HLEN);
4115
4116 #ifdef CONFIG_BNXT_SRIOV
4117         if (BNXT_VF(bp))
4118                 def_vlan = bp->vf.vlan;
4119 #endif
4120         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4121                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4122         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4123                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
4124
4125         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4126 }
4127
4128 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4129 {
4130         u32 rc = 0;
4131
4132         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4133                 struct hwrm_vnic_free_input req = {0};
4134
4135                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4136                 req.vnic_id =
4137                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4138
4139                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4140                 if (rc)
4141                         return rc;
4142                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4143         }
4144         return rc;
4145 }
4146
4147 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4148 {
4149         u16 i;
4150
4151         for (i = 0; i < bp->nr_vnics; i++)
4152                 bnxt_hwrm_vnic_free_one(bp, i);
4153 }
4154
4155 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4156                                 unsigned int start_rx_ring_idx,
4157                                 unsigned int nr_rings)
4158 {
4159         int rc = 0;
4160         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4161         struct hwrm_vnic_alloc_input req = {0};
4162         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4163
4164         /* map ring groups to this vnic */
4165         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4166                 grp_idx = bp->rx_ring[i].bnapi->index;
4167                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4168                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4169                                    j, nr_rings);
4170                         break;
4171                 }
4172                 bp->vnic_info[vnic_id].fw_grp_ids[j] =
4173                                         bp->grp_info[grp_idx].fw_grp_id;
4174         }
4175
4176         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[0] = INVALID_HW_RING_ID;
4177         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[1] = INVALID_HW_RING_ID;
4178         if (vnic_id == 0)
4179                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4180
4181         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4182
4183         mutex_lock(&bp->hwrm_cmd_lock);
4184         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4185         if (!rc)
4186                 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
4187         mutex_unlock(&bp->hwrm_cmd_lock);
4188         return rc;
4189 }
4190
4191 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4192 {
4193         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4194         struct hwrm_vnic_qcaps_input req = {0};
4195         int rc;
4196
4197         if (bp->hwrm_spec_code < 0x10600)
4198                 return 0;
4199
4200         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4201         mutex_lock(&bp->hwrm_cmd_lock);
4202         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4203         if (!rc) {
4204                 u32 flags = le32_to_cpu(resp->flags);
4205
4206                 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)
4207                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4208                 if (flags &
4209                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
4210                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
4211         }
4212         mutex_unlock(&bp->hwrm_cmd_lock);
4213         return rc;
4214 }
4215
4216 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4217 {
4218         u16 i;
4219         u32 rc = 0;
4220
4221         mutex_lock(&bp->hwrm_cmd_lock);
4222         for (i = 0; i < bp->rx_nr_rings; i++) {
4223                 struct hwrm_ring_grp_alloc_input req = {0};
4224                 struct hwrm_ring_grp_alloc_output *resp =
4225                                         bp->hwrm_cmd_resp_addr;
4226                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
4227
4228                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4229
4230                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4231                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4232                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4233                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
4234
4235                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4236                                         HWRM_CMD_TIMEOUT);
4237                 if (rc)
4238                         break;
4239
4240                 bp->grp_info[grp_idx].fw_grp_id =
4241                         le32_to_cpu(resp->ring_group_id);
4242         }
4243         mutex_unlock(&bp->hwrm_cmd_lock);
4244         return rc;
4245 }
4246
4247 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4248 {
4249         u16 i;
4250         u32 rc = 0;
4251         struct hwrm_ring_grp_free_input req = {0};
4252
4253         if (!bp->grp_info)
4254                 return 0;
4255
4256         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4257
4258         mutex_lock(&bp->hwrm_cmd_lock);
4259         for (i = 0; i < bp->cp_nr_rings; i++) {
4260                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4261                         continue;
4262                 req.ring_group_id =
4263                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
4264
4265                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4266                                         HWRM_CMD_TIMEOUT);
4267                 if (rc)
4268                         break;
4269                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4270         }
4271         mutex_unlock(&bp->hwrm_cmd_lock);
4272         return rc;
4273 }
4274
4275 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4276                                     struct bnxt_ring_struct *ring,
4277                                     u32 ring_type, u32 map_index)
4278 {
4279         int rc = 0, err = 0;
4280         struct hwrm_ring_alloc_input req = {0};
4281         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4282         struct bnxt_ring_grp_info *grp_info;
4283         u16 ring_id;
4284
4285         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4286
4287         req.enables = 0;
4288         if (ring->nr_pages > 1) {
4289                 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
4290                 /* Page size is in log2 units */
4291                 req.page_size = BNXT_PAGE_SHIFT;
4292                 req.page_tbl_depth = 1;
4293         } else {
4294                 req.page_tbl_addr =  cpu_to_le64(ring->dma_arr[0]);
4295         }
4296         req.fbo = 0;
4297         /* Association of ring index with doorbell index and MSIX number */
4298         req.logical_id = cpu_to_le16(map_index);
4299
4300         switch (ring_type) {
4301         case HWRM_RING_ALLOC_TX:
4302                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4303                 /* Association of transmit ring with completion ring */
4304                 grp_info = &bp->grp_info[ring->grp_idx];
4305                 req.cmpl_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
4306                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4307                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4308                 req.queue_id = cpu_to_le16(ring->queue_id);
4309                 break;
4310         case HWRM_RING_ALLOC_RX:
4311                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4312                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4313                 break;
4314         case HWRM_RING_ALLOC_AGG:
4315                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4316                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4317                 break;
4318         case HWRM_RING_ALLOC_CMPL:
4319                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
4320                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4321                 if (bp->flags & BNXT_FLAG_USING_MSIX)
4322                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4323                 break;
4324         default:
4325                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4326                            ring_type);
4327                 return -1;
4328         }
4329
4330         mutex_lock(&bp->hwrm_cmd_lock);
4331         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4332         err = le16_to_cpu(resp->error_code);
4333         ring_id = le16_to_cpu(resp->ring_id);
4334         mutex_unlock(&bp->hwrm_cmd_lock);
4335
4336         if (rc || err) {
4337                 switch (ring_type) {
4338                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4339                         netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
4340                                    rc, err);
4341                         return -1;
4342
4343                 case RING_FREE_REQ_RING_TYPE_RX:
4344                         netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
4345                                    rc, err);
4346                         return -1;
4347
4348                 case RING_FREE_REQ_RING_TYPE_TX:
4349                         netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
4350                                    rc, err);
4351                         return -1;
4352
4353                 default:
4354                         netdev_err(bp->dev, "Invalid ring\n");
4355                         return -1;
4356                 }
4357         }
4358         ring->fw_ring_id = ring_id;
4359         return rc;
4360 }
4361
4362 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4363 {
4364         int rc;
4365
4366         if (BNXT_PF(bp)) {
4367                 struct hwrm_func_cfg_input req = {0};
4368
4369                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4370                 req.fid = cpu_to_le16(0xffff);
4371                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4372                 req.async_event_cr = cpu_to_le16(idx);
4373                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4374         } else {
4375                 struct hwrm_func_vf_cfg_input req = {0};
4376
4377                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4378                 req.enables =
4379                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4380                 req.async_event_cr = cpu_to_le16(idx);
4381                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4382         }
4383         return rc;
4384 }
4385
4386 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4387 {
4388         int i, rc = 0;
4389
4390         for (i = 0; i < bp->cp_nr_rings; i++) {
4391                 struct bnxt_napi *bnapi = bp->bnapi[i];
4392                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4393                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4394                 u32 map_idx = ring->map_idx;
4395
4396                 cpr->cp_doorbell = bp->bar1 + map_idx * 0x80;
4397                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL,
4398                                               map_idx);
4399                 if (rc)
4400                         goto err_out;
4401                 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4402                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
4403
4404                 if (!i) {
4405                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4406                         if (rc)
4407                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4408                 }
4409         }
4410
4411         for (i = 0; i < bp->tx_nr_rings; i++) {
4412                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4413                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4414                 u32 map_idx = i;
4415
4416                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
4417                                               map_idx);
4418                 if (rc)
4419                         goto err_out;
4420                 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
4421         }
4422
4423         for (i = 0; i < bp->rx_nr_rings; i++) {
4424                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4425                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4426                 u32 map_idx = rxr->bnapi->index;
4427
4428                 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
4429                                               map_idx);
4430                 if (rc)
4431                         goto err_out;
4432                 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
4433                 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
4434                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
4435         }
4436
4437         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4438                 for (i = 0; i < bp->rx_nr_rings; i++) {
4439                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4440                         struct bnxt_ring_struct *ring =
4441                                                 &rxr->rx_agg_ring_struct;
4442                         u32 grp_idx = ring->grp_idx;
4443                         u32 map_idx = grp_idx + bp->rx_nr_rings;
4444
4445                         rc = hwrm_ring_alloc_send_msg(bp, ring,
4446                                                       HWRM_RING_ALLOC_AGG,
4447                                                       map_idx);
4448                         if (rc)
4449                                 goto err_out;
4450
4451                         rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
4452                         writel(DB_KEY_RX | rxr->rx_agg_prod,
4453                                rxr->rx_agg_doorbell);
4454                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
4455                 }
4456         }
4457 err_out:
4458         return rc;
4459 }
4460
4461 static int hwrm_ring_free_send_msg(struct bnxt *bp,
4462                                    struct bnxt_ring_struct *ring,
4463                                    u32 ring_type, int cmpl_ring_id)
4464 {
4465         int rc;
4466         struct hwrm_ring_free_input req = {0};
4467         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
4468         u16 error_code;
4469
4470         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
4471         req.ring_type = ring_type;
4472         req.ring_id = cpu_to_le16(ring->fw_ring_id);
4473
4474         mutex_lock(&bp->hwrm_cmd_lock);
4475         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4476         error_code = le16_to_cpu(resp->error_code);
4477         mutex_unlock(&bp->hwrm_cmd_lock);
4478
4479         if (rc || error_code) {
4480                 switch (ring_type) {
4481                 case RING_FREE_REQ_RING_TYPE_L2_CMPL:
4482                         netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
4483                                    rc);
4484                         return rc;
4485                 case RING_FREE_REQ_RING_TYPE_RX:
4486                         netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
4487                                    rc);
4488                         return rc;
4489                 case RING_FREE_REQ_RING_TYPE_TX:
4490                         netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
4491                                    rc);
4492                         return rc;
4493                 default:
4494                         netdev_err(bp->dev, "Invalid ring\n");
4495                         return -1;
4496                 }
4497         }
4498         return 0;
4499 }
4500
4501 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
4502 {
4503         int i;
4504
4505         if (!bp->bnapi)
4506                 return;
4507
4508         for (i = 0; i < bp->tx_nr_rings; i++) {
4509                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4510                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4511                 u32 grp_idx = txr->bnapi->index;
4512                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4513
4514                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4515                         hwrm_ring_free_send_msg(bp, ring,
4516                                                 RING_FREE_REQ_RING_TYPE_TX,
4517                                                 close_path ? cmpl_ring_id :
4518                                                 INVALID_HW_RING_ID);
4519                         ring->fw_ring_id = INVALID_HW_RING_ID;
4520                 }
4521         }
4522
4523         for (i = 0; i < bp->rx_nr_rings; i++) {
4524                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4525                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4526                 u32 grp_idx = rxr->bnapi->index;
4527                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4528
4529                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4530                         hwrm_ring_free_send_msg(bp, ring,
4531                                                 RING_FREE_REQ_RING_TYPE_RX,
4532                                                 close_path ? cmpl_ring_id :
4533                                                 INVALID_HW_RING_ID);
4534                         ring->fw_ring_id = INVALID_HW_RING_ID;
4535                         bp->grp_info[grp_idx].rx_fw_ring_id =
4536                                 INVALID_HW_RING_ID;
4537                 }
4538         }
4539
4540         for (i = 0; i < bp->rx_nr_rings; i++) {
4541                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4542                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
4543                 u32 grp_idx = rxr->bnapi->index;
4544                 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
4545
4546                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4547                         hwrm_ring_free_send_msg(bp, ring,
4548                                                 RING_FREE_REQ_RING_TYPE_RX,
4549                                                 close_path ? cmpl_ring_id :
4550                                                 INVALID_HW_RING_ID);
4551                         ring->fw_ring_id = INVALID_HW_RING_ID;
4552                         bp->grp_info[grp_idx].agg_fw_ring_id =
4553                                 INVALID_HW_RING_ID;
4554                 }
4555         }
4556
4557         /* The completion rings are about to be freed.  After that the
4558          * IRQ doorbell will not work anymore.  So we need to disable
4559          * IRQ here.
4560          */
4561         bnxt_disable_int_sync(bp);
4562
4563         for (i = 0; i < bp->cp_nr_rings; i++) {
4564                 struct bnxt_napi *bnapi = bp->bnapi[i];
4565                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4566                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4567
4568                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
4569                         hwrm_ring_free_send_msg(bp, ring,
4570                                                 RING_FREE_REQ_RING_TYPE_L2_CMPL,
4571                                                 INVALID_HW_RING_ID);
4572                         ring->fw_ring_id = INVALID_HW_RING_ID;
4573                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4574                 }
4575         }
4576 }
4577
4578 static int bnxt_hwrm_get_rings(struct bnxt *bp)
4579 {
4580         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4581         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
4582         struct hwrm_func_qcfg_input req = {0};
4583         int rc;
4584
4585         if (bp->hwrm_spec_code < 0x10601)
4586                 return 0;
4587
4588         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4589         req.fid = cpu_to_le16(0xffff);
4590         mutex_lock(&bp->hwrm_cmd_lock);
4591         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4592         if (rc) {
4593                 mutex_unlock(&bp->hwrm_cmd_lock);
4594                 return -EIO;
4595         }
4596
4597         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
4598         if (bp->flags & BNXT_FLAG_NEW_RM) {
4599                 u16 cp, stats;
4600
4601                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
4602                 hw_resc->resv_hw_ring_grps =
4603                         le32_to_cpu(resp->alloc_hw_ring_grps);
4604                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
4605                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
4606                 stats = le16_to_cpu(resp->alloc_stat_ctx);
4607                 cp = min_t(u16, cp, stats);
4608                 hw_resc->resv_cp_rings = cp;
4609         }
4610         mutex_unlock(&bp->hwrm_cmd_lock);
4611         return 0;
4612 }
4613
4614 /* Caller must hold bp->hwrm_cmd_lock */
4615 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
4616 {
4617         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4618         struct hwrm_func_qcfg_input req = {0};
4619         int rc;
4620
4621         if (bp->hwrm_spec_code < 0x10601)
4622                 return 0;
4623
4624         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
4625         req.fid = cpu_to_le16(fid);
4626         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4627         if (!rc)
4628                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
4629
4630         return rc;
4631 }
4632
4633 static void
4634 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
4635                              int tx_rings, int rx_rings, int ring_grps,
4636                              int cp_rings, int vnics)
4637 {
4638         u32 enables = 0;
4639
4640         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
4641         req->fid = cpu_to_le16(0xffff);
4642         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
4643         req->num_tx_rings = cpu_to_le16(tx_rings);
4644         if (bp->flags & BNXT_FLAG_NEW_RM) {
4645                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
4646                 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
4647                                       FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
4648                 enables |= ring_grps ?
4649                            FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
4650                 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
4651
4652                 req->num_rx_rings = cpu_to_le16(rx_rings);
4653                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
4654                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
4655                 req->num_stat_ctxs = req->num_cmpl_rings;
4656                 req->num_vnics = cpu_to_le16(vnics);
4657         }
4658         req->enables = cpu_to_le32(enables);
4659 }
4660
4661 static void
4662 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
4663                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
4664                              int rx_rings, int ring_grps, int cp_rings,
4665                              int vnics)
4666 {
4667         u32 enables = 0;
4668
4669         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
4670         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
4671         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
4672         enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
4673                               FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
4674         enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
4675         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
4676
4677         req->num_tx_rings = cpu_to_le16(tx_rings);
4678         req->num_rx_rings = cpu_to_le16(rx_rings);
4679         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
4680         req->num_cmpl_rings = cpu_to_le16(cp_rings);
4681         req->num_stat_ctxs = req->num_cmpl_rings;
4682         req->num_vnics = cpu_to_le16(vnics);
4683
4684         req->enables = cpu_to_le32(enables);
4685 }
4686
4687 static int
4688 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
4689                            int ring_grps, int cp_rings, int vnics)
4690 {
4691         struct hwrm_func_cfg_input req = {0};
4692         int rc;
4693
4694         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
4695                                      cp_rings, vnics);
4696         if (!req.enables)
4697                 return 0;
4698
4699         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4700         if (rc)
4701                 return -ENOMEM;
4702
4703         if (bp->hwrm_spec_code < 0x10601)
4704                 bp->hw_resc.resv_tx_rings = tx_rings;
4705
4706         rc = bnxt_hwrm_get_rings(bp);
4707         return rc;
4708 }
4709
4710 static int
4711 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
4712                            int ring_grps, int cp_rings, int vnics)
4713 {
4714         struct hwrm_func_vf_cfg_input req = {0};
4715         int rc;
4716
4717         if (!(bp->flags & BNXT_FLAG_NEW_RM)) {
4718                 bp->hw_resc.resv_tx_rings = tx_rings;
4719                 return 0;
4720         }
4721
4722         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
4723                                      cp_rings, vnics);
4724         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4725         if (rc)
4726                 return -ENOMEM;
4727
4728         rc = bnxt_hwrm_get_rings(bp);
4729         return rc;
4730 }
4731
4732 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
4733                                    int cp, int vnic)
4734 {
4735         if (BNXT_PF(bp))
4736                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic);
4737         else
4738                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic);
4739 }
4740
4741 static int bnxt_cp_rings_in_use(struct bnxt *bp)
4742 {
4743         int cp = bp->cp_nr_rings;
4744         int ulp_msix, ulp_base;
4745
4746         ulp_msix = bnxt_get_ulp_msix_num(bp);
4747         if (ulp_msix) {
4748                 ulp_base = bnxt_get_ulp_msix_base(bp);
4749                 cp += ulp_msix;
4750                 if ((ulp_base + ulp_msix) > cp)
4751                         cp = ulp_base + ulp_msix;
4752         }
4753         return cp;
4754 }
4755
4756 static bool bnxt_need_reserve_rings(struct bnxt *bp)
4757 {
4758         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
4759         int cp = bnxt_cp_rings_in_use(bp);
4760         int rx = bp->rx_nr_rings;
4761         int vnic = 1, grp = rx;
4762
4763         if (bp->hwrm_spec_code < 0x10601)
4764                 return false;
4765
4766         if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
4767                 return true;
4768
4769         if (bp->flags & BNXT_FLAG_RFS)
4770                 vnic = rx + 1;
4771         if (bp->flags & BNXT_FLAG_AGG_RINGS)
4772                 rx <<= 1;
4773         if ((bp->flags & BNXT_FLAG_NEW_RM) &&
4774             (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
4775              hw_resc->resv_hw_ring_grps != grp || hw_resc->resv_vnics != vnic))
4776                 return true;
4777         return false;
4778 }
4779
4780 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4781                            bool shared);
4782
4783 static int __bnxt_reserve_rings(struct bnxt *bp)
4784 {
4785         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
4786         int cp = bnxt_cp_rings_in_use(bp);
4787         int tx = bp->tx_nr_rings;
4788         int rx = bp->rx_nr_rings;
4789         int grp, rx_rings, rc;
4790         bool sh = false;
4791         int vnic = 1;
4792
4793         if (!bnxt_need_reserve_rings(bp))
4794                 return 0;
4795
4796         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4797                 sh = true;
4798         if (bp->flags & BNXT_FLAG_RFS)
4799                 vnic = rx + 1;
4800         if (bp->flags & BNXT_FLAG_AGG_RINGS)
4801                 rx <<= 1;
4802         grp = bp->rx_nr_rings;
4803
4804         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic);
4805         if (rc)
4806                 return rc;
4807
4808         tx = hw_resc->resv_tx_rings;
4809         if (bp->flags & BNXT_FLAG_NEW_RM) {
4810                 rx = hw_resc->resv_rx_rings;
4811                 cp = hw_resc->resv_cp_rings;
4812                 grp = hw_resc->resv_hw_ring_grps;
4813                 vnic = hw_resc->resv_vnics;
4814         }
4815
4816         rx_rings = rx;
4817         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4818                 if (rx >= 2) {
4819                         rx_rings = rx >> 1;
4820                 } else {
4821                         if (netif_running(bp->dev))
4822                                 return -ENOMEM;
4823
4824                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4825                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4826                         bp->dev->hw_features &= ~NETIF_F_LRO;
4827                         bp->dev->features &= ~NETIF_F_LRO;
4828                         bnxt_set_ring_params(bp);
4829                 }
4830         }
4831         rx_rings = min_t(int, rx_rings, grp);
4832         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
4833         if (bp->flags & BNXT_FLAG_AGG_RINGS)
4834                 rx = rx_rings << 1;
4835         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
4836         bp->tx_nr_rings = tx;
4837         bp->rx_nr_rings = rx_rings;
4838         bp->cp_nr_rings = cp;
4839
4840         if (!tx || !rx || !cp || !grp || !vnic)
4841                 return -ENOMEM;
4842
4843         return rc;
4844 }
4845
4846 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
4847                                     int ring_grps, int cp_rings, int vnics)
4848 {
4849         struct hwrm_func_vf_cfg_input req = {0};
4850         u32 flags;
4851         int rc;
4852
4853         if (!(bp->flags & BNXT_FLAG_NEW_RM))
4854                 return 0;
4855
4856         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
4857                                      cp_rings, vnics);
4858         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
4859                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
4860                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
4861                 FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST |
4862                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
4863                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
4864
4865         req.flags = cpu_to_le32(flags);
4866         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4867         if (rc)
4868                 return -ENOMEM;
4869         return 0;
4870 }
4871
4872 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
4873                                     int ring_grps, int cp_rings, int vnics)
4874 {
4875         struct hwrm_func_cfg_input req = {0};
4876         u32 flags;
4877         int rc;
4878
4879         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
4880                                      cp_rings, vnics);
4881         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
4882         if (bp->flags & BNXT_FLAG_NEW_RM)
4883                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
4884                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
4885                          FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST |
4886                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
4887                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
4888
4889         req.flags = cpu_to_le32(flags);
4890         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4891         if (rc)
4892                 return -ENOMEM;
4893         return 0;
4894 }
4895
4896 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
4897                                  int ring_grps, int cp_rings, int vnics)
4898 {
4899         if (bp->hwrm_spec_code < 0x10801)
4900                 return 0;
4901
4902         if (BNXT_PF(bp))
4903                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
4904                                                 ring_grps, cp_rings, vnics);
4905
4906         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
4907                                         cp_rings, vnics);
4908 }
4909
4910 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
4911         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
4912 {
4913         u16 val, tmr, max, flags;
4914
4915         max = hw_coal->bufs_per_record * 128;
4916         if (hw_coal->budget)
4917                 max = hw_coal->bufs_per_record * hw_coal->budget;
4918
4919         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
4920         req->num_cmpl_aggr_int = cpu_to_le16(val);
4921
4922         /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
4923         val = min_t(u16, val, 63);
4924         req->num_cmpl_dma_aggr = cpu_to_le16(val);
4925
4926         /* This is a 6-bit value and must not be 0, or we'll get non stop IRQ */
4927         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 63);
4928         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
4929
4930         tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks);
4931         tmr = max_t(u16, tmr, 1);
4932         req->int_lat_tmr_max = cpu_to_le16(tmr);
4933
4934         /* min timer set to 1/2 of interrupt timer */
4935         val = tmr / 2;
4936         req->int_lat_tmr_min = cpu_to_le16(val);
4937
4938         /* buf timer set to 1/4 of interrupt timer */
4939         val = max_t(u16, tmr / 4, 1);
4940         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
4941
4942         tmr = BNXT_USEC_TO_COAL_TIMER(hw_coal->coal_ticks_irq);
4943         tmr = max_t(u16, tmr, 1);
4944         req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
4945
4946         flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
4947         if (hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
4948                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
4949         req->flags = cpu_to_le16(flags);
4950 }
4951
4952 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
4953 {
4954         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
4955         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4956         struct bnxt_coal coal;
4957         unsigned int grp_idx;
4958
4959         /* Tick values in micro seconds.
4960          * 1 coal_buf x bufs_per_record = 1 completion record.
4961          */
4962         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
4963
4964         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
4965         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
4966
4967         if (!bnapi->rx_ring)
4968                 return -ENODEV;
4969
4970         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
4971                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4972
4973         bnxt_hwrm_set_coal_params(&coal, &req_rx);
4974
4975         grp_idx = bnapi->index;
4976         req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4977
4978         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
4979                                  HWRM_CMD_TIMEOUT);
4980 }
4981
4982 int bnxt_hwrm_set_coal(struct bnxt *bp)
4983 {
4984         int i, rc = 0;
4985         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
4986                                                            req_tx = {0}, *req;
4987
4988         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
4989                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4990         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
4991                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
4992
4993         bnxt_hwrm_set_coal_params(&bp->rx_coal, &req_rx);
4994         bnxt_hwrm_set_coal_params(&bp->tx_coal, &req_tx);
4995
4996         mutex_lock(&bp->hwrm_cmd_lock);
4997         for (i = 0; i < bp->cp_nr_rings; i++) {
4998                 struct bnxt_napi *bnapi = bp->bnapi[i];
4999
5000                 req = &req_rx;
5001                 if (!bnapi->rx_ring)
5002                         req = &req_tx;
5003                 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
5004
5005                 rc = _hwrm_send_message(bp, req, sizeof(*req),
5006                                         HWRM_CMD_TIMEOUT);
5007                 if (rc)
5008                         break;
5009         }
5010         mutex_unlock(&bp->hwrm_cmd_lock);
5011         return rc;
5012 }
5013
5014 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
5015 {
5016         int rc = 0, i;
5017         struct hwrm_stat_ctx_free_input req = {0};
5018
5019         if (!bp->bnapi)
5020                 return 0;
5021
5022         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5023                 return 0;
5024
5025         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
5026
5027         mutex_lock(&bp->hwrm_cmd_lock);
5028         for (i = 0; i < bp->cp_nr_rings; i++) {
5029                 struct bnxt_napi *bnapi = bp->bnapi[i];
5030                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5031
5032                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
5033                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
5034
5035                         rc = _hwrm_send_message(bp, &req, sizeof(req),
5036                                                 HWRM_CMD_TIMEOUT);
5037                         if (rc)
5038                                 break;
5039
5040                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
5041                 }
5042         }
5043         mutex_unlock(&bp->hwrm_cmd_lock);
5044         return rc;
5045 }
5046
5047 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
5048 {
5049         int rc = 0, i;
5050         struct hwrm_stat_ctx_alloc_input req = {0};
5051         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5052
5053         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5054                 return 0;
5055
5056         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
5057
5058         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
5059
5060         mutex_lock(&bp->hwrm_cmd_lock);
5061         for (i = 0; i < bp->cp_nr_rings; i++) {
5062                 struct bnxt_napi *bnapi = bp->bnapi[i];
5063                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5064
5065                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
5066
5067                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5068                                         HWRM_CMD_TIMEOUT);
5069                 if (rc)
5070                         break;
5071
5072                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
5073
5074                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
5075         }
5076         mutex_unlock(&bp->hwrm_cmd_lock);
5077         return rc;
5078 }
5079
5080 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
5081 {
5082         struct hwrm_func_qcfg_input req = {0};
5083         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5084         u16 flags;
5085         int rc;
5086
5087         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5088         req.fid = cpu_to_le16(0xffff);
5089         mutex_lock(&bp->hwrm_cmd_lock);
5090         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5091         if (rc)
5092                 goto func_qcfg_exit;
5093
5094 #ifdef CONFIG_BNXT_SRIOV
5095         if (BNXT_VF(bp)) {
5096                 struct bnxt_vf_info *vf = &bp->vf;
5097
5098                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
5099         }
5100 #endif
5101         flags = le16_to_cpu(resp->flags);
5102         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
5103                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
5104                 bp->flags |= BNXT_FLAG_FW_LLDP_AGENT;
5105                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
5106                         bp->flags |= BNXT_FLAG_FW_DCBX_AGENT;
5107         }
5108         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
5109                 bp->flags |= BNXT_FLAG_MULTI_HOST;
5110
5111         switch (resp->port_partition_type) {
5112         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
5113         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
5114         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
5115                 bp->port_partition_type = resp->port_partition_type;
5116                 break;
5117         }
5118         if (bp->hwrm_spec_code < 0x10707 ||
5119             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
5120                 bp->br_mode = BRIDGE_MODE_VEB;
5121         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
5122                 bp->br_mode = BRIDGE_MODE_VEPA;
5123         else
5124                 bp->br_mode = BRIDGE_MODE_UNDEF;
5125
5126         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
5127         if (!bp->max_mtu)
5128                 bp->max_mtu = BNXT_MAX_MTU;
5129
5130 func_qcfg_exit:
5131         mutex_unlock(&bp->hwrm_cmd_lock);
5132         return rc;
5133 }
5134
5135 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
5136 {
5137         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5138         struct hwrm_func_resource_qcaps_input req = {0};
5139         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5140         int rc;
5141
5142         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
5143         req.fid = cpu_to_le16(0xffff);
5144
5145         mutex_lock(&bp->hwrm_cmd_lock);
5146         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5147         if (rc) {
5148                 rc = -EIO;
5149                 goto hwrm_func_resc_qcaps_exit;
5150         }
5151
5152         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
5153         if (!all)
5154                 goto hwrm_func_resc_qcaps_exit;
5155
5156         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
5157         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
5158         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
5159         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
5160         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
5161         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
5162         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
5163         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
5164         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
5165         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
5166         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
5167         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
5168         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
5169         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
5170         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
5171         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
5172
5173         if (BNXT_PF(bp)) {
5174                 struct bnxt_pf_info *pf = &bp->pf;
5175
5176                 pf->vf_resv_strategy =
5177                         le16_to_cpu(resp->vf_reservation_strategy);
5178                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL)
5179                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
5180         }
5181 hwrm_func_resc_qcaps_exit:
5182         mutex_unlock(&bp->hwrm_cmd_lock);
5183         return rc;
5184 }
5185
5186 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
5187 {
5188         int rc = 0;
5189         struct hwrm_func_qcaps_input req = {0};
5190         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5191         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5192         u32 flags;
5193
5194         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
5195         req.fid = cpu_to_le16(0xffff);
5196
5197         mutex_lock(&bp->hwrm_cmd_lock);
5198         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5199         if (rc)
5200                 goto hwrm_func_qcaps_exit;
5201
5202         flags = le32_to_cpu(resp->flags);
5203         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
5204                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
5205         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
5206                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
5207
5208         bp->tx_push_thresh = 0;
5209         if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
5210                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
5211
5212         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
5213         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
5214         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
5215         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
5216         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
5217         if (!hw_resc->max_hw_ring_grps)
5218                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
5219         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
5220         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
5221         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
5222
5223         if (BNXT_PF(bp)) {
5224                 struct bnxt_pf_info *pf = &bp->pf;
5225
5226                 pf->fw_fid = le16_to_cpu(resp->fid);
5227                 pf->port_id = le16_to_cpu(resp->port_id);
5228                 bp->dev->dev_port = pf->port_id;
5229                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
5230                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
5231                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
5232                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
5233                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
5234                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
5235                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
5236                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
5237                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
5238                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
5239                         bp->flags |= BNXT_FLAG_WOL_CAP;
5240         } else {
5241 #ifdef CONFIG_BNXT_SRIOV
5242                 struct bnxt_vf_info *vf = &bp->vf;
5243
5244                 vf->fw_fid = le16_to_cpu(resp->fid);
5245                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
5246 #endif
5247         }
5248
5249 hwrm_func_qcaps_exit:
5250         mutex_unlock(&bp->hwrm_cmd_lock);
5251         return rc;
5252 }
5253
5254 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
5255 {
5256         int rc;
5257
5258         rc = __bnxt_hwrm_func_qcaps(bp);
5259         if (rc)
5260                 return rc;
5261         if (bp->hwrm_spec_code >= 0x10803) {
5262                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
5263                 if (!rc)
5264                         bp->flags |= BNXT_FLAG_NEW_RM;
5265         }
5266         return 0;
5267 }
5268
5269 static int bnxt_hwrm_func_reset(struct bnxt *bp)
5270 {
5271         struct hwrm_func_reset_input req = {0};
5272
5273         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
5274         req.enables = 0;
5275
5276         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
5277 }
5278
5279 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
5280 {
5281         int rc = 0;
5282         struct hwrm_queue_qportcfg_input req = {0};
5283         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
5284         u8 i, *qptr;
5285
5286         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
5287
5288         mutex_lock(&bp->hwrm_cmd_lock);
5289         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5290         if (rc)
5291                 goto qportcfg_exit;
5292
5293         if (!resp->max_configurable_queues) {
5294                 rc = -EINVAL;
5295                 goto qportcfg_exit;
5296         }
5297         bp->max_tc = resp->max_configurable_queues;
5298         bp->max_lltc = resp->max_configurable_lossless_queues;
5299         if (bp->max_tc > BNXT_MAX_QUEUE)
5300                 bp->max_tc = BNXT_MAX_QUEUE;
5301
5302         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
5303                 bp->max_tc = 1;
5304
5305         if (bp->max_lltc > bp->max_tc)
5306                 bp->max_lltc = bp->max_tc;
5307
5308         qptr = &resp->queue_id0;
5309         for (i = 0; i < bp->max_tc; i++) {
5310                 bp->q_info[i].queue_id = *qptr++;
5311                 bp->q_info[i].queue_profile = *qptr++;
5312         }
5313
5314 qportcfg_exit:
5315         mutex_unlock(&bp->hwrm_cmd_lock);
5316         return rc;
5317 }
5318
5319 static int bnxt_hwrm_ver_get(struct bnxt *bp)
5320 {
5321         int rc;
5322         struct hwrm_ver_get_input req = {0};
5323         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
5324         u32 dev_caps_cfg;
5325
5326         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
5327         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
5328         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
5329         req.hwrm_intf_min = HWRM_VERSION_MINOR;
5330         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
5331         mutex_lock(&bp->hwrm_cmd_lock);
5332         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5333         if (rc)
5334                 goto hwrm_ver_get_exit;
5335
5336         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
5337
5338         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
5339                              resp->hwrm_intf_min_8b << 8 |
5340                              resp->hwrm_intf_upd_8b;
5341         if (resp->hwrm_intf_maj_8b < 1) {
5342                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
5343                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
5344                             resp->hwrm_intf_upd_8b);
5345                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
5346         }
5347         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
5348                  resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
5349                  resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
5350
5351         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
5352         if (!bp->hwrm_cmd_timeout)
5353                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
5354
5355         if (resp->hwrm_intf_maj_8b >= 1)
5356                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
5357
5358         bp->chip_num = le16_to_cpu(resp->chip_num);
5359         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
5360             !resp->chip_metal)
5361                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
5362
5363         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
5364         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
5365             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
5366                 bp->flags |= BNXT_FLAG_SHORT_CMD;
5367
5368 hwrm_ver_get_exit:
5369         mutex_unlock(&bp->hwrm_cmd_lock);
5370         return rc;
5371 }
5372
5373 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
5374 {
5375         struct hwrm_fw_set_time_input req = {0};
5376         struct tm tm;
5377         time64_t now = ktime_get_real_seconds();
5378
5379         if (bp->hwrm_spec_code < 0x10400)
5380                 return -EOPNOTSUPP;
5381
5382         time64_to_tm(now, 0, &tm);
5383         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
5384         req.year = cpu_to_le16(1900 + tm.tm_year);
5385         req.month = 1 + tm.tm_mon;
5386         req.day = tm.tm_mday;
5387         req.hour = tm.tm_hour;
5388         req.minute = tm.tm_min;
5389         req.second = tm.tm_sec;
5390         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5391 }
5392
5393 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
5394 {
5395         int rc;
5396         struct bnxt_pf_info *pf = &bp->pf;
5397         struct hwrm_port_qstats_input req = {0};
5398
5399         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
5400                 return 0;
5401
5402         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
5403         req.port_id = cpu_to_le16(pf->port_id);
5404         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
5405         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
5406         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5407         return rc;
5408 }
5409
5410 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
5411 {
5412         struct hwrm_port_qstats_ext_input req = {0};
5413         struct bnxt_pf_info *pf = &bp->pf;
5414
5415         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
5416                 return 0;
5417
5418         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
5419         req.port_id = cpu_to_le16(pf->port_id);
5420         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
5421         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
5422         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5423 }
5424
5425 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
5426 {
5427         if (bp->vxlan_port_cnt) {
5428                 bnxt_hwrm_tunnel_dst_port_free(
5429                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5430         }
5431         bp->vxlan_port_cnt = 0;
5432         if (bp->nge_port_cnt) {
5433                 bnxt_hwrm_tunnel_dst_port_free(
5434                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
5435         }
5436         bp->nge_port_cnt = 0;
5437 }
5438
5439 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
5440 {
5441         int rc, i;
5442         u32 tpa_flags = 0;
5443
5444         if (set_tpa)
5445                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
5446         for (i = 0; i < bp->nr_vnics; i++) {
5447                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
5448                 if (rc) {
5449                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
5450                                    i, rc);
5451                         return rc;
5452                 }
5453         }
5454         return 0;
5455 }
5456
5457 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
5458 {
5459         int i;
5460
5461         for (i = 0; i < bp->nr_vnics; i++)
5462                 bnxt_hwrm_vnic_set_rss(bp, i, false);
5463 }
5464
5465 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
5466                                     bool irq_re_init)
5467 {
5468         if (bp->vnic_info) {
5469                 bnxt_hwrm_clear_vnic_filter(bp);
5470                 /* clear all RSS setting before free vnic ctx */
5471                 bnxt_hwrm_clear_vnic_rss(bp);
5472                 bnxt_hwrm_vnic_ctx_free(bp);
5473                 /* before free the vnic, undo the vnic tpa settings */
5474                 if (bp->flags & BNXT_FLAG_TPA)
5475                         bnxt_set_tpa(bp, false);
5476                 bnxt_hwrm_vnic_free(bp);
5477         }
5478         bnxt_hwrm_ring_free(bp, close_path);
5479         bnxt_hwrm_ring_grp_free(bp);
5480         if (irq_re_init) {
5481                 bnxt_hwrm_stat_ctx_free(bp);
5482                 bnxt_hwrm_free_tunnel_ports(bp);
5483         }
5484 }
5485
5486 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
5487 {
5488         struct hwrm_func_cfg_input req = {0};
5489         int rc;
5490
5491         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5492         req.fid = cpu_to_le16(0xffff);
5493         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
5494         if (br_mode == BRIDGE_MODE_VEB)
5495                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
5496         else if (br_mode == BRIDGE_MODE_VEPA)
5497                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
5498         else
5499                 return -EINVAL;
5500         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5501         if (rc)
5502                 rc = -EIO;
5503         return rc;
5504 }
5505
5506 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
5507 {
5508         struct hwrm_func_cfg_input req = {0};
5509         int rc;
5510
5511         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
5512                 return 0;
5513
5514         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5515         req.fid = cpu_to_le16(0xffff);
5516         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
5517         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
5518         if (size == 128)
5519                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
5520
5521         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5522         if (rc)
5523                 rc = -EIO;
5524         return rc;
5525 }
5526
5527 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
5528 {
5529         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5530         int rc;
5531
5532         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
5533                 goto skip_rss_ctx;
5534
5535         /* allocate context for vnic */
5536         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
5537         if (rc) {
5538                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5539                            vnic_id, rc);
5540                 goto vnic_setup_err;
5541         }
5542         bp->rsscos_nr_ctxs++;
5543
5544         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5545                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
5546                 if (rc) {
5547                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
5548                                    vnic_id, rc);
5549                         goto vnic_setup_err;
5550                 }
5551                 bp->rsscos_nr_ctxs++;
5552         }
5553
5554 skip_rss_ctx:
5555         /* configure default vnic, ring grp */
5556         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
5557         if (rc) {
5558                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
5559                            vnic_id, rc);
5560                 goto vnic_setup_err;
5561         }
5562
5563         /* Enable RSS hashing on vnic */
5564         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
5565         if (rc) {
5566                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
5567                            vnic_id, rc);
5568                 goto vnic_setup_err;
5569         }
5570
5571         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5572                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
5573                 if (rc) {
5574                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
5575                                    vnic_id, rc);
5576                 }
5577         }
5578
5579 vnic_setup_err:
5580         return rc;
5581 }
5582
5583 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
5584 {
5585 #ifdef CONFIG_RFS_ACCEL
5586         int i, rc = 0;
5587
5588         for (i = 0; i < bp->rx_nr_rings; i++) {
5589                 struct bnxt_vnic_info *vnic;
5590                 u16 vnic_id = i + 1;
5591                 u16 ring_id = i;
5592
5593                 if (vnic_id >= bp->nr_vnics)
5594                         break;
5595
5596                 vnic = &bp->vnic_info[vnic_id];
5597                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
5598                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
5599                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
5600                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
5601                 if (rc) {
5602                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
5603                                    vnic_id, rc);
5604                         break;
5605                 }
5606                 rc = bnxt_setup_vnic(bp, vnic_id);
5607                 if (rc)
5608                         break;
5609         }
5610         return rc;
5611 #else
5612         return 0;
5613 #endif
5614 }
5615
5616 /* Allow PF and VF with default VLAN to be in promiscuous mode */
5617 static bool bnxt_promisc_ok(struct bnxt *bp)
5618 {
5619 #ifdef CONFIG_BNXT_SRIOV
5620         if (BNXT_VF(bp) && !bp->vf.vlan)
5621                 return false;
5622 #endif
5623         return true;
5624 }
5625
5626 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
5627 {
5628         unsigned int rc = 0;
5629
5630         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
5631         if (rc) {
5632                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5633                            rc);
5634                 return rc;
5635         }
5636
5637         rc = bnxt_hwrm_vnic_cfg(bp, 1);
5638         if (rc) {
5639                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
5640                            rc);
5641                 return rc;
5642         }
5643         return rc;
5644 }
5645
5646 static int bnxt_cfg_rx_mode(struct bnxt *);
5647 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
5648
5649 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
5650 {
5651         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5652         int rc = 0;
5653         unsigned int rx_nr_rings = bp->rx_nr_rings;
5654
5655         if (irq_re_init) {
5656                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
5657                 if (rc) {
5658                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
5659                                    rc);
5660                         goto err_out;
5661                 }
5662         }
5663
5664         rc = bnxt_hwrm_ring_alloc(bp);
5665         if (rc) {
5666                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
5667                 goto err_out;
5668         }
5669
5670         rc = bnxt_hwrm_ring_grp_alloc(bp);
5671         if (rc) {
5672                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
5673                 goto err_out;
5674         }
5675
5676         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5677                 rx_nr_rings--;
5678
5679         /* default vnic 0 */
5680         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
5681         if (rc) {
5682                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
5683                 goto err_out;
5684         }
5685
5686         rc = bnxt_setup_vnic(bp, 0);
5687         if (rc)
5688                 goto err_out;
5689
5690         if (bp->flags & BNXT_FLAG_RFS) {
5691                 rc = bnxt_alloc_rfs_vnics(bp);
5692                 if (rc)
5693                         goto err_out;
5694         }
5695
5696         if (bp->flags & BNXT_FLAG_TPA) {
5697                 rc = bnxt_set_tpa(bp, true);
5698                 if (rc)
5699                         goto err_out;
5700         }
5701
5702         if (BNXT_VF(bp))
5703                 bnxt_update_vf_mac(bp);
5704
5705         /* Filter for default vnic 0 */
5706         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
5707         if (rc) {
5708                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
5709                 goto err_out;
5710         }
5711         vnic->uc_filter_count = 1;
5712
5713         vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
5714
5715         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
5716                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5717
5718         if (bp->dev->flags & IFF_ALLMULTI) {
5719                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5720                 vnic->mc_list_count = 0;
5721         } else {
5722                 u32 mask = 0;
5723
5724                 bnxt_mc_list_updated(bp, &mask);
5725                 vnic->rx_mask |= mask;
5726         }
5727
5728         rc = bnxt_cfg_rx_mode(bp);
5729         if (rc)
5730                 goto err_out;
5731
5732         rc = bnxt_hwrm_set_coal(bp);
5733         if (rc)
5734                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
5735                                 rc);
5736
5737         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
5738                 rc = bnxt_setup_nitroa0_vnic(bp);
5739                 if (rc)
5740                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
5741                                    rc);
5742         }
5743
5744         if (BNXT_VF(bp)) {
5745                 bnxt_hwrm_func_qcfg(bp);
5746                 netdev_update_features(bp->dev);
5747         }
5748
5749         return 0;
5750
5751 err_out:
5752         bnxt_hwrm_resource_free(bp, 0, true);
5753
5754         return rc;
5755 }
5756
5757 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
5758 {
5759         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
5760         return 0;
5761 }
5762
5763 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
5764 {
5765         bnxt_init_cp_rings(bp);
5766         bnxt_init_rx_rings(bp);
5767         bnxt_init_tx_rings(bp);
5768         bnxt_init_ring_grps(bp, irq_re_init);
5769         bnxt_init_vnics(bp);
5770
5771         return bnxt_init_chip(bp, irq_re_init);
5772 }
5773
5774 static int bnxt_set_real_num_queues(struct bnxt *bp)
5775 {
5776         int rc;
5777         struct net_device *dev = bp->dev;
5778
5779         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
5780                                           bp->tx_nr_rings_xdp);
5781         if (rc)
5782                 return rc;
5783
5784         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
5785         if (rc)
5786                 return rc;
5787
5788 #ifdef CONFIG_RFS_ACCEL
5789         if (bp->flags & BNXT_FLAG_RFS)
5790                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
5791 #endif
5792
5793         return rc;
5794 }
5795
5796 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5797                            bool shared)
5798 {
5799         int _rx = *rx, _tx = *tx;
5800
5801         if (shared) {
5802                 *rx = min_t(int, _rx, max);
5803                 *tx = min_t(int, _tx, max);
5804         } else {
5805                 if (max < 2)
5806                         return -ENOMEM;
5807
5808                 while (_rx + _tx > max) {
5809                         if (_rx > _tx && _rx > 1)
5810                                 _rx--;
5811                         else if (_tx > 1)
5812                                 _tx--;
5813                 }
5814                 *rx = _rx;
5815                 *tx = _tx;
5816         }
5817         return 0;
5818 }
5819
5820 static void bnxt_setup_msix(struct bnxt *bp)
5821 {
5822         const int len = sizeof(bp->irq_tbl[0].name);
5823         struct net_device *dev = bp->dev;
5824         int tcs, i;
5825
5826         tcs = netdev_get_num_tc(dev);
5827         if (tcs > 1) {
5828                 int i, off, count;
5829
5830                 for (i = 0; i < tcs; i++) {
5831                         count = bp->tx_nr_rings_per_tc;
5832                         off = i * count;
5833                         netdev_set_tc_queue(dev, i, count, off);
5834                 }
5835         }
5836
5837         for (i = 0; i < bp->cp_nr_rings; i++) {
5838                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5839                 char *attr;
5840
5841                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5842                         attr = "TxRx";
5843                 else if (i < bp->rx_nr_rings)
5844                         attr = "rx";
5845                 else
5846                         attr = "tx";
5847
5848                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
5849                          attr, i);
5850                 bp->irq_tbl[map_idx].handler = bnxt_msix;
5851         }
5852 }
5853
5854 static void bnxt_setup_inta(struct bnxt *bp)
5855 {
5856         const int len = sizeof(bp->irq_tbl[0].name);
5857
5858         if (netdev_get_num_tc(bp->dev))
5859                 netdev_reset_tc(bp->dev);
5860
5861         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
5862                  0);
5863         bp->irq_tbl[0].handler = bnxt_inta;
5864 }
5865
5866 static int bnxt_setup_int_mode(struct bnxt *bp)
5867 {
5868         int rc;
5869
5870         if (bp->flags & BNXT_FLAG_USING_MSIX)
5871                 bnxt_setup_msix(bp);
5872         else
5873                 bnxt_setup_inta(bp);
5874
5875         rc = bnxt_set_real_num_queues(bp);
5876         return rc;
5877 }
5878
5879 #ifdef CONFIG_RFS_ACCEL
5880 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
5881 {
5882         return bp->hw_resc.max_rsscos_ctxs;
5883 }
5884
5885 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
5886 {
5887         return bp->hw_resc.max_vnics;
5888 }
5889 #endif
5890
5891 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
5892 {
5893         return bp->hw_resc.max_stat_ctxs;
5894 }
5895
5896 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
5897 {
5898         bp->hw_resc.max_stat_ctxs = max;
5899 }
5900
5901 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
5902 {
5903         return bp->hw_resc.max_cp_rings;
5904 }
5905
5906 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max)
5907 {
5908         bp->hw_resc.max_cp_rings = max;
5909 }
5910
5911 unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
5912 {
5913         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5914
5915         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
5916 }
5917
5918 void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
5919 {
5920         bp->hw_resc.max_irqs = max_irqs;
5921 }
5922
5923 int bnxt_get_avail_msix(struct bnxt *bp, int num)
5924 {
5925         int max_cp = bnxt_get_max_func_cp_rings(bp);
5926         int max_irq = bnxt_get_max_func_irqs(bp);
5927         int total_req = bp->cp_nr_rings + num;
5928         int max_idx, avail_msix;
5929
5930         max_idx = min_t(int, bp->total_irqs, max_cp);
5931         avail_msix = max_idx - bp->cp_nr_rings;
5932         if (!(bp->flags & BNXT_FLAG_NEW_RM) || avail_msix >= num)
5933                 return avail_msix;
5934
5935         if (max_irq < total_req) {
5936                 num = max_irq - bp->cp_nr_rings;
5937                 if (num <= 0)
5938                         return 0;
5939         }
5940         return num;
5941 }
5942
5943 static int bnxt_get_num_msix(struct bnxt *bp)
5944 {
5945         if (!(bp->flags & BNXT_FLAG_NEW_RM))
5946                 return bnxt_get_max_func_irqs(bp);
5947
5948         return bnxt_cp_rings_in_use(bp);
5949 }
5950
5951 static int bnxt_init_msix(struct bnxt *bp)
5952 {
5953         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
5954         struct msix_entry *msix_ent;
5955
5956         total_vecs = bnxt_get_num_msix(bp);
5957         max = bnxt_get_max_func_irqs(bp);
5958         if (total_vecs > max)
5959                 total_vecs = max;
5960
5961         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
5962         if (!msix_ent)
5963                 return -ENOMEM;
5964
5965         for (i = 0; i < total_vecs; i++) {
5966                 msix_ent[i].entry = i;
5967                 msix_ent[i].vector = 0;
5968         }
5969
5970         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
5971                 min = 2;
5972
5973         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
5974         ulp_msix = bnxt_get_ulp_msix_num(bp);
5975         if (total_vecs < 0 || total_vecs < ulp_msix) {
5976                 rc = -ENODEV;
5977                 goto msix_setup_exit;
5978         }
5979
5980         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
5981         if (bp->irq_tbl) {
5982                 for (i = 0; i < total_vecs; i++)
5983                         bp->irq_tbl[i].vector = msix_ent[i].vector;
5984
5985                 bp->total_irqs = total_vecs;
5986                 /* Trim rings based upon num of vectors allocated */
5987                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
5988                                      total_vecs - ulp_msix, min == 1);
5989                 if (rc)
5990                         goto msix_setup_exit;
5991
5992                 bp->cp_nr_rings = (min == 1) ?
5993                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
5994                                   bp->tx_nr_rings + bp->rx_nr_rings;
5995
5996         } else {
5997                 rc = -ENOMEM;
5998                 goto msix_setup_exit;
5999         }
6000         bp->flags |= BNXT_FLAG_USING_MSIX;
6001         kfree(msix_ent);
6002         return 0;
6003
6004 msix_setup_exit:
6005         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
6006         kfree(bp->irq_tbl);
6007         bp->irq_tbl = NULL;
6008         pci_disable_msix(bp->pdev);
6009         kfree(msix_ent);
6010         return rc;
6011 }
6012
6013 static int bnxt_init_inta(struct bnxt *bp)
6014 {
6015         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
6016         if (!bp->irq_tbl)
6017                 return -ENOMEM;
6018
6019         bp->total_irqs = 1;
6020         bp->rx_nr_rings = 1;
6021         bp->tx_nr_rings = 1;
6022         bp->cp_nr_rings = 1;
6023         bp->flags |= BNXT_FLAG_SHARED_RINGS;
6024         bp->irq_tbl[0].vector = bp->pdev->irq;
6025         return 0;
6026 }
6027
6028 static int bnxt_init_int_mode(struct bnxt *bp)
6029 {
6030         int rc = 0;
6031
6032         if (bp->flags & BNXT_FLAG_MSIX_CAP)
6033                 rc = bnxt_init_msix(bp);
6034
6035         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
6036                 /* fallback to INTA */
6037                 rc = bnxt_init_inta(bp);
6038         }
6039         return rc;
6040 }
6041
6042 static void bnxt_clear_int_mode(struct bnxt *bp)
6043 {
6044         if (bp->flags & BNXT_FLAG_USING_MSIX)
6045                 pci_disable_msix(bp->pdev);
6046
6047         kfree(bp->irq_tbl);
6048         bp->irq_tbl = NULL;
6049         bp->flags &= ~BNXT_FLAG_USING_MSIX;
6050 }
6051
6052 int bnxt_reserve_rings(struct bnxt *bp)
6053 {
6054         int tcs = netdev_get_num_tc(bp->dev);
6055         int rc;
6056
6057         if (!bnxt_need_reserve_rings(bp))
6058                 return 0;
6059
6060         rc = __bnxt_reserve_rings(bp);
6061         if (rc) {
6062                 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc);
6063                 return rc;
6064         }
6065         if ((bp->flags & BNXT_FLAG_NEW_RM) &&
6066             (bnxt_get_num_msix(bp) != bp->total_irqs)) {
6067                 bnxt_ulp_irq_stop(bp);
6068                 bnxt_clear_int_mode(bp);
6069                 rc = bnxt_init_int_mode(bp);
6070                 bnxt_ulp_irq_restart(bp, rc);
6071                 if (rc)
6072                         return rc;
6073         }
6074         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
6075                 netdev_err(bp->dev, "tx ring reservation failure\n");
6076                 netdev_reset_tc(bp->dev);
6077                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
6078                 return -ENOMEM;
6079         }
6080         bp->num_stat_ctxs = bp->cp_nr_rings;
6081         return 0;
6082 }
6083
6084 static void bnxt_free_irq(struct bnxt *bp)
6085 {
6086         struct bnxt_irq *irq;
6087         int i;
6088
6089 #ifdef CONFIG_RFS_ACCEL
6090         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
6091         bp->dev->rx_cpu_rmap = NULL;
6092 #endif
6093         if (!bp->irq_tbl || !bp->bnapi)
6094                 return;
6095
6096         for (i = 0; i < bp->cp_nr_rings; i++) {
6097                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
6098
6099                 irq = &bp->irq_tbl[map_idx];
6100                 if (irq->requested) {
6101                         if (irq->have_cpumask) {
6102                                 irq_set_affinity_hint(irq->vector, NULL);
6103                                 free_cpumask_var(irq->cpu_mask);
6104                                 irq->have_cpumask = 0;
6105                         }
6106                         free_irq(irq->vector, bp->bnapi[i]);
6107                 }
6108
6109                 irq->requested = 0;
6110         }
6111 }
6112
6113 static int bnxt_request_irq(struct bnxt *bp)
6114 {
6115         int i, j, rc = 0;
6116         unsigned long flags = 0;
6117 #ifdef CONFIG_RFS_ACCEL
6118         struct cpu_rmap *rmap;
6119 #endif
6120
6121         rc = bnxt_setup_int_mode(bp);
6122         if (rc) {
6123                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
6124                            rc);
6125                 return rc;
6126         }
6127 #ifdef CONFIG_RFS_ACCEL
6128         rmap = bp->dev->rx_cpu_rmap;
6129 #endif
6130         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
6131                 flags = IRQF_SHARED;
6132
6133         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
6134                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
6135                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
6136
6137 #ifdef CONFIG_RFS_ACCEL
6138                 if (rmap && bp->bnapi[i]->rx_ring) {
6139                         rc = irq_cpu_rmap_add(rmap, irq->vector);
6140                         if (rc)
6141                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
6142                                             j);
6143                         j++;
6144                 }
6145 #endif
6146                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
6147                                  bp->bnapi[i]);
6148                 if (rc)
6149                         break;
6150
6151                 irq->requested = 1;
6152
6153                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
6154                         int numa_node = dev_to_node(&bp->pdev->dev);
6155
6156                         irq->have_cpumask = 1;
6157                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
6158                                         irq->cpu_mask);
6159                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
6160                         if (rc) {
6161                                 netdev_warn(bp->dev,
6162                                             "Set affinity failed, IRQ = %d\n",
6163                                             irq->vector);
6164                                 break;
6165                         }
6166                 }
6167         }
6168         return rc;
6169 }
6170
6171 static void bnxt_del_napi(struct bnxt *bp)
6172 {
6173         int i;
6174
6175         if (!bp->bnapi)
6176                 return;
6177
6178         for (i = 0; i < bp->cp_nr_rings; i++) {
6179                 struct bnxt_napi *bnapi = bp->bnapi[i];
6180
6181                 napi_hash_del(&bnapi->napi);
6182                 netif_napi_del(&bnapi->napi);
6183         }
6184         /* We called napi_hash_del() before netif_napi_del(), we need
6185          * to respect an RCU grace period before freeing napi structures.
6186          */
6187         synchronize_net();
6188 }
6189
6190 static void bnxt_init_napi(struct bnxt *bp)
6191 {
6192         int i;
6193         unsigned int cp_nr_rings = bp->cp_nr_rings;
6194         struct bnxt_napi *bnapi;
6195
6196         if (bp->flags & BNXT_FLAG_USING_MSIX) {
6197                 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6198                         cp_nr_rings--;
6199                 for (i = 0; i < cp_nr_rings; i++) {
6200                         bnapi = bp->bnapi[i];
6201                         netif_napi_add(bp->dev, &bnapi->napi,
6202                                        bnxt_poll, 64);
6203                 }
6204                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
6205                         bnapi = bp->bnapi[cp_nr_rings];
6206                         netif_napi_add(bp->dev, &bnapi->napi,
6207                                        bnxt_poll_nitroa0, 64);
6208                 }
6209         } else {
6210                 bnapi = bp->bnapi[0];
6211                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
6212         }
6213 }
6214
6215 static void bnxt_disable_napi(struct bnxt *bp)
6216 {
6217         int i;
6218
6219         if (!bp->bnapi)
6220                 return;
6221
6222         for (i = 0; i < bp->cp_nr_rings; i++) {
6223                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
6224
6225                 if (bp->bnapi[i]->rx_ring)
6226                         cancel_work_sync(&cpr->dim.work);
6227
6228                 napi_disable(&bp->bnapi[i]->napi);
6229         }
6230 }
6231
6232 static void bnxt_enable_napi(struct bnxt *bp)
6233 {
6234         int i;
6235
6236         for (i = 0; i < bp->cp_nr_rings; i++) {
6237                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
6238                 bp->bnapi[i]->in_reset = false;
6239
6240                 if (bp->bnapi[i]->rx_ring) {
6241                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
6242                         cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6243                 }
6244                 napi_enable(&bp->bnapi[i]->napi);
6245         }
6246 }
6247
6248 void bnxt_tx_disable(struct bnxt *bp)
6249 {
6250         int i;
6251         struct bnxt_tx_ring_info *txr;
6252
6253         if (bp->tx_ring) {
6254                 for (i = 0; i < bp->tx_nr_rings; i++) {
6255                         txr = &bp->tx_ring[i];
6256                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
6257                 }
6258         }
6259         /* Stop all TX queues */
6260         netif_tx_disable(bp->dev);
6261         netif_carrier_off(bp->dev);
6262 }
6263
6264 void bnxt_tx_enable(struct bnxt *bp)
6265 {
6266         int i;
6267         struct bnxt_tx_ring_info *txr;
6268
6269         for (i = 0; i < bp->tx_nr_rings; i++) {
6270                 txr = &bp->tx_ring[i];
6271                 txr->dev_state = 0;
6272         }
6273         netif_tx_wake_all_queues(bp->dev);
6274         if (bp->link_info.link_up)
6275                 netif_carrier_on(bp->dev);
6276 }
6277
6278 static void bnxt_report_link(struct bnxt *bp)
6279 {
6280         if (bp->link_info.link_up) {
6281                 const char *duplex;
6282                 const char *flow_ctrl;
6283                 u32 speed;
6284                 u16 fec;
6285
6286                 netif_carrier_on(bp->dev);
6287                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
6288                         duplex = "full";
6289                 else
6290                         duplex = "half";
6291                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
6292                         flow_ctrl = "ON - receive & transmit";
6293                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
6294                         flow_ctrl = "ON - transmit";
6295                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
6296                         flow_ctrl = "ON - receive";
6297                 else
6298                         flow_ctrl = "none";
6299                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
6300                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
6301                             speed, duplex, flow_ctrl);
6302                 if (bp->flags & BNXT_FLAG_EEE_CAP)
6303                         netdev_info(bp->dev, "EEE is %s\n",
6304                                     bp->eee.eee_active ? "active" :
6305                                                          "not active");
6306                 fec = bp->link_info.fec_cfg;
6307                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
6308                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
6309                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
6310                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
6311                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
6312         } else {
6313                 netif_carrier_off(bp->dev);
6314                 netdev_err(bp->dev, "NIC Link is Down\n");
6315         }
6316 }
6317
6318 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
6319 {
6320         int rc = 0;
6321         struct hwrm_port_phy_qcaps_input req = {0};
6322         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6323         struct bnxt_link_info *link_info = &bp->link_info;
6324
6325         if (bp->hwrm_spec_code < 0x10201)
6326                 return 0;
6327
6328         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
6329
6330         mutex_lock(&bp->hwrm_cmd_lock);
6331         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6332         if (rc)
6333                 goto hwrm_phy_qcaps_exit;
6334
6335         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
6336                 struct ethtool_eee *eee = &bp->eee;
6337                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
6338
6339                 bp->flags |= BNXT_FLAG_EEE_CAP;
6340                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
6341                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
6342                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
6343                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
6344                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
6345         }
6346         if (resp->supported_speeds_auto_mode)
6347                 link_info->support_auto_speeds =
6348                         le16_to_cpu(resp->supported_speeds_auto_mode);
6349
6350         bp->port_count = resp->port_cnt;
6351
6352 hwrm_phy_qcaps_exit:
6353         mutex_unlock(&bp->hwrm_cmd_lock);
6354         return rc;
6355 }
6356
6357 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
6358 {
6359         int rc = 0;
6360         struct bnxt_link_info *link_info = &bp->link_info;
6361         struct hwrm_port_phy_qcfg_input req = {0};
6362         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6363         u8 link_up = link_info->link_up;
6364         u16 diff;
6365
6366         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
6367
6368         mutex_lock(&bp->hwrm_cmd_lock);
6369         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6370         if (rc) {
6371                 mutex_unlock(&bp->hwrm_cmd_lock);
6372                 return rc;
6373         }
6374
6375         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
6376         link_info->phy_link_status = resp->link;
6377         link_info->duplex = resp->duplex_cfg;
6378         if (bp->hwrm_spec_code >= 0x10800)
6379                 link_info->duplex = resp->duplex_state;
6380         link_info->pause = resp->pause;
6381         link_info->auto_mode = resp->auto_mode;
6382         link_info->auto_pause_setting = resp->auto_pause;
6383         link_info->lp_pause = resp->link_partner_adv_pause;
6384         link_info->force_pause_setting = resp->force_pause;
6385         link_info->duplex_setting = resp->duplex_cfg;
6386         if (link_info->phy_link_status == BNXT_LINK_LINK)
6387                 link_info->link_speed = le16_to_cpu(resp->link_speed);
6388         else
6389                 link_info->link_speed = 0;
6390         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
6391         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
6392         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
6393         link_info->lp_auto_link_speeds =
6394                 le16_to_cpu(resp->link_partner_adv_speeds);
6395         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
6396         link_info->phy_ver[0] = resp->phy_maj;
6397         link_info->phy_ver[1] = resp->phy_min;
6398         link_info->phy_ver[2] = resp->phy_bld;
6399         link_info->media_type = resp->media_type;
6400         link_info->phy_type = resp->phy_type;
6401         link_info->transceiver = resp->xcvr_pkg_type;
6402         link_info->phy_addr = resp->eee_config_phy_addr &
6403                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
6404         link_info->module_status = resp->module_status;
6405
6406         if (bp->flags & BNXT_FLAG_EEE_CAP) {
6407                 struct ethtool_eee *eee = &bp->eee;
6408                 u16 fw_speeds;
6409
6410                 eee->eee_active = 0;
6411                 if (resp->eee_config_phy_addr &
6412                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
6413                         eee->eee_active = 1;
6414                         fw_speeds = le16_to_cpu(
6415                                 resp->link_partner_adv_eee_link_speed_mask);
6416                         eee->lp_advertised =
6417                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
6418                 }
6419
6420                 /* Pull initial EEE config */
6421                 if (!chng_link_state) {
6422                         if (resp->eee_config_phy_addr &
6423                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
6424                                 eee->eee_enabled = 1;
6425
6426                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
6427                         eee->advertised =
6428                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
6429
6430                         if (resp->eee_config_phy_addr &
6431                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
6432                                 __le32 tmr;
6433
6434                                 eee->tx_lpi_enabled = 1;
6435                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
6436                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
6437                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
6438                         }
6439                 }
6440         }
6441
6442         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
6443         if (bp->hwrm_spec_code >= 0x10504)
6444                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
6445
6446         /* TODO: need to add more logic to report VF link */
6447         if (chng_link_state) {
6448                 if (link_info->phy_link_status == BNXT_LINK_LINK)
6449                         link_info->link_up = 1;
6450                 else
6451                         link_info->link_up = 0;
6452                 if (link_up != link_info->link_up)
6453                         bnxt_report_link(bp);
6454         } else {
6455                 /* alwasy link down if not require to update link state */
6456                 link_info->link_up = 0;
6457         }
6458         mutex_unlock(&bp->hwrm_cmd_lock);
6459
6460         diff = link_info->support_auto_speeds ^ link_info->advertising;
6461         if ((link_info->support_auto_speeds | diff) !=
6462             link_info->support_auto_speeds) {
6463                 /* An advertised speed is no longer supported, so we need to
6464                  * update the advertisement settings.  Caller holds RTNL
6465                  * so we can modify link settings.
6466                  */
6467                 link_info->advertising = link_info->support_auto_speeds;
6468                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
6469                         bnxt_hwrm_set_link_setting(bp, true, false);
6470         }
6471         return 0;
6472 }
6473
6474 static void bnxt_get_port_module_status(struct bnxt *bp)
6475 {
6476         struct bnxt_link_info *link_info = &bp->link_info;
6477         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
6478         u8 module_status;
6479
6480         if (bnxt_update_link(bp, true))
6481                 return;
6482
6483         module_status = link_info->module_status;
6484         switch (module_status) {
6485         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
6486         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
6487         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
6488                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
6489                             bp->pf.port_id);
6490                 if (bp->hwrm_spec_code >= 0x10201) {
6491                         netdev_warn(bp->dev, "Module part number %s\n",
6492                                     resp->phy_vendor_partnumber);
6493                 }
6494                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
6495                         netdev_warn(bp->dev, "TX is disabled\n");
6496                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
6497                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
6498         }
6499 }
6500
6501 static void
6502 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
6503 {
6504         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
6505                 if (bp->hwrm_spec_code >= 0x10201)
6506                         req->auto_pause =
6507                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
6508                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
6509                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
6510                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
6511                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
6512                 req->enables |=
6513                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
6514         } else {
6515                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
6516                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
6517                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
6518                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
6519                 req->enables |=
6520                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
6521                 if (bp->hwrm_spec_code >= 0x10201) {
6522                         req->auto_pause = req->force_pause;
6523                         req->enables |= cpu_to_le32(
6524                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
6525                 }
6526         }
6527 }
6528
6529 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
6530                                       struct hwrm_port_phy_cfg_input *req)
6531 {
6532         u8 autoneg = bp->link_info.autoneg;
6533         u16 fw_link_speed = bp->link_info.req_link_speed;
6534         u16 advertising = bp->link_info.advertising;
6535
6536         if (autoneg & BNXT_AUTONEG_SPEED) {
6537                 req->auto_mode |=
6538                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
6539
6540                 req->enables |= cpu_to_le32(
6541                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
6542                 req->auto_link_speed_mask = cpu_to_le16(advertising);
6543
6544                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
6545                 req->flags |=
6546                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
6547         } else {
6548                 req->force_link_speed = cpu_to_le16(fw_link_speed);
6549                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
6550         }
6551
6552         /* tell chimp that the setting takes effect immediately */
6553         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
6554 }
6555
6556 int bnxt_hwrm_set_pause(struct bnxt *bp)
6557 {
6558         struct hwrm_port_phy_cfg_input req = {0};
6559         int rc;
6560
6561         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6562         bnxt_hwrm_set_pause_common(bp, &req);
6563
6564         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
6565             bp->link_info.force_link_chng)
6566                 bnxt_hwrm_set_link_common(bp, &req);
6567
6568         mutex_lock(&bp->hwrm_cmd_lock);
6569         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6570         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
6571                 /* since changing of pause setting doesn't trigger any link
6572                  * change event, the driver needs to update the current pause
6573                  * result upon successfully return of the phy_cfg command
6574                  */
6575                 bp->link_info.pause =
6576                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
6577                 bp->link_info.auto_pause_setting = 0;
6578                 if (!bp->link_info.force_link_chng)
6579                         bnxt_report_link(bp);
6580         }
6581         bp->link_info.force_link_chng = false;
6582         mutex_unlock(&bp->hwrm_cmd_lock);
6583         return rc;
6584 }
6585
6586 static void bnxt_hwrm_set_eee(struct bnxt *bp,
6587                               struct hwrm_port_phy_cfg_input *req)
6588 {
6589         struct ethtool_eee *eee = &bp->eee;
6590
6591         if (eee->eee_enabled) {
6592                 u16 eee_speeds;
6593                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
6594
6595                 if (eee->tx_lpi_enabled)
6596                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
6597                 else
6598                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
6599
6600                 req->flags |= cpu_to_le32(flags);
6601                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
6602                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
6603                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
6604         } else {
6605                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
6606         }
6607 }
6608
6609 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
6610 {
6611         struct hwrm_port_phy_cfg_input req = {0};
6612
6613         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6614         if (set_pause)
6615                 bnxt_hwrm_set_pause_common(bp, &req);
6616
6617         bnxt_hwrm_set_link_common(bp, &req);
6618
6619         if (set_eee)
6620                 bnxt_hwrm_set_eee(bp, &req);
6621         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6622 }
6623
6624 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
6625 {
6626         struct hwrm_port_phy_cfg_input req = {0};
6627
6628         if (!BNXT_SINGLE_PF(bp))
6629                 return 0;
6630
6631         if (pci_num_vf(bp->pdev))
6632                 return 0;
6633
6634         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
6635         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
6636         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6637 }
6638
6639 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
6640 {
6641         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6642         struct hwrm_port_led_qcaps_input req = {0};
6643         struct bnxt_pf_info *pf = &bp->pf;
6644         int rc;
6645
6646         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
6647                 return 0;
6648
6649         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
6650         req.port_id = cpu_to_le16(pf->port_id);
6651         mutex_lock(&bp->hwrm_cmd_lock);
6652         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6653         if (rc) {
6654                 mutex_unlock(&bp->hwrm_cmd_lock);
6655                 return rc;
6656         }
6657         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
6658                 int i;
6659
6660                 bp->num_leds = resp->num_leds;
6661                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
6662                                                  bp->num_leds);
6663                 for (i = 0; i < bp->num_leds; i++) {
6664                         struct bnxt_led_info *led = &bp->leds[i];
6665                         __le16 caps = led->led_state_caps;
6666
6667                         if (!led->led_group_id ||
6668                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
6669                                 bp->num_leds = 0;
6670                                 break;
6671                         }
6672                 }
6673         }
6674         mutex_unlock(&bp->hwrm_cmd_lock);
6675         return 0;
6676 }
6677
6678 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
6679 {
6680         struct hwrm_wol_filter_alloc_input req = {0};
6681         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6682         int rc;
6683
6684         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
6685         req.port_id = cpu_to_le16(bp->pf.port_id);
6686         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
6687         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
6688         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
6689         mutex_lock(&bp->hwrm_cmd_lock);
6690         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6691         if (!rc)
6692                 bp->wol_filter_id = resp->wol_filter_id;
6693         mutex_unlock(&bp->hwrm_cmd_lock);
6694         return rc;
6695 }
6696
6697 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
6698 {
6699         struct hwrm_wol_filter_free_input req = {0};
6700         int rc;
6701
6702         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
6703         req.port_id = cpu_to_le16(bp->pf.port_id);
6704         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
6705         req.wol_filter_id = bp->wol_filter_id;
6706         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6707         return rc;
6708 }
6709
6710 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
6711 {
6712         struct hwrm_wol_filter_qcfg_input req = {0};
6713         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6714         u16 next_handle = 0;
6715         int rc;
6716
6717         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
6718         req.port_id = cpu_to_le16(bp->pf.port_id);
6719         req.handle = cpu_to_le16(handle);
6720         mutex_lock(&bp->hwrm_cmd_lock);
6721         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6722         if (!rc) {
6723                 next_handle = le16_to_cpu(resp->next_handle);
6724                 if (next_handle != 0) {
6725                         if (resp->wol_type ==
6726                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
6727                                 bp->wol = 1;
6728                                 bp->wol_filter_id = resp->wol_filter_id;
6729                         }
6730                 }
6731         }
6732         mutex_unlock(&bp->hwrm_cmd_lock);
6733         return next_handle;
6734 }
6735
6736 static void bnxt_get_wol_settings(struct bnxt *bp)
6737 {
6738         u16 handle = 0;
6739
6740         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
6741                 return;
6742
6743         do {
6744                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
6745         } while (handle && handle != 0xffff);
6746 }
6747
6748 static bool bnxt_eee_config_ok(struct bnxt *bp)
6749 {
6750         struct ethtool_eee *eee = &bp->eee;
6751         struct bnxt_link_info *link_info = &bp->link_info;
6752
6753         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
6754                 return true;
6755
6756         if (eee->eee_enabled) {
6757                 u32 advertising =
6758                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
6759
6760                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6761                         eee->eee_enabled = 0;
6762                         return false;
6763                 }
6764                 if (eee->advertised & ~advertising) {
6765                         eee->advertised = advertising & eee->supported;
6766                         return false;
6767                 }
6768         }
6769         return true;
6770 }
6771
6772 static int bnxt_update_phy_setting(struct bnxt *bp)
6773 {
6774         int rc;
6775         bool update_link = false;
6776         bool update_pause = false;
6777         bool update_eee = false;
6778         struct bnxt_link_info *link_info = &bp->link_info;
6779
6780         rc = bnxt_update_link(bp, true);
6781         if (rc) {
6782                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
6783                            rc);
6784                 return rc;
6785         }
6786         if (!BNXT_SINGLE_PF(bp))
6787                 return 0;
6788
6789         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6790             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
6791             link_info->req_flow_ctrl)
6792                 update_pause = true;
6793         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
6794             link_info->force_pause_setting != link_info->req_flow_ctrl)
6795                 update_pause = true;
6796         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
6797                 if (BNXT_AUTO_MODE(link_info->auto_mode))
6798                         update_link = true;
6799                 if (link_info->req_link_speed != link_info->force_link_speed)
6800                         update_link = true;
6801                 if (link_info->req_duplex != link_info->duplex_setting)
6802                         update_link = true;
6803         } else {
6804                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
6805                         update_link = true;
6806                 if (link_info->advertising != link_info->auto_link_speeds)
6807                         update_link = true;
6808         }
6809
6810         /* The last close may have shutdown the link, so need to call
6811          * PHY_CFG to bring it back up.
6812          */
6813         if (!netif_carrier_ok(bp->dev))
6814                 update_link = true;
6815
6816         if (!bnxt_eee_config_ok(bp))
6817                 update_eee = true;
6818
6819         if (update_link)
6820                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
6821         else if (update_pause)
6822                 rc = bnxt_hwrm_set_pause(bp);
6823         if (rc) {
6824                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
6825                            rc);
6826                 return rc;
6827         }
6828
6829         return rc;
6830 }
6831
6832 /* Common routine to pre-map certain register block to different GRC window.
6833  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
6834  * in PF and 3 windows in VF that can be customized to map in different
6835  * register blocks.
6836  */
6837 static void bnxt_preset_reg_win(struct bnxt *bp)
6838 {
6839         if (BNXT_PF(bp)) {
6840                 /* CAG registers map to GRC window #4 */
6841                 writel(BNXT_CAG_REG_BASE,
6842                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
6843         }
6844 }
6845
6846 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6847 {
6848         int rc = 0;
6849
6850         bnxt_preset_reg_win(bp);
6851         netif_carrier_off(bp->dev);
6852         if (irq_re_init) {
6853                 rc = bnxt_reserve_rings(bp);
6854                 if (rc)
6855                         return rc;
6856         }
6857         if ((bp->flags & BNXT_FLAG_RFS) &&
6858             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
6859                 /* disable RFS if falling back to INTA */
6860                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
6861                 bp->flags &= ~BNXT_FLAG_RFS;
6862         }
6863
6864         rc = bnxt_alloc_mem(bp, irq_re_init);
6865         if (rc) {
6866                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6867                 goto open_err_free_mem;
6868         }
6869
6870         if (irq_re_init) {
6871                 bnxt_init_napi(bp);
6872                 rc = bnxt_request_irq(bp);
6873                 if (rc) {
6874                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
6875                         goto open_err;
6876                 }
6877         }
6878
6879         bnxt_enable_napi(bp);
6880
6881         rc = bnxt_init_nic(bp, irq_re_init);
6882         if (rc) {
6883                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6884                 goto open_err;
6885         }
6886
6887         if (link_re_init) {
6888                 mutex_lock(&bp->link_lock);
6889                 rc = bnxt_update_phy_setting(bp);
6890                 mutex_unlock(&bp->link_lock);
6891                 if (rc)
6892                         netdev_warn(bp->dev, "failed to update phy settings\n");
6893         }
6894
6895         if (irq_re_init)
6896                 udp_tunnel_get_rx_info(bp->dev);
6897
6898         set_bit(BNXT_STATE_OPEN, &bp->state);
6899         bnxt_enable_int(bp);
6900         /* Enable TX queues */
6901         bnxt_tx_enable(bp);
6902         mod_timer(&bp->timer, jiffies + bp->current_interval);
6903         /* Poll link status and check for SFP+ module status */
6904         bnxt_get_port_module_status(bp);
6905
6906         /* VF-reps may need to be re-opened after the PF is re-opened */
6907         if (BNXT_PF(bp))
6908                 bnxt_vf_reps_open(bp);
6909         return 0;
6910
6911 open_err:
6912         bnxt_disable_napi(bp);
6913         bnxt_del_napi(bp);
6914
6915 open_err_free_mem:
6916         bnxt_free_skbs(bp);
6917         bnxt_free_irq(bp);
6918         bnxt_free_mem(bp, true);
6919         return rc;
6920 }
6921
6922 /* rtnl_lock held */
6923 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
6924 {
6925         int rc = 0;
6926
6927         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
6928         if (rc) {
6929                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
6930                 dev_close(bp->dev);
6931         }
6932         return rc;
6933 }
6934
6935 /* rtnl_lock held, open the NIC half way by allocating all resources, but
6936  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
6937  * self tests.
6938  */
6939 int bnxt_half_open_nic(struct bnxt *bp)
6940 {
6941         int rc = 0;
6942
6943         rc = bnxt_alloc_mem(bp, false);
6944         if (rc) {
6945                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
6946                 goto half_open_err;
6947         }
6948         rc = bnxt_init_nic(bp, false);
6949         if (rc) {
6950                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
6951                 goto half_open_err;
6952         }
6953         return 0;
6954
6955 half_open_err:
6956         bnxt_free_skbs(bp);
6957         bnxt_free_mem(bp, false);
6958         dev_close(bp->dev);
6959         return rc;
6960 }
6961
6962 /* rtnl_lock held, this call can only be made after a previous successful
6963  * call to bnxt_half_open_nic().
6964  */
6965 void bnxt_half_close_nic(struct bnxt *bp)
6966 {
6967         bnxt_hwrm_resource_free(bp, false, false);
6968         bnxt_free_skbs(bp);
6969         bnxt_free_mem(bp, false);
6970 }
6971
6972 static int bnxt_open(struct net_device *dev)
6973 {
6974         struct bnxt *bp = netdev_priv(dev);
6975
6976         return __bnxt_open_nic(bp, true, true);
6977 }
6978
6979 static bool bnxt_drv_busy(struct bnxt *bp)
6980 {
6981         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
6982                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
6983 }
6984
6985 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
6986                              bool link_re_init)
6987 {
6988         /* Close the VF-reps before closing PF */
6989         if (BNXT_PF(bp))
6990                 bnxt_vf_reps_close(bp);
6991
6992         /* Change device state to avoid TX queue wake up's */
6993         bnxt_tx_disable(bp);
6994
6995         clear_bit(BNXT_STATE_OPEN, &bp->state);
6996         smp_mb__after_atomic();
6997         while (bnxt_drv_busy(bp))
6998                 msleep(20);
6999
7000         /* Flush rings and and disable interrupts */
7001         bnxt_shutdown_nic(bp, irq_re_init);
7002
7003         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
7004
7005         bnxt_disable_napi(bp);
7006         del_timer_sync(&bp->timer);
7007         bnxt_free_skbs(bp);
7008
7009         if (irq_re_init) {
7010                 bnxt_free_irq(bp);
7011                 bnxt_del_napi(bp);
7012         }
7013         bnxt_free_mem(bp, irq_re_init);
7014 }
7015
7016 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
7017 {
7018         int rc = 0;
7019
7020 #ifdef CONFIG_BNXT_SRIOV
7021         if (bp->sriov_cfg) {
7022                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
7023                                                       !bp->sriov_cfg,
7024                                                       BNXT_SRIOV_CFG_WAIT_TMO);
7025                 if (rc)
7026                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
7027         }
7028 #endif
7029         __bnxt_close_nic(bp, irq_re_init, link_re_init);
7030         return rc;
7031 }
7032
7033 static int bnxt_close(struct net_device *dev)
7034 {
7035         struct bnxt *bp = netdev_priv(dev);
7036
7037         bnxt_close_nic(bp, true, true);
7038         bnxt_hwrm_shutdown_link(bp);
7039         return 0;
7040 }
7041
7042 /* rtnl_lock held */
7043 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
7044 {
7045         switch (cmd) {
7046         case SIOCGMIIPHY:
7047                 /* fallthru */
7048         case SIOCGMIIREG: {
7049                 if (!netif_running(dev))
7050                         return -EAGAIN;
7051
7052                 return 0;
7053         }
7054
7055         case SIOCSMIIREG:
7056                 if (!netif_running(dev))
7057                         return -EAGAIN;
7058
7059                 return 0;
7060
7061         default:
7062                 /* do nothing */
7063                 break;
7064         }
7065         return -EOPNOTSUPP;
7066 }
7067
7068 static void
7069 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
7070 {
7071         u32 i;
7072         struct bnxt *bp = netdev_priv(dev);
7073
7074         set_bit(BNXT_STATE_READ_STATS, &bp->state);
7075         /* Make sure bnxt_close_nic() sees that we are reading stats before
7076          * we check the BNXT_STATE_OPEN flag.
7077          */
7078         smp_mb__after_atomic();
7079         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7080                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
7081                 return;
7082         }
7083
7084         /* TODO check if we need to synchronize with bnxt_close path */
7085         for (i = 0; i < bp->cp_nr_rings; i++) {
7086                 struct bnxt_napi *bnapi = bp->bnapi[i];
7087                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7088                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
7089
7090                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
7091                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
7092                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
7093
7094                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
7095                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
7096                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
7097
7098                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
7099                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
7100                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
7101
7102                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
7103                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
7104                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
7105
7106                 stats->rx_missed_errors +=
7107                         le64_to_cpu(hw_stats->rx_discard_pkts);
7108
7109                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
7110
7111                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
7112         }
7113
7114         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7115                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
7116                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
7117
7118                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
7119                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
7120                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
7121                                           le64_to_cpu(rx->rx_ovrsz_frames) +
7122                                           le64_to_cpu(rx->rx_runt_frames);
7123                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
7124                                    le64_to_cpu(rx->rx_jbr_frames);
7125                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
7126                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
7127                 stats->tx_errors = le64_to_cpu(tx->tx_err);
7128         }
7129         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
7130 }
7131
7132 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
7133 {
7134         struct net_device *dev = bp->dev;
7135         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7136         struct netdev_hw_addr *ha;
7137         u8 *haddr;
7138         int mc_count = 0;
7139         bool update = false;
7140         int off = 0;
7141
7142         netdev_for_each_mc_addr(ha, dev) {
7143                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
7144                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7145                         vnic->mc_list_count = 0;
7146                         return false;
7147                 }
7148                 haddr = ha->addr;
7149                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
7150                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
7151                         update = true;
7152                 }
7153                 off += ETH_ALEN;
7154                 mc_count++;
7155         }
7156         if (mc_count)
7157                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
7158
7159         if (mc_count != vnic->mc_list_count) {
7160                 vnic->mc_list_count = mc_count;
7161                 update = true;
7162         }
7163         return update;
7164 }
7165
7166 static bool bnxt_uc_list_updated(struct bnxt *bp)
7167 {
7168         struct net_device *dev = bp->dev;
7169         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7170         struct netdev_hw_addr *ha;
7171         int off = 0;
7172
7173         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
7174                 return true;
7175
7176         netdev_for_each_uc_addr(ha, dev) {
7177                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
7178                         return true;
7179
7180                 off += ETH_ALEN;
7181         }
7182         return false;
7183 }
7184
7185 static void bnxt_set_rx_mode(struct net_device *dev)
7186 {
7187         struct bnxt *bp = netdev_priv(dev);
7188         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7189         u32 mask = vnic->rx_mask;
7190         bool mc_update = false;
7191         bool uc_update;
7192
7193         if (!netif_running(dev))
7194                 return;
7195
7196         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
7197                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
7198                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
7199
7200         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7201                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7202
7203         uc_update = bnxt_uc_list_updated(bp);
7204
7205         if (dev->flags & IFF_ALLMULTI) {
7206                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7207                 vnic->mc_list_count = 0;
7208         } else {
7209                 mc_update = bnxt_mc_list_updated(bp, &mask);
7210         }
7211
7212         if (mask != vnic->rx_mask || uc_update || mc_update) {
7213                 vnic->rx_mask = mask;
7214
7215                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
7216                 bnxt_queue_sp_work(bp);
7217         }
7218 }
7219
7220 static int bnxt_cfg_rx_mode(struct bnxt *bp)
7221 {
7222         struct net_device *dev = bp->dev;
7223         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7224         struct netdev_hw_addr *ha;
7225         int i, off = 0, rc;
7226         bool uc_update;
7227
7228         netif_addr_lock_bh(dev);
7229         uc_update = bnxt_uc_list_updated(bp);
7230         netif_addr_unlock_bh(dev);
7231
7232         if (!uc_update)
7233                 goto skip_uc;
7234
7235         mutex_lock(&bp->hwrm_cmd_lock);
7236         for (i = 1; i < vnic->uc_filter_count; i++) {
7237                 struct hwrm_cfa_l2_filter_free_input req = {0};
7238
7239                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
7240                                        -1);
7241
7242                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
7243
7244                 rc = _hwrm_send_message(bp, &req, sizeof(req),
7245                                         HWRM_CMD_TIMEOUT);
7246         }
7247         mutex_unlock(&bp->hwrm_cmd_lock);
7248
7249         vnic->uc_filter_count = 1;
7250
7251         netif_addr_lock_bh(dev);
7252         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
7253                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7254         } else {
7255                 netdev_for_each_uc_addr(ha, dev) {
7256                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
7257                         off += ETH_ALEN;
7258                         vnic->uc_filter_count++;
7259                 }
7260         }
7261         netif_addr_unlock_bh(dev);
7262
7263         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
7264                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
7265                 if (rc) {
7266                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
7267                                    rc);
7268                         vnic->uc_filter_count = i;
7269                         return rc;
7270                 }
7271         }
7272
7273 skip_uc:
7274         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
7275         if (rc)
7276                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
7277                            rc);
7278
7279         return rc;
7280 }
7281
7282 /* If the chip and firmware supports RFS */
7283 static bool bnxt_rfs_supported(struct bnxt *bp)
7284 {
7285         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
7286                 return true;
7287         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7288                 return true;
7289         return false;
7290 }
7291
7292 /* If runtime conditions support RFS */
7293 static bool bnxt_rfs_capable(struct bnxt *bp)
7294 {
7295 #ifdef CONFIG_RFS_ACCEL
7296         int vnics, max_vnics, max_rss_ctxs;
7297
7298         if (!(bp->flags & BNXT_FLAG_MSIX_CAP))
7299                 return false;
7300
7301         vnics = 1 + bp->rx_nr_rings;
7302         max_vnics = bnxt_get_max_func_vnics(bp);
7303         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
7304
7305         /* RSS contexts not a limiting factor */
7306         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7307                 max_rss_ctxs = max_vnics;
7308         if (vnics > max_vnics || vnics > max_rss_ctxs) {
7309                 if (bp->rx_nr_rings > 1)
7310                         netdev_warn(bp->dev,
7311                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
7312                                     min(max_rss_ctxs - 1, max_vnics - 1));
7313                 return false;
7314         }
7315
7316         if (!(bp->flags & BNXT_FLAG_NEW_RM))
7317                 return true;
7318
7319         if (vnics == bp->hw_resc.resv_vnics)
7320                 return true;
7321
7322         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics);
7323         if (vnics <= bp->hw_resc.resv_vnics)
7324                 return true;
7325
7326         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
7327         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1);
7328         return false;
7329 #else
7330         return false;
7331 #endif
7332 }
7333
7334 static netdev_features_t bnxt_fix_features(struct net_device *dev,
7335                                            netdev_features_t features)
7336 {
7337         struct bnxt *bp = netdev_priv(dev);
7338
7339         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
7340                 features &= ~NETIF_F_NTUPLE;
7341
7342         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
7343                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
7344
7345         if (!(features & NETIF_F_GRO))
7346                 features &= ~NETIF_F_GRO_HW;
7347
7348         if (features & NETIF_F_GRO_HW)
7349                 features &= ~NETIF_F_LRO;
7350
7351         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
7352          * turned on or off together.
7353          */
7354         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
7355             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
7356                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
7357                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
7358                                       NETIF_F_HW_VLAN_STAG_RX);
7359                 else
7360                         features |= NETIF_F_HW_VLAN_CTAG_RX |
7361                                     NETIF_F_HW_VLAN_STAG_RX;
7362         }
7363 #ifdef CONFIG_BNXT_SRIOV
7364         if (BNXT_VF(bp)) {
7365                 if (bp->vf.vlan) {
7366                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
7367                                       NETIF_F_HW_VLAN_STAG_RX);
7368                 }
7369         }
7370 #endif
7371         return features;
7372 }
7373
7374 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
7375 {
7376         struct bnxt *bp = netdev_priv(dev);
7377         u32 flags = bp->flags;
7378         u32 changes;
7379         int rc = 0;
7380         bool re_init = false;
7381         bool update_tpa = false;
7382
7383         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
7384         if (features & NETIF_F_GRO_HW)
7385                 flags |= BNXT_FLAG_GRO;
7386         else if (features & NETIF_F_LRO)
7387                 flags |= BNXT_FLAG_LRO;
7388
7389         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
7390                 flags &= ~BNXT_FLAG_TPA;
7391
7392         if (features & NETIF_F_HW_VLAN_CTAG_RX)
7393                 flags |= BNXT_FLAG_STRIP_VLAN;
7394
7395         if (features & NETIF_F_NTUPLE)
7396                 flags |= BNXT_FLAG_RFS;
7397
7398         changes = flags ^ bp->flags;
7399         if (changes & BNXT_FLAG_TPA) {
7400                 update_tpa = true;
7401                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
7402                     (flags & BNXT_FLAG_TPA) == 0)
7403                         re_init = true;
7404         }
7405
7406         if (changes & ~BNXT_FLAG_TPA)
7407                 re_init = true;
7408
7409         if (flags != bp->flags) {
7410                 u32 old_flags = bp->flags;
7411
7412                 bp->flags = flags;
7413
7414                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7415                         if (update_tpa)
7416                                 bnxt_set_ring_params(bp);
7417                         return rc;
7418                 }
7419
7420                 if (re_init) {
7421                         bnxt_close_nic(bp, false, false);
7422                         if (update_tpa)
7423                                 bnxt_set_ring_params(bp);
7424
7425                         return bnxt_open_nic(bp, false, false);
7426                 }
7427                 if (update_tpa) {
7428                         rc = bnxt_set_tpa(bp,
7429                                           (flags & BNXT_FLAG_TPA) ?
7430                                           true : false);
7431                         if (rc)
7432                                 bp->flags = old_flags;
7433                 }
7434         }
7435         return rc;
7436 }
7437
7438 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
7439 {
7440         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
7441         int i = bnapi->index;
7442
7443         if (!txr)
7444                 return;
7445
7446         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
7447                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
7448                     txr->tx_cons);
7449 }
7450
7451 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
7452 {
7453         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
7454         int i = bnapi->index;
7455
7456         if (!rxr)
7457                 return;
7458
7459         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
7460                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
7461                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
7462                     rxr->rx_sw_agg_prod);
7463 }
7464
7465 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
7466 {
7467         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7468         int i = bnapi->index;
7469
7470         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
7471                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
7472 }
7473
7474 static void bnxt_dbg_dump_states(struct bnxt *bp)
7475 {
7476         int i;
7477         struct bnxt_napi *bnapi;
7478
7479         for (i = 0; i < bp->cp_nr_rings; i++) {
7480                 bnapi = bp->bnapi[i];
7481                 if (netif_msg_drv(bp)) {
7482                         bnxt_dump_tx_sw_state(bnapi);
7483                         bnxt_dump_rx_sw_state(bnapi);
7484                         bnxt_dump_cp_sw_state(bnapi);
7485                 }
7486         }
7487 }
7488
7489 static void bnxt_reset_task(struct bnxt *bp, bool silent)
7490 {
7491         if (!silent)
7492                 bnxt_dbg_dump_states(bp);
7493         if (netif_running(bp->dev)) {
7494                 int rc;
7495
7496                 if (!silent)
7497                         bnxt_ulp_stop(bp);
7498                 bnxt_close_nic(bp, false, false);
7499                 rc = bnxt_open_nic(bp, false, false);
7500                 if (!silent && !rc)
7501                         bnxt_ulp_start(bp);
7502         }
7503 }
7504
7505 static void bnxt_tx_timeout(struct net_device *dev)
7506 {
7507         struct bnxt *bp = netdev_priv(dev);
7508
7509         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
7510         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
7511         bnxt_queue_sp_work(bp);
7512 }
7513
7514 #ifdef CONFIG_NET_POLL_CONTROLLER
7515 static void bnxt_poll_controller(struct net_device *dev)
7516 {
7517         struct bnxt *bp = netdev_priv(dev);
7518         int i;
7519
7520         /* Only process tx rings/combined rings in netpoll mode. */
7521         for (i = 0; i < bp->tx_nr_rings; i++) {
7522                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7523
7524                 napi_schedule(&txr->bnapi->napi);
7525         }
7526 }
7527 #endif
7528
7529 static void bnxt_timer(struct timer_list *t)
7530 {
7531         struct bnxt *bp = from_timer(bp, t, timer);
7532         struct net_device *dev = bp->dev;
7533
7534         if (!netif_running(dev))
7535                 return;
7536
7537         if (atomic_read(&bp->intr_sem) != 0)
7538                 goto bnxt_restart_timer;
7539
7540         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
7541             bp->stats_coal_ticks) {
7542                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
7543                 bnxt_queue_sp_work(bp);
7544         }
7545
7546         if (bnxt_tc_flower_enabled(bp)) {
7547                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
7548                 bnxt_queue_sp_work(bp);
7549         }
7550 bnxt_restart_timer:
7551         mod_timer(&bp->timer, jiffies + bp->current_interval);
7552 }
7553
7554 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
7555 {
7556         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
7557          * set.  If the device is being closed, bnxt_close() may be holding
7558          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
7559          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
7560          */
7561         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7562         rtnl_lock();
7563 }
7564
7565 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
7566 {
7567         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7568         rtnl_unlock();
7569 }
7570
7571 /* Only called from bnxt_sp_task() */
7572 static void bnxt_reset(struct bnxt *bp, bool silent)
7573 {
7574         bnxt_rtnl_lock_sp(bp);
7575         if (test_bit(BNXT_STATE_OPEN, &bp->state))
7576                 bnxt_reset_task(bp, silent);
7577         bnxt_rtnl_unlock_sp(bp);
7578 }
7579
7580 static void bnxt_cfg_ntp_filters(struct bnxt *);
7581
7582 static void bnxt_sp_task(struct work_struct *work)
7583 {
7584         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
7585
7586         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7587         smp_mb__after_atomic();
7588         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
7589                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7590                 return;
7591         }
7592
7593         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
7594                 bnxt_cfg_rx_mode(bp);
7595
7596         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
7597                 bnxt_cfg_ntp_filters(bp);
7598         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
7599                 bnxt_hwrm_exec_fwd_req(bp);
7600         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7601                 bnxt_hwrm_tunnel_dst_port_alloc(
7602                         bp, bp->vxlan_port,
7603                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7604         }
7605         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7606                 bnxt_hwrm_tunnel_dst_port_free(
7607                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7608         }
7609         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
7610                 bnxt_hwrm_tunnel_dst_port_alloc(
7611                         bp, bp->nge_port,
7612                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7613         }
7614         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
7615                 bnxt_hwrm_tunnel_dst_port_free(
7616                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7617         }
7618         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
7619                 bnxt_hwrm_port_qstats(bp);
7620                 bnxt_hwrm_port_qstats_ext(bp);
7621         }
7622
7623         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
7624                 int rc;
7625
7626                 mutex_lock(&bp->link_lock);
7627                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
7628                                        &bp->sp_event))
7629                         bnxt_hwrm_phy_qcaps(bp);
7630
7631                 rc = bnxt_update_link(bp, true);
7632                 mutex_unlock(&bp->link_lock);
7633                 if (rc)
7634                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
7635                                    rc);
7636         }
7637         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
7638                 mutex_lock(&bp->link_lock);
7639                 bnxt_get_port_module_status(bp);
7640                 mutex_unlock(&bp->link_lock);
7641         }
7642
7643         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
7644                 bnxt_tc_flow_stats_work(bp);
7645
7646         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
7647          * must be the last functions to be called before exiting.
7648          */
7649         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
7650                 bnxt_reset(bp, false);
7651
7652         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
7653                 bnxt_reset(bp, true);
7654
7655         smp_mb__before_atomic();
7656         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
7657 }
7658
7659 /* Under rtnl_lock */
7660 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
7661                      int tx_xdp)
7662 {
7663         int max_rx, max_tx, tx_sets = 1;
7664         int tx_rings_needed;
7665         int rx_rings = rx;
7666         int cp, vnics, rc;
7667
7668         if (tcs)
7669                 tx_sets = tcs;
7670
7671         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
7672         if (rc)
7673                 return rc;
7674
7675         if (max_rx < rx)
7676                 return -ENOMEM;
7677
7678         tx_rings_needed = tx * tx_sets + tx_xdp;
7679         if (max_tx < tx_rings_needed)
7680                 return -ENOMEM;
7681
7682         vnics = 1;
7683         if (bp->flags & BNXT_FLAG_RFS)
7684                 vnics += rx_rings;
7685
7686         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7687                 rx_rings <<= 1;
7688         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
7689         if (bp->flags & BNXT_FLAG_NEW_RM)
7690                 cp += bnxt_get_ulp_msix_num(bp);
7691         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
7692                                      vnics);
7693 }
7694
7695 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
7696 {
7697         if (bp->bar2) {
7698                 pci_iounmap(pdev, bp->bar2);
7699                 bp->bar2 = NULL;
7700         }
7701
7702         if (bp->bar1) {
7703                 pci_iounmap(pdev, bp->bar1);
7704                 bp->bar1 = NULL;
7705         }
7706
7707         if (bp->bar0) {
7708                 pci_iounmap(pdev, bp->bar0);
7709                 bp->bar0 = NULL;
7710         }
7711 }
7712
7713 static void bnxt_cleanup_pci(struct bnxt *bp)
7714 {
7715         bnxt_unmap_bars(bp, bp->pdev);
7716         pci_release_regions(bp->pdev);
7717         pci_disable_device(bp->pdev);
7718 }
7719
7720 static void bnxt_init_dflt_coal(struct bnxt *bp)
7721 {
7722         struct bnxt_coal *coal;
7723
7724         /* Tick values in micro seconds.
7725          * 1 coal_buf x bufs_per_record = 1 completion record.
7726          */
7727         coal = &bp->rx_coal;
7728         coal->coal_ticks = 14;
7729         coal->coal_bufs = 30;
7730         coal->coal_ticks_irq = 1;
7731         coal->coal_bufs_irq = 2;
7732         coal->idle_thresh = 25;
7733         coal->bufs_per_record = 2;
7734         coal->budget = 64;              /* NAPI budget */
7735
7736         coal = &bp->tx_coal;
7737         coal->coal_ticks = 28;
7738         coal->coal_bufs = 30;
7739         coal->coal_ticks_irq = 2;
7740         coal->coal_bufs_irq = 2;
7741         coal->bufs_per_record = 1;
7742
7743         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
7744 }
7745
7746 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
7747 {
7748         int rc;
7749         struct bnxt *bp = netdev_priv(dev);
7750
7751         SET_NETDEV_DEV(dev, &pdev->dev);
7752
7753         /* enable device (incl. PCI PM wakeup), and bus-mastering */
7754         rc = pci_enable_device(pdev);
7755         if (rc) {
7756                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
7757                 goto init_err;
7758         }
7759
7760         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
7761                 dev_err(&pdev->dev,
7762                         "Cannot find PCI device base address, aborting\n");
7763                 rc = -ENODEV;
7764                 goto init_err_disable;
7765         }
7766
7767         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
7768         if (rc) {
7769                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
7770                 goto init_err_disable;
7771         }
7772
7773         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
7774             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
7775                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
7776                 goto init_err_disable;
7777         }
7778
7779         pci_set_master(pdev);
7780
7781         bp->dev = dev;
7782         bp->pdev = pdev;
7783
7784         bp->bar0 = pci_ioremap_bar(pdev, 0);
7785         if (!bp->bar0) {
7786                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
7787                 rc = -ENOMEM;
7788                 goto init_err_release;
7789         }
7790
7791         bp->bar1 = pci_ioremap_bar(pdev, 2);
7792         if (!bp->bar1) {
7793                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
7794                 rc = -ENOMEM;
7795                 goto init_err_release;
7796         }
7797
7798         bp->bar2 = pci_ioremap_bar(pdev, 4);
7799         if (!bp->bar2) {
7800                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
7801                 rc = -ENOMEM;
7802                 goto init_err_release;
7803         }
7804
7805         pci_enable_pcie_error_reporting(pdev);
7806
7807         INIT_WORK(&bp->sp_task, bnxt_sp_task);
7808
7809         spin_lock_init(&bp->ntp_fltr_lock);
7810
7811         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
7812         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
7813
7814         bnxt_init_dflt_coal(bp);
7815
7816         timer_setup(&bp->timer, bnxt_timer, 0);
7817         bp->current_interval = BNXT_TIMER_INTERVAL;
7818
7819         clear_bit(BNXT_STATE_OPEN, &bp->state);
7820         return 0;
7821
7822 init_err_release:
7823         bnxt_unmap_bars(bp, pdev);
7824         pci_release_regions(pdev);
7825
7826 init_err_disable:
7827         pci_disable_device(pdev);
7828
7829 init_err:
7830         return rc;
7831 }
7832
7833 /* rtnl_lock held */
7834 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
7835 {
7836         struct sockaddr *addr = p;
7837         struct bnxt *bp = netdev_priv(dev);
7838         int rc = 0;
7839
7840         if (!is_valid_ether_addr(addr->sa_data))
7841                 return -EADDRNOTAVAIL;
7842
7843         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
7844                 return 0;
7845
7846         rc = bnxt_approve_mac(bp, addr->sa_data);
7847         if (rc)
7848                 return rc;
7849
7850         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
7851         if (netif_running(dev)) {
7852                 bnxt_close_nic(bp, false, false);
7853                 rc = bnxt_open_nic(bp, false, false);
7854         }
7855
7856         return rc;
7857 }
7858
7859 /* rtnl_lock held */
7860 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
7861 {
7862         struct bnxt *bp = netdev_priv(dev);
7863
7864         if (netif_running(dev))
7865                 bnxt_close_nic(bp, false, false);
7866
7867         dev->mtu = new_mtu;
7868         bnxt_set_ring_params(bp);
7869
7870         if (netif_running(dev))
7871                 return bnxt_open_nic(bp, false, false);
7872
7873         return 0;
7874 }
7875
7876 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
7877 {
7878         struct bnxt *bp = netdev_priv(dev);
7879         bool sh = false;
7880         int rc;
7881
7882         if (tc > bp->max_tc) {
7883                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
7884                            tc, bp->max_tc);
7885                 return -EINVAL;
7886         }
7887
7888         if (netdev_get_num_tc(dev) == tc)
7889                 return 0;
7890
7891         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7892                 sh = true;
7893
7894         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
7895                               sh, tc, bp->tx_nr_rings_xdp);
7896         if (rc)
7897                 return rc;
7898
7899         /* Needs to close the device and do hw resource re-allocations */
7900         if (netif_running(bp->dev))
7901                 bnxt_close_nic(bp, true, false);
7902
7903         if (tc) {
7904                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
7905                 netdev_set_num_tc(dev, tc);
7906         } else {
7907                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
7908                 netdev_reset_tc(dev);
7909         }
7910         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
7911         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7912                                bp->tx_nr_rings + bp->rx_nr_rings;
7913         bp->num_stat_ctxs = bp->cp_nr_rings;
7914
7915         if (netif_running(bp->dev))
7916                 return bnxt_open_nic(bp, true, false);
7917
7918         return 0;
7919 }
7920
7921 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
7922                                   void *cb_priv)
7923 {
7924         struct bnxt *bp = cb_priv;
7925
7926         if (!bnxt_tc_flower_enabled(bp) ||
7927             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
7928                 return -EOPNOTSUPP;
7929
7930         switch (type) {
7931         case TC_SETUP_CLSFLOWER:
7932                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
7933         default:
7934                 return -EOPNOTSUPP;
7935         }
7936 }
7937
7938 static int bnxt_setup_tc_block(struct net_device *dev,
7939                                struct tc_block_offload *f)
7940 {
7941         struct bnxt *bp = netdev_priv(dev);
7942
7943         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
7944                 return -EOPNOTSUPP;
7945
7946         switch (f->command) {
7947         case TC_BLOCK_BIND:
7948                 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb,
7949                                              bp, bp);
7950         case TC_BLOCK_UNBIND:
7951                 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp);
7952                 return 0;
7953         default:
7954                 return -EOPNOTSUPP;
7955         }
7956 }
7957
7958 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
7959                          void *type_data)
7960 {
7961         switch (type) {
7962         case TC_SETUP_BLOCK:
7963                 return bnxt_setup_tc_block(dev, type_data);
7964         case TC_SETUP_QDISC_MQPRIO: {
7965                 struct tc_mqprio_qopt *mqprio = type_data;
7966
7967                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
7968
7969                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
7970         }
7971         default:
7972                 return -EOPNOTSUPP;
7973         }
7974 }
7975
7976 #ifdef CONFIG_RFS_ACCEL
7977 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
7978                             struct bnxt_ntuple_filter *f2)
7979 {
7980         struct flow_keys *keys1 = &f1->fkeys;
7981         struct flow_keys *keys2 = &f2->fkeys;
7982
7983         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
7984             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
7985             keys1->ports.ports == keys2->ports.ports &&
7986             keys1->basic.ip_proto == keys2->basic.ip_proto &&
7987             keys1->basic.n_proto == keys2->basic.n_proto &&
7988             keys1->control.flags == keys2->control.flags &&
7989             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
7990             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
7991                 return true;
7992
7993         return false;
7994 }
7995
7996 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
7997                               u16 rxq_index, u32 flow_id)
7998 {
7999         struct bnxt *bp = netdev_priv(dev);
8000         struct bnxt_ntuple_filter *fltr, *new_fltr;
8001         struct flow_keys *fkeys;
8002         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
8003         int rc = 0, idx, bit_id, l2_idx = 0;
8004         struct hlist_head *head;
8005
8006         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
8007                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8008                 int off = 0, j;
8009
8010                 netif_addr_lock_bh(dev);
8011                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
8012                         if (ether_addr_equal(eth->h_dest,
8013                                              vnic->uc_list + off)) {
8014                                 l2_idx = j + 1;
8015                                 break;
8016                         }
8017                 }
8018                 netif_addr_unlock_bh(dev);
8019                 if (!l2_idx)
8020                         return -EINVAL;
8021         }
8022         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
8023         if (!new_fltr)
8024                 return -ENOMEM;
8025
8026         fkeys = &new_fltr->fkeys;
8027         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
8028                 rc = -EPROTONOSUPPORT;
8029                 goto err_free;
8030         }
8031
8032         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
8033              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
8034             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
8035              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
8036                 rc = -EPROTONOSUPPORT;
8037                 goto err_free;
8038         }
8039         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
8040             bp->hwrm_spec_code < 0x10601) {
8041                 rc = -EPROTONOSUPPORT;
8042                 goto err_free;
8043         }
8044         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
8045             bp->hwrm_spec_code < 0x10601) {
8046                 rc = -EPROTONOSUPPORT;
8047                 goto err_free;
8048         }
8049
8050         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
8051         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
8052
8053         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
8054         head = &bp->ntp_fltr_hash_tbl[idx];
8055         rcu_read_lock();
8056         hlist_for_each_entry_rcu(fltr, head, hash) {
8057                 if (bnxt_fltr_match(fltr, new_fltr)) {
8058                         rcu_read_unlock();
8059                         rc = 0;
8060                         goto err_free;
8061                 }
8062         }
8063         rcu_read_unlock();
8064
8065         spin_lock_bh(&bp->ntp_fltr_lock);
8066         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
8067                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
8068         if (bit_id < 0) {
8069                 spin_unlock_bh(&bp->ntp_fltr_lock);
8070                 rc = -ENOMEM;
8071                 goto err_free;
8072         }
8073
8074         new_fltr->sw_id = (u16)bit_id;
8075         new_fltr->flow_id = flow_id;
8076         new_fltr->l2_fltr_idx = l2_idx;
8077         new_fltr->rxq = rxq_index;
8078         hlist_add_head_rcu(&new_fltr->hash, head);
8079         bp->ntp_fltr_count++;
8080         spin_unlock_bh(&bp->ntp_fltr_lock);
8081
8082         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
8083         bnxt_queue_sp_work(bp);
8084
8085         return new_fltr->sw_id;
8086
8087 err_free:
8088         kfree(new_fltr);
8089         return rc;
8090 }
8091
8092 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
8093 {
8094         int i;
8095
8096         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
8097                 struct hlist_head *head;
8098                 struct hlist_node *tmp;
8099                 struct bnxt_ntuple_filter *fltr;
8100                 int rc;
8101
8102                 head = &bp->ntp_fltr_hash_tbl[i];
8103                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
8104                         bool del = false;
8105
8106                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
8107                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
8108                                                         fltr->flow_id,
8109                                                         fltr->sw_id)) {
8110                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
8111                                                                          fltr);
8112                                         del = true;
8113                                 }
8114                         } else {
8115                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
8116                                                                        fltr);
8117                                 if (rc)
8118                                         del = true;
8119                                 else
8120                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
8121                         }
8122
8123                         if (del) {
8124                                 spin_lock_bh(&bp->ntp_fltr_lock);
8125                                 hlist_del_rcu(&fltr->hash);
8126                                 bp->ntp_fltr_count--;
8127                                 spin_unlock_bh(&bp->ntp_fltr_lock);
8128                                 synchronize_rcu();
8129                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
8130                                 kfree(fltr);
8131                         }
8132                 }
8133         }
8134         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
8135                 netdev_info(bp->dev, "Receive PF driver unload event!");
8136 }
8137
8138 #else
8139
8140 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
8141 {
8142 }
8143
8144 #endif /* CONFIG_RFS_ACCEL */
8145
8146 static void bnxt_udp_tunnel_add(struct net_device *dev,
8147                                 struct udp_tunnel_info *ti)
8148 {
8149         struct bnxt *bp = netdev_priv(dev);
8150
8151         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
8152                 return;
8153
8154         if (!netif_running(dev))
8155                 return;
8156
8157         switch (ti->type) {
8158         case UDP_TUNNEL_TYPE_VXLAN:
8159                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
8160                         return;
8161
8162                 bp->vxlan_port_cnt++;
8163                 if (bp->vxlan_port_cnt == 1) {
8164                         bp->vxlan_port = ti->port;
8165                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
8166                         bnxt_queue_sp_work(bp);
8167                 }
8168                 break;
8169         case UDP_TUNNEL_TYPE_GENEVE:
8170                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
8171                         return;
8172
8173                 bp->nge_port_cnt++;
8174                 if (bp->nge_port_cnt == 1) {
8175                         bp->nge_port = ti->port;
8176                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
8177                 }
8178                 break;
8179         default:
8180                 return;
8181         }
8182
8183         bnxt_queue_sp_work(bp);
8184 }
8185
8186 static void bnxt_udp_tunnel_del(struct net_device *dev,
8187                                 struct udp_tunnel_info *ti)
8188 {
8189         struct bnxt *bp = netdev_priv(dev);
8190
8191         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
8192                 return;
8193
8194         if (!netif_running(dev))
8195                 return;
8196
8197         switch (ti->type) {
8198         case UDP_TUNNEL_TYPE_VXLAN:
8199                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
8200                         return;
8201                 bp->vxlan_port_cnt--;
8202
8203                 if (bp->vxlan_port_cnt != 0)
8204                         return;
8205
8206                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
8207                 break;
8208         case UDP_TUNNEL_TYPE_GENEVE:
8209                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
8210                         return;
8211                 bp->nge_port_cnt--;
8212
8213                 if (bp->nge_port_cnt != 0)
8214                         return;
8215
8216                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
8217                 break;
8218         default:
8219                 return;
8220         }
8221
8222         bnxt_queue_sp_work(bp);
8223 }
8224
8225 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
8226                                struct net_device *dev, u32 filter_mask,
8227                                int nlflags)
8228 {
8229         struct bnxt *bp = netdev_priv(dev);
8230
8231         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
8232                                        nlflags, filter_mask, NULL);
8233 }
8234
8235 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
8236                                u16 flags)
8237 {
8238         struct bnxt *bp = netdev_priv(dev);
8239         struct nlattr *attr, *br_spec;
8240         int rem, rc = 0;
8241
8242         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
8243                 return -EOPNOTSUPP;
8244
8245         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
8246         if (!br_spec)
8247                 return -EINVAL;
8248
8249         nla_for_each_nested(attr, br_spec, rem) {
8250                 u16 mode;
8251
8252                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
8253                         continue;
8254
8255                 if (nla_len(attr) < sizeof(mode))
8256                         return -EINVAL;
8257
8258                 mode = nla_get_u16(attr);
8259                 if (mode == bp->br_mode)
8260                         break;
8261
8262                 rc = bnxt_hwrm_set_br_mode(bp, mode);
8263                 if (!rc)
8264                         bp->br_mode = mode;
8265                 break;
8266         }
8267         return rc;
8268 }
8269
8270 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
8271                                    size_t len)
8272 {
8273         struct bnxt *bp = netdev_priv(dev);
8274         int rc;
8275
8276         /* The PF and it's VF-reps only support the switchdev framework */
8277         if (!BNXT_PF(bp))
8278                 return -EOPNOTSUPP;
8279
8280         rc = snprintf(buf, len, "p%d", bp->pf.port_id);
8281
8282         if (rc >= len)
8283                 return -EOPNOTSUPP;
8284         return 0;
8285 }
8286
8287 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
8288 {
8289         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
8290                 return -EOPNOTSUPP;
8291
8292         /* The PF and it's VF-reps only support the switchdev framework */
8293         if (!BNXT_PF(bp))
8294                 return -EOPNOTSUPP;
8295
8296         switch (attr->id) {
8297         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
8298                 attr->u.ppid.id_len = sizeof(bp->switch_id);
8299                 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len);
8300                 break;
8301         default:
8302                 return -EOPNOTSUPP;
8303         }
8304         return 0;
8305 }
8306
8307 static int bnxt_swdev_port_attr_get(struct net_device *dev,
8308                                     struct switchdev_attr *attr)
8309 {
8310         return bnxt_port_attr_get(netdev_priv(dev), attr);
8311 }
8312
8313 static const struct switchdev_ops bnxt_switchdev_ops = {
8314         .switchdev_port_attr_get        = bnxt_swdev_port_attr_get
8315 };
8316
8317 static const struct net_device_ops bnxt_netdev_ops = {
8318         .ndo_open               = bnxt_open,
8319         .ndo_start_xmit         = bnxt_start_xmit,
8320         .ndo_stop               = bnxt_close,
8321         .ndo_get_stats64        = bnxt_get_stats64,
8322         .ndo_set_rx_mode        = bnxt_set_rx_mode,
8323         .ndo_do_ioctl           = bnxt_ioctl,
8324         .ndo_validate_addr      = eth_validate_addr,
8325         .ndo_set_mac_address    = bnxt_change_mac_addr,
8326         .ndo_change_mtu         = bnxt_change_mtu,
8327         .ndo_fix_features       = bnxt_fix_features,
8328         .ndo_set_features       = bnxt_set_features,
8329         .ndo_tx_timeout         = bnxt_tx_timeout,
8330 #ifdef CONFIG_BNXT_SRIOV
8331         .ndo_get_vf_config      = bnxt_get_vf_config,
8332         .ndo_set_vf_mac         = bnxt_set_vf_mac,
8333         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
8334         .ndo_set_vf_rate        = bnxt_set_vf_bw,
8335         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
8336         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
8337         .ndo_set_vf_trust       = bnxt_set_vf_trust,
8338 #endif
8339 #ifdef CONFIG_NET_POLL_CONTROLLER
8340         .ndo_poll_controller    = bnxt_poll_controller,
8341 #endif
8342         .ndo_setup_tc           = bnxt_setup_tc,
8343 #ifdef CONFIG_RFS_ACCEL
8344         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
8345 #endif
8346         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
8347         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
8348         .ndo_bpf                = bnxt_xdp,
8349         .ndo_bridge_getlink     = bnxt_bridge_getlink,
8350         .ndo_bridge_setlink     = bnxt_bridge_setlink,
8351         .ndo_get_phys_port_name = bnxt_get_phys_port_name
8352 };
8353
8354 static void bnxt_remove_one(struct pci_dev *pdev)
8355 {
8356         struct net_device *dev = pci_get_drvdata(pdev);
8357         struct bnxt *bp = netdev_priv(dev);
8358
8359         if (BNXT_PF(bp)) {
8360                 bnxt_sriov_disable(bp);
8361                 bnxt_dl_unregister(bp);
8362         }
8363
8364         pci_disable_pcie_error_reporting(pdev);
8365         unregister_netdev(dev);
8366         bnxt_shutdown_tc(bp);
8367         bnxt_cancel_sp_work(bp);
8368         bp->sp_event = 0;
8369
8370         bnxt_clear_int_mode(bp);
8371         bnxt_hwrm_func_drv_unrgtr(bp);
8372         bnxt_free_hwrm_resources(bp);
8373         bnxt_free_hwrm_short_cmd_req(bp);
8374         bnxt_ethtool_free(bp);
8375         bnxt_dcb_free(bp);
8376         kfree(bp->edev);
8377         bp->edev = NULL;
8378         bnxt_cleanup_pci(bp);
8379         free_netdev(dev);
8380 }
8381
8382 static int bnxt_probe_phy(struct bnxt *bp)
8383 {
8384         int rc = 0;
8385         struct bnxt_link_info *link_info = &bp->link_info;
8386
8387         rc = bnxt_hwrm_phy_qcaps(bp);
8388         if (rc) {
8389                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
8390                            rc);
8391                 return rc;
8392         }
8393         mutex_init(&bp->link_lock);
8394
8395         rc = bnxt_update_link(bp, false);
8396         if (rc) {
8397                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
8398                            rc);
8399                 return rc;
8400         }
8401
8402         /* Older firmware does not have supported_auto_speeds, so assume
8403          * that all supported speeds can be autonegotiated.
8404          */
8405         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
8406                 link_info->support_auto_speeds = link_info->support_speeds;
8407
8408         /*initialize the ethool setting copy with NVM settings */
8409         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
8410                 link_info->autoneg = BNXT_AUTONEG_SPEED;
8411                 if (bp->hwrm_spec_code >= 0x10201) {
8412                         if (link_info->auto_pause_setting &
8413                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
8414                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
8415                 } else {
8416                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
8417                 }
8418                 link_info->advertising = link_info->auto_link_speeds;
8419         } else {
8420                 link_info->req_link_speed = link_info->force_link_speed;
8421                 link_info->req_duplex = link_info->duplex_setting;
8422         }
8423         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
8424                 link_info->req_flow_ctrl =
8425                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
8426         else
8427                 link_info->req_flow_ctrl = link_info->force_pause_setting;
8428         return rc;
8429 }
8430
8431 static int bnxt_get_max_irq(struct pci_dev *pdev)
8432 {
8433         u16 ctrl;
8434
8435         if (!pdev->msix_cap)
8436                 return 1;
8437
8438         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
8439         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
8440 }
8441
8442 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
8443                                 int *max_cp)
8444 {
8445         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8446         int max_ring_grps = 0;
8447
8448         *max_tx = hw_resc->max_tx_rings;
8449         *max_rx = hw_resc->max_rx_rings;
8450         *max_cp = min_t(int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8451         *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs);
8452         max_ring_grps = hw_resc->max_hw_ring_grps;
8453         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
8454                 *max_cp -= 1;
8455                 *max_rx -= 2;
8456         }
8457         if (bp->flags & BNXT_FLAG_AGG_RINGS)
8458                 *max_rx >>= 1;
8459         *max_rx = min_t(int, *max_rx, max_ring_grps);
8460 }
8461
8462 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
8463 {
8464         int rx, tx, cp;
8465
8466         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
8467         if (!rx || !tx || !cp)
8468                 return -ENOMEM;
8469
8470         *max_rx = rx;
8471         *max_tx = tx;
8472         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
8473 }
8474
8475 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
8476                                bool shared)
8477 {
8478         int rc;
8479
8480         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
8481         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
8482                 /* Not enough rings, try disabling agg rings. */
8483                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
8484                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
8485                 if (rc)
8486                         return rc;
8487                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
8488                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
8489                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
8490                 bnxt_set_ring_params(bp);
8491         }
8492
8493         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
8494                 int max_cp, max_stat, max_irq;
8495
8496                 /* Reserve minimum resources for RoCE */
8497                 max_cp = bnxt_get_max_func_cp_rings(bp);
8498                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
8499                 max_irq = bnxt_get_max_func_irqs(bp);
8500                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
8501                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
8502                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
8503                         return 0;
8504
8505                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
8506                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
8507                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
8508                 max_cp = min_t(int, max_cp, max_irq);
8509                 max_cp = min_t(int, max_cp, max_stat);
8510                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
8511                 if (rc)
8512                         rc = 0;
8513         }
8514         return rc;
8515 }
8516
8517 /* In initial default shared ring setting, each shared ring must have a
8518  * RX/TX ring pair.
8519  */
8520 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
8521 {
8522         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
8523         bp->rx_nr_rings = bp->cp_nr_rings;
8524         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
8525         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
8526 }
8527
8528 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
8529 {
8530         int dflt_rings, max_rx_rings, max_tx_rings, rc;
8531
8532         if (sh)
8533                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
8534         dflt_rings = netif_get_num_default_rss_queues();
8535         /* Reduce default rings on multi-port cards so that total default
8536          * rings do not exceed CPU count.
8537          */
8538         if (bp->port_count > 1) {
8539                 int max_rings =
8540                         max_t(int, num_online_cpus() / bp->port_count, 1);
8541
8542                 dflt_rings = min_t(int, dflt_rings, max_rings);
8543         }
8544         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
8545         if (rc)
8546                 return rc;
8547         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
8548         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
8549         if (sh)
8550                 bnxt_trim_dflt_sh_rings(bp);
8551         else
8552                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
8553         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
8554
8555         rc = __bnxt_reserve_rings(bp);
8556         if (rc)
8557                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
8558         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8559         if (sh)
8560                 bnxt_trim_dflt_sh_rings(bp);
8561
8562         /* Rings may have been trimmed, re-reserve the trimmed rings. */
8563         if (bnxt_need_reserve_rings(bp)) {
8564                 rc = __bnxt_reserve_rings(bp);
8565                 if (rc)
8566                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
8567                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8568         }
8569         bp->num_stat_ctxs = bp->cp_nr_rings;
8570         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8571                 bp->rx_nr_rings++;
8572                 bp->cp_nr_rings++;
8573         }
8574         return rc;
8575 }
8576
8577 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
8578 {
8579         int rc;
8580
8581         ASSERT_RTNL();
8582         bnxt_hwrm_func_qcaps(bp);
8583
8584         if (netif_running(bp->dev))
8585                 __bnxt_close_nic(bp, true, false);
8586
8587         bnxt_ulp_irq_stop(bp);
8588         bnxt_clear_int_mode(bp);
8589         rc = bnxt_init_int_mode(bp);
8590         bnxt_ulp_irq_restart(bp, rc);
8591
8592         if (netif_running(bp->dev)) {
8593                 if (rc)
8594                         dev_close(bp->dev);
8595                 else
8596                         rc = bnxt_open_nic(bp, true, false);
8597         }
8598
8599         return rc;
8600 }
8601
8602 static int bnxt_init_mac_addr(struct bnxt *bp)
8603 {
8604         int rc = 0;
8605
8606         if (BNXT_PF(bp)) {
8607                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
8608         } else {
8609 #ifdef CONFIG_BNXT_SRIOV
8610                 struct bnxt_vf_info *vf = &bp->vf;
8611
8612                 if (is_valid_ether_addr(vf->mac_addr)) {
8613                         /* overwrite netdev dev_addr with admin VF MAC */
8614                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
8615                 } else {
8616                         eth_hw_addr_random(bp->dev);
8617                         rc = bnxt_approve_mac(bp, bp->dev->dev_addr);
8618                 }
8619 #endif
8620         }
8621         return rc;
8622 }
8623
8624 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
8625 {
8626         enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
8627         enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
8628
8629         if (pcie_get_minimum_link(pci_physfn(bp->pdev), &speed, &width) ||
8630             speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
8631                 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
8632         else
8633                 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
8634                             speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
8635                             speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
8636                             speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
8637                             "Unknown", width);
8638 }
8639
8640 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
8641 {
8642         static int version_printed;
8643         struct net_device *dev;
8644         struct bnxt *bp;
8645         int rc, max_irqs;
8646
8647         if (pci_is_bridge(pdev))
8648                 return -ENODEV;
8649
8650         if (version_printed++ == 0)
8651                 pr_info("%s", version);
8652
8653         max_irqs = bnxt_get_max_irq(pdev);
8654         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
8655         if (!dev)
8656                 return -ENOMEM;
8657
8658         bp = netdev_priv(dev);
8659
8660         if (bnxt_vf_pciid(ent->driver_data))
8661                 bp->flags |= BNXT_FLAG_VF;
8662
8663         if (pdev->msix_cap)
8664                 bp->flags |= BNXT_FLAG_MSIX_CAP;
8665
8666         rc = bnxt_init_board(pdev, dev);
8667         if (rc < 0)
8668                 goto init_err_free;
8669
8670         dev->netdev_ops = &bnxt_netdev_ops;
8671         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
8672         dev->ethtool_ops = &bnxt_ethtool_ops;
8673         SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
8674         pci_set_drvdata(pdev, dev);
8675
8676         rc = bnxt_alloc_hwrm_resources(bp);
8677         if (rc)
8678                 goto init_err_pci_clean;
8679
8680         mutex_init(&bp->hwrm_cmd_lock);
8681         rc = bnxt_hwrm_ver_get(bp);
8682         if (rc)
8683                 goto init_err_pci_clean;
8684
8685         if (bp->flags & BNXT_FLAG_SHORT_CMD) {
8686                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
8687                 if (rc)
8688                         goto init_err_pci_clean;
8689         }
8690
8691         rc = bnxt_hwrm_func_reset(bp);
8692         if (rc)
8693                 goto init_err_pci_clean;
8694
8695         bnxt_hwrm_fw_set_time(bp);
8696
8697         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8698                            NETIF_F_TSO | NETIF_F_TSO6 |
8699                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8700                            NETIF_F_GSO_IPXIP4 |
8701                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8702                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
8703                            NETIF_F_RXCSUM | NETIF_F_GRO;
8704
8705         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
8706                 dev->hw_features |= NETIF_F_LRO;
8707
8708         dev->hw_enc_features =
8709                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
8710                         NETIF_F_TSO | NETIF_F_TSO6 |
8711                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
8712                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
8713                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
8714         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
8715                                     NETIF_F_GSO_GRE_CSUM;
8716         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
8717         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
8718                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
8719         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
8720                 dev->hw_features |= NETIF_F_GRO_HW;
8721         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
8722         if (dev->features & NETIF_F_GRO_HW)
8723                 dev->features &= ~NETIF_F_LRO;
8724         dev->priv_flags |= IFF_UNICAST_FLT;
8725
8726 #ifdef CONFIG_BNXT_SRIOV
8727         init_waitqueue_head(&bp->sriov_cfg_wait);
8728         mutex_init(&bp->sriov_lock);
8729 #endif
8730         bp->gro_func = bnxt_gro_func_5730x;
8731         if (BNXT_CHIP_P4_PLUS(bp))
8732                 bp->gro_func = bnxt_gro_func_5731x;
8733         else
8734                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
8735
8736         rc = bnxt_hwrm_func_drv_rgtr(bp);
8737         if (rc)
8738                 goto init_err_pci_clean;
8739
8740         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
8741         if (rc)
8742                 goto init_err_pci_clean;
8743
8744         bp->ulp_probe = bnxt_ulp_probe;
8745
8746         /* Get the MAX capabilities for this function */
8747         rc = bnxt_hwrm_func_qcaps(bp);
8748         if (rc) {
8749                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
8750                            rc);
8751                 rc = -1;
8752                 goto init_err_pci_clean;
8753         }
8754         rc = bnxt_init_mac_addr(bp);
8755         if (rc) {
8756                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
8757                 rc = -EADDRNOTAVAIL;
8758                 goto init_err_pci_clean;
8759         }
8760         rc = bnxt_hwrm_queue_qportcfg(bp);
8761         if (rc) {
8762                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
8763                            rc);
8764                 rc = -1;
8765                 goto init_err_pci_clean;
8766         }
8767
8768         bnxt_hwrm_func_qcfg(bp);
8769         bnxt_hwrm_port_led_qcaps(bp);
8770         bnxt_ethtool_init(bp);
8771         bnxt_dcb_init(bp);
8772
8773         /* MTU range: 60 - FW defined max */
8774         dev->min_mtu = ETH_ZLEN;
8775         dev->max_mtu = bp->max_mtu;
8776
8777         rc = bnxt_probe_phy(bp);
8778         if (rc)
8779                 goto init_err_pci_clean;
8780
8781         bnxt_set_rx_skb_mode(bp, false);
8782         bnxt_set_tpa_flags(bp);
8783         bnxt_set_ring_params(bp);
8784         bnxt_set_max_func_irqs(bp, max_irqs);
8785         rc = bnxt_set_dflt_rings(bp, true);
8786         if (rc) {
8787                 netdev_err(bp->dev, "Not enough rings available.\n");
8788                 rc = -ENOMEM;
8789                 goto init_err_pci_clean;
8790         }
8791
8792         /* Default RSS hash cfg. */
8793         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
8794                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
8795                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
8796                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
8797         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
8798                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
8799                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
8800                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
8801         }
8802
8803         bnxt_hwrm_vnic_qcaps(bp);
8804         if (bnxt_rfs_supported(bp)) {
8805                 dev->hw_features |= NETIF_F_NTUPLE;
8806                 if (bnxt_rfs_capable(bp)) {
8807                         bp->flags |= BNXT_FLAG_RFS;
8808                         dev->features |= NETIF_F_NTUPLE;
8809                 }
8810         }
8811
8812         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
8813                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
8814
8815         rc = bnxt_init_int_mode(bp);
8816         if (rc)
8817                 goto init_err_pci_clean;
8818
8819         /* No TC has been set yet and rings may have been trimmed due to
8820          * limited MSIX, so we re-initialize the TX rings per TC.
8821          */
8822         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8823
8824         bnxt_get_wol_settings(bp);
8825         if (bp->flags & BNXT_FLAG_WOL_CAP)
8826                 device_set_wakeup_enable(&pdev->dev, bp->wol);
8827         else
8828                 device_set_wakeup_capable(&pdev->dev, false);
8829
8830         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
8831
8832         if (BNXT_PF(bp)) {
8833                 if (!bnxt_pf_wq) {
8834                         bnxt_pf_wq =
8835                                 create_singlethread_workqueue("bnxt_pf_wq");
8836                         if (!bnxt_pf_wq) {
8837                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
8838                                 goto init_err_pci_clean;
8839                         }
8840                 }
8841                 bnxt_init_tc(bp);
8842         }
8843
8844         rc = register_netdev(dev);
8845         if (rc)
8846                 goto init_err_cleanup_tc;
8847
8848         if (BNXT_PF(bp))
8849                 bnxt_dl_register(bp);
8850
8851         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
8852                     board_info[ent->driver_data].name,
8853                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
8854
8855         bnxt_parse_log_pcie_link(bp);
8856
8857         return 0;
8858
8859 init_err_cleanup_tc:
8860         bnxt_shutdown_tc(bp);
8861         bnxt_clear_int_mode(bp);
8862
8863 init_err_pci_clean:
8864         bnxt_cleanup_pci(bp);
8865
8866 init_err_free:
8867         free_netdev(dev);
8868         return rc;
8869 }
8870
8871 static void bnxt_shutdown(struct pci_dev *pdev)
8872 {
8873         struct net_device *dev = pci_get_drvdata(pdev);
8874         struct bnxt *bp;
8875
8876         if (!dev)
8877                 return;
8878
8879         rtnl_lock();
8880         bp = netdev_priv(dev);
8881         if (!bp)
8882                 goto shutdown_exit;
8883
8884         if (netif_running(dev))
8885                 dev_close(dev);
8886
8887         bnxt_ulp_shutdown(bp);
8888
8889         if (system_state == SYSTEM_POWER_OFF) {
8890                 bnxt_clear_int_mode(bp);
8891                 pci_wake_from_d3(pdev, bp->wol);
8892                 pci_set_power_state(pdev, PCI_D3hot);
8893         }
8894
8895 shutdown_exit:
8896         rtnl_unlock();
8897 }
8898
8899 #ifdef CONFIG_PM_SLEEP
8900 static int bnxt_suspend(struct device *device)
8901 {
8902         struct pci_dev *pdev = to_pci_dev(device);
8903         struct net_device *dev = pci_get_drvdata(pdev);
8904         struct bnxt *bp = netdev_priv(dev);
8905         int rc = 0;
8906
8907         rtnl_lock();
8908         if (netif_running(dev)) {
8909                 netif_device_detach(dev);
8910                 rc = bnxt_close(dev);
8911         }
8912         bnxt_hwrm_func_drv_unrgtr(bp);
8913         rtnl_unlock();
8914         return rc;
8915 }
8916
8917 static int bnxt_resume(struct device *device)
8918 {
8919         struct pci_dev *pdev = to_pci_dev(device);
8920         struct net_device *dev = pci_get_drvdata(pdev);
8921         struct bnxt *bp = netdev_priv(dev);
8922         int rc = 0;
8923
8924         rtnl_lock();
8925         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
8926                 rc = -ENODEV;
8927                 goto resume_exit;
8928         }
8929         rc = bnxt_hwrm_func_reset(bp);
8930         if (rc) {
8931                 rc = -EBUSY;
8932                 goto resume_exit;
8933         }
8934         bnxt_get_wol_settings(bp);
8935         if (netif_running(dev)) {
8936                 rc = bnxt_open(dev);
8937                 if (!rc)
8938                         netif_device_attach(dev);
8939         }
8940
8941 resume_exit:
8942         rtnl_unlock();
8943         return rc;
8944 }
8945
8946 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
8947 #define BNXT_PM_OPS (&bnxt_pm_ops)
8948
8949 #else
8950
8951 #define BNXT_PM_OPS NULL
8952
8953 #endif /* CONFIG_PM_SLEEP */
8954
8955 /**
8956  * bnxt_io_error_detected - called when PCI error is detected
8957  * @pdev: Pointer to PCI device
8958  * @state: The current pci connection state
8959  *
8960  * This function is called after a PCI bus error affecting
8961  * this device has been detected.
8962  */
8963 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
8964                                                pci_channel_state_t state)
8965 {
8966         struct net_device *netdev = pci_get_drvdata(pdev);
8967         struct bnxt *bp = netdev_priv(netdev);
8968
8969         netdev_info(netdev, "PCI I/O error detected\n");
8970
8971         rtnl_lock();
8972         netif_device_detach(netdev);
8973
8974         bnxt_ulp_stop(bp);
8975
8976         if (state == pci_channel_io_perm_failure) {
8977                 rtnl_unlock();
8978                 return PCI_ERS_RESULT_DISCONNECT;
8979         }
8980
8981         if (netif_running(netdev))
8982                 bnxt_close(netdev);
8983
8984         pci_disable_device(pdev);
8985         rtnl_unlock();
8986
8987         /* Request a slot slot reset. */
8988         return PCI_ERS_RESULT_NEED_RESET;
8989 }
8990
8991 /**
8992  * bnxt_io_slot_reset - called after the pci bus has been reset.
8993  * @pdev: Pointer to PCI device
8994  *
8995  * Restart the card from scratch, as if from a cold-boot.
8996  * At this point, the card has exprienced a hard reset,
8997  * followed by fixups by BIOS, and has its config space
8998  * set up identically to what it was at cold boot.
8999  */
9000 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
9001 {
9002         struct net_device *netdev = pci_get_drvdata(pdev);
9003         struct bnxt *bp = netdev_priv(netdev);
9004         int err = 0;
9005         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
9006
9007         netdev_info(bp->dev, "PCI Slot Reset\n");
9008
9009         rtnl_lock();
9010
9011         if (pci_enable_device(pdev)) {
9012                 dev_err(&pdev->dev,
9013                         "Cannot re-enable PCI device after reset.\n");
9014         } else {
9015                 pci_set_master(pdev);
9016
9017                 err = bnxt_hwrm_func_reset(bp);
9018                 if (!err && netif_running(netdev))
9019                         err = bnxt_open(netdev);
9020
9021                 if (!err) {
9022                         result = PCI_ERS_RESULT_RECOVERED;
9023                         bnxt_ulp_start(bp);
9024                 }
9025         }
9026
9027         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
9028                 dev_close(netdev);
9029
9030         rtnl_unlock();
9031
9032         err = pci_cleanup_aer_uncorrect_error_status(pdev);
9033         if (err) {
9034                 dev_err(&pdev->dev,
9035                         "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
9036                          err); /* non-fatal, continue */
9037         }
9038
9039         return PCI_ERS_RESULT_RECOVERED;
9040 }
9041
9042 /**
9043  * bnxt_io_resume - called when traffic can start flowing again.
9044  * @pdev: Pointer to PCI device
9045  *
9046  * This callback is called when the error recovery driver tells
9047  * us that its OK to resume normal operation.
9048  */
9049 static void bnxt_io_resume(struct pci_dev *pdev)
9050 {
9051         struct net_device *netdev = pci_get_drvdata(pdev);
9052
9053         rtnl_lock();
9054
9055         netif_device_attach(netdev);
9056
9057         rtnl_unlock();
9058 }
9059
9060 static const struct pci_error_handlers bnxt_err_handler = {
9061         .error_detected = bnxt_io_error_detected,
9062         .slot_reset     = bnxt_io_slot_reset,
9063         .resume         = bnxt_io_resume
9064 };
9065
9066 static struct pci_driver bnxt_pci_driver = {
9067         .name           = DRV_MODULE_NAME,
9068         .id_table       = bnxt_pci_tbl,
9069         .probe          = bnxt_init_one,
9070         .remove         = bnxt_remove_one,
9071         .shutdown       = bnxt_shutdown,
9072         .driver.pm      = BNXT_PM_OPS,
9073         .err_handler    = &bnxt_err_handler,
9074 #if defined(CONFIG_BNXT_SRIOV)
9075         .sriov_configure = bnxt_sriov_configure,
9076 #endif
9077 };
9078
9079 static int __init bnxt_init(void)
9080 {
9081         return pci_register_driver(&bnxt_pci_driver);
9082 }
9083
9084 static void __exit bnxt_exit(void)
9085 {
9086         pci_unregister_driver(&bnxt_pci_driver);
9087         if (bnxt_pf_wq)
9088                 destroy_workqueue(bnxt_pf_wq);
9089 }
9090
9091 module_init(bnxt_init);
9092 module_exit(bnxt_exit);