drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72
  73 static const char version[] =
  74         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  75
  76 MODULE_LICENSE("GPL");
  77 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  78 MODULE_VERSION(DRV_MODULE_VERSION);
  79
  80 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  81 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  82 #define BNXT_RX_COPY_THRESH 256
  83
  84 #define BNXT_TX_PUSH_THRESH 164
  85
  86 enum board_idx {
  87         BCM57301,
  88         BCM57302,
  89         BCM57304,
  90         BCM57417_NPAR,
  91         BCM58700,
  92         BCM57311,
  93         BCM57312,
  94         BCM57402,
  95         BCM57404,
  96         BCM57406,
  97         BCM57402_NPAR,
  98         BCM57407,
  99         BCM57412,
 100         BCM57414,
 101         BCM57416,
 102         BCM57417,
 103         BCM57412_NPAR,
 104         BCM57314,
 105         BCM57417_SFP,
 106         BCM57416_SFP,
 107         BCM57404_NPAR,
 108         BCM57406_NPAR,
 109         BCM57407_SFP,
 110         BCM57407_NPAR,
 111         BCM57414_NPAR,
 112         BCM57416_NPAR,
 113         BCM57452,
 114         BCM57454,
 115         BCM5745x_NPAR,
 116         BCM57508,
 117         BCM57504,
 118         BCM57502,
 119         BCM57508_NPAR,
 120         BCM57504_NPAR,
 121         BCM57502_NPAR,
 122         BCM58802,
 123         BCM58804,
 124         BCM58808,
 125         NETXTREME_E_VF,
 126         NETXTREME_C_VF,
 127         NETXTREME_S_VF,
 128         NETXTREME_E_P5_VF,
 129 };
 130
 131 /* indexed by enum above */
 132 static const struct {
 133         char *name;
 134 } board_info[] = {
 135         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 136         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 137         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 138         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 139         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 140         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 141         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 142         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 143         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 144         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 145         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 146         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 148         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 149         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 150         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 151         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 152         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 153         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 154         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 155         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 156         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 157         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 158         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 159         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 160         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 161         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 162         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 163         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 164         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 165         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 166         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 167         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 168         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 169         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 170         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 171         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 172         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 173         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 174         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 175         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 176         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 177 };
 178
 179 static const struct pci_device_id bnxt_pci_tbl[] = {
 180         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 183         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 185         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 188         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 189         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 190         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 192         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 195         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 197         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 198         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 199         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 200         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 201         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 202         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 203         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 207         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 211         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 212         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 213         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 214         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 215         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 216         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 217         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 218         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 222         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 223         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 224         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 225         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 226 #ifdef CONFIG_BNXT_SRIOV
 227         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 234         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 235         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 236         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 237         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 238 #endif
 239         { 0 }
 240 };
 241
 242 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 243
 244 static const u16 bnxt_vf_req_snif[] = {
 245         HWRM_FUNC_CFG,
 246         HWRM_FUNC_VF_CFG,
 247         HWRM_PORT_PHY_QCFG,
 248         HWRM_CFA_L2_FILTER_ALLOC,
 249 };
 250
 251 static const u16 bnxt_async_events_arr[] = {
 252         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 253         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 254         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 255         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 257 };
 258
 259 static struct workqueue_struct *bnxt_pf_wq;
 260
 261 static bool bnxt_vf_pciid(enum board_idx idx)
 262 {
 263         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 264                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 265 }
 266
 267 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 268 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 269 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 270
 271 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 272                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 273
 274 #define BNXT_DB_CQ(db, idx)                                             \
 275         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 276
 277 #define BNXT_DB_NQ_P5(db, idx)                                          \
 278         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 279
 280 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 281         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 282
 283 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 284         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 285
 286 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 287 {
 288         if (bp->flags & BNXT_FLAG_CHIP_P5)
 289                 BNXT_DB_NQ_P5(db, idx);
 290         else
 291                 BNXT_DB_CQ(db, idx);
 292 }
 293
 294 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 295 {
 296         if (bp->flags & BNXT_FLAG_CHIP_P5)
 297                 BNXT_DB_NQ_ARM_P5(db, idx);
 298         else
 299                 BNXT_DB_CQ_ARM(db, idx);
 300 }
 301
 302 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 303 {
 304         if (bp->flags & BNXT_FLAG_CHIP_P5)
 305                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 306                        db->doorbell);
 307         else
 308                 BNXT_DB_CQ(db, idx);
 309 }
 310
 311 const u16 bnxt_lhint_arr[] = {
 312         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 313         TX_BD_FLAGS_LHINT_512_TO_1023,
 314         TX_BD_FLAGS_LHINT_1024_TO_2047,
 315         TX_BD_FLAGS_LHINT_1024_TO_2047,
 316         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 317         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331 };
 332
 333 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 334 {
 335         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 336
 337         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 338                 return 0;
 339
 340         return md_dst->u.port_info.port_id;
 341 }
 342
 343 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 344 {
 345         struct bnxt *bp = netdev_priv(dev);
 346         struct tx_bd *txbd;
 347         struct tx_bd_ext *txbd1;
 348         struct netdev_queue *txq;
 349         int i;
 350         dma_addr_t mapping;
 351         unsigned int length, pad = 0;
 352         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 353         u16 prod, last_frag;
 354         struct pci_dev *pdev = bp->pdev;
 355         struct bnxt_tx_ring_info *txr;
 356         struct bnxt_sw_tx_bd *tx_buf;
 357
 358         i = skb_get_queue_mapping(skb);
 359         if (unlikely(i >= bp->tx_nr_rings)) {
 360                 dev_kfree_skb_any(skb);
 361                 return NETDEV_TX_OK;
 362         }
 363
 364         txq = netdev_get_tx_queue(dev, i);
 365         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 366         prod = txr->tx_prod;
 367
 368         free_size = bnxt_tx_avail(bp, txr);
 369         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 370                 netif_tx_stop_queue(txq);
 371                 return NETDEV_TX_BUSY;
 372         }
 373
 374         length = skb->len;
 375         len = skb_headlen(skb);
 376         last_frag = skb_shinfo(skb)->nr_frags;
 377
 378         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 379
 380         txbd->tx_bd_opaque = prod;
 381
 382         tx_buf = &txr->tx_buf_ring[prod];
 383         tx_buf->skb = skb;
 384         tx_buf->nr_frags = last_frag;
 385
 386         vlan_tag_flags = 0;
 387         cfa_action = bnxt_xmit_get_cfa_action(skb);
 388         if (skb_vlan_tag_present(skb)) {
 389                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 390                                  skb_vlan_tag_get(skb);
 391                 /* Currently supports 8021Q, 8021AD vlan offloads
 392                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 393                  */
 394                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 395                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 396         }
 397
 398         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 399                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 400                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 401                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 402                 void __iomem *db = txr->tx_db.doorbell;
 403                 void *pdata = tx_push_buf->data;
 404                 u64 *end;
 405                 int j, push_len;
 406
 407                 /* Set COAL_NOW to be ready quickly for the next push */
 408                 tx_push->tx_bd_len_flags_type =
 409                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 410                                         TX_BD_TYPE_LONG_TX_BD |
 411                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 412                                         TX_BD_FLAGS_COAL_NOW |
 413                                         TX_BD_FLAGS_PACKET_END |
 414                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 415
 416                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 417                         tx_push1->tx_bd_hsize_lflags =
 418                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 419                 else
 420                         tx_push1->tx_bd_hsize_lflags = 0;
 421
 422                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 423                 tx_push1->tx_bd_cfa_action =
 424                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 425
 426                 end = pdata + length;
 427                 end = PTR_ALIGN(end, 8) - 1;
 428                 *end = 0;
 429
 430                 skb_copy_from_linear_data(skb, pdata, len);
 431                 pdata += len;
 432                 for (j = 0; j < last_frag; j++) {
 433                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 434                         void *fptr;
 435
 436                         fptr = skb_frag_address_safe(frag);
 437                         if (!fptr)
 438                                 goto normal_tx;
 439
 440                         memcpy(pdata, fptr, skb_frag_size(frag));
 441                         pdata += skb_frag_size(frag);
 442                 }
 443
 444                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 445                 txbd->tx_bd_haddr = txr->data_mapping;
 446                 prod = NEXT_TX(prod);
 447                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 448                 memcpy(txbd, tx_push1, sizeof(*txbd));
 449                 prod = NEXT_TX(prod);
 450                 tx_push->doorbell =
 451                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 452                 txr->tx_prod = prod;
 453
 454                 tx_buf->is_push = 1;
 455                 netdev_tx_sent_queue(txq, skb->len);
 456                 wmb();  /* Sync is_push and byte queue before pushing data */
 457
 458                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 459                 if (push_len > 16) {
 460                         __iowrite64_copy(db, tx_push_buf, 16);
 461                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 462                                          (push_len - 16) << 1);
 463                 } else {
 464                         __iowrite64_copy(db, tx_push_buf, push_len);
 465                 }
 466
 467                 goto tx_done;
 468         }
 469
 470 normal_tx:
 471         if (length < BNXT_MIN_PKT_SIZE) {
 472                 pad = BNXT_MIN_PKT_SIZE - length;
 473                 if (skb_pad(skb, pad)) {
 474                         /* SKB already freed. */
 475                         tx_buf->skb = NULL;
 476                         return NETDEV_TX_OK;
 477                 }
 478                 length = BNXT_MIN_PKT_SIZE;
 479         }
 480
 481         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 482
 483         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 484                 dev_kfree_skb_any(skb);
 485                 tx_buf->skb = NULL;
 486                 return NETDEV_TX_OK;
 487         }
 488
 489         dma_unmap_addr_set(tx_buf, mapping, mapping);
 490         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 491                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 492
 493         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 494
 495         prod = NEXT_TX(prod);
 496         txbd1 = (struct tx_bd_ext *)
 497                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 498
 499         txbd1->tx_bd_hsize_lflags = 0;
 500         if (skb_is_gso(skb)) {
 501                 u32 hdr_len;
 502
 503                 if (skb->encapsulation)
 504                         hdr_len = skb_inner_network_offset(skb) +
 505                                 skb_inner_network_header_len(skb) +
 506                                 inner_tcp_hdrlen(skb);
 507                 else
 508                         hdr_len = skb_transport_offset(skb) +
 509                                 tcp_hdrlen(skb);
 510
 511                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 512                                         TX_BD_FLAGS_T_IPID |
 513                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 514                 length = skb_shinfo(skb)->gso_size;
 515                 txbd1->tx_bd_mss = cpu_to_le32(length);
 516                 length += hdr_len;
 517         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 518                 txbd1->tx_bd_hsize_lflags =
 519                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 520                 txbd1->tx_bd_mss = 0;
 521         }
 522
 523         length >>= 9;
 524         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 525                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 526                                      skb->len);
 527                 i = 0;
 528                 goto tx_dma_error;
 529         }
 530         flags |= bnxt_lhint_arr[length];
 531         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 532
 533         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 534         txbd1->tx_bd_cfa_action =
 535                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 536         for (i = 0; i < last_frag; i++) {
 537                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 538
 539                 prod = NEXT_TX(prod);
 540                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 541
 542                 len = skb_frag_size(frag);
 543                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 544                                            DMA_TO_DEVICE);
 545
 546                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 547                         goto tx_dma_error;
 548
 549                 tx_buf = &txr->tx_buf_ring[prod];
 550                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 551
 552                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 553
 554                 flags = len << TX_BD_LEN_SHIFT;
 555                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 556         }
 557
 558         flags &= ~TX_BD_LEN;
 559         txbd->tx_bd_len_flags_type =
 560                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 561                             TX_BD_FLAGS_PACKET_END);
 562
 563         netdev_tx_sent_queue(txq, skb->len);
 564
 565         /* Sync BD data before updating doorbell */
 566         wmb();
 567
 568         prod = NEXT_TX(prod);
 569         txr->tx_prod = prod;
 570
 571         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 572                 bnxt_db_write(bp, &txr->tx_db, prod);
 573
 574 tx_done:
 575
 576         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 577                 if (netdev_xmit_more() && !tx_buf->is_push)
 578                         bnxt_db_write(bp, &txr->tx_db, prod);
 579
 580                 netif_tx_stop_queue(txq);
 581
 582                 /* netif_tx_stop_queue() must be done before checking
 583                  * tx index in bnxt_tx_avail() below, because in
 584                  * bnxt_tx_int(), we update tx index before checking for
 585                  * netif_tx_queue_stopped().
 586                  */
 587                 smp_mb();
 588                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 589                         netif_tx_wake_queue(txq);
 590         }
 591         return NETDEV_TX_OK;
 592
 593 tx_dma_error:
 594         last_frag = i;
 595
 596         /* start back at beginning and unmap skb */
 597         prod = txr->tx_prod;
 598         tx_buf = &txr->tx_buf_ring[prod];
 599         tx_buf->skb = NULL;
 600         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 601                          skb_headlen(skb), PCI_DMA_TODEVICE);
 602         prod = NEXT_TX(prod);
 603
 604         /* unmap remaining mapped pages */
 605         for (i = 0; i < last_frag; i++) {
 606                 prod = NEXT_TX(prod);
 607                 tx_buf = &txr->tx_buf_ring[prod];
 608                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 609                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 610                                PCI_DMA_TODEVICE);
 611         }
 612
 613         dev_kfree_skb_any(skb);
 614         return NETDEV_TX_OK;
 615 }
 616
 617 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 618 {
 619         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 620         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 621         u16 cons = txr->tx_cons;
 622         struct pci_dev *pdev = bp->pdev;
 623         int i;
 624         unsigned int tx_bytes = 0;
 625
 626         for (i = 0; i < nr_pkts; i++) {
 627                 struct bnxt_sw_tx_bd *tx_buf;
 628                 struct sk_buff *skb;
 629                 int j, last;
 630
 631                 tx_buf = &txr->tx_buf_ring[cons];
 632                 cons = NEXT_TX(cons);
 633                 skb = tx_buf->skb;
 634                 tx_buf->skb = NULL;
 635
 636                 if (tx_buf->is_push) {
 637                         tx_buf->is_push = 0;
 638                         goto next_tx_int;
 639                 }
 640
 641                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 642                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 643                 last = tx_buf->nr_frags;
 644
 645                 for (j = 0; j < last; j++) {
 646                         cons = NEXT_TX(cons);
 647                         tx_buf = &txr->tx_buf_ring[cons];
 648                         dma_unmap_page(
 649                                 &pdev->dev,
 650                                 dma_unmap_addr(tx_buf, mapping),
 651                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 652                                 PCI_DMA_TODEVICE);
 653                 }
 654
 655 next_tx_int:
 656                 cons = NEXT_TX(cons);
 657
 658                 tx_bytes += skb->len;
 659                 dev_kfree_skb_any(skb);
 660         }
 661
 662         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 663         txr->tx_cons = cons;
 664
 665         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 666          * before checking for netif_tx_queue_stopped().  Without the
 667          * memory barrier, there is a small possibility that bnxt_start_xmit()
 668          * will miss it and cause the queue to be stopped forever.
 669          */
 670         smp_mb();
 671
 672         if (unlikely(netif_tx_queue_stopped(txq)) &&
 673             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 674                 __netif_tx_lock(txq, smp_processor_id());
 675                 if (netif_tx_queue_stopped(txq) &&
 676                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 677                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 678                         netif_tx_wake_queue(txq);
 679                 __netif_tx_unlock(txq);
 680         }
 681 }
 682
 683 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 684                                          struct bnxt_rx_ring_info *rxr,
 685                                          gfp_t gfp)
 686 {
 687         struct device *dev = &bp->pdev->dev;
 688         struct page *page;
 689
 690         page = page_pool_dev_alloc_pages(rxr->page_pool);
 691         if (!page)
 692                 return NULL;
 693
 694         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 695                                       DMA_ATTR_WEAK_ORDERING);
 696         if (dma_mapping_error(dev, *mapping)) {
 697                 page_pool_recycle_direct(rxr->page_pool, page);
 698                 return NULL;
 699         }
 700         *mapping += bp->rx_dma_offset;
 701         return page;
 702 }
 703
 704 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 705                                        gfp_t gfp)
 706 {
 707         u8 *data;
 708         struct pci_dev *pdev = bp->pdev;
 709
 710         data = kmalloc(bp->rx_buf_size, gfp);
 711         if (!data)
 712                 return NULL;
 713
 714         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 715                                         bp->rx_buf_use_size, bp->rx_dir,
 716                                         DMA_ATTR_WEAK_ORDERING);
 717
 718         if (dma_mapping_error(&pdev->dev, *mapping)) {
 719                 kfree(data);
 720                 data = NULL;
 721         }
 722         return data;
 723 }
 724
 725 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 726                        u16 prod, gfp_t gfp)
 727 {
 728         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 729         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 730         dma_addr_t mapping;
 731
 732         if (BNXT_RX_PAGE_MODE(bp)) {
 733                 struct page *page =
 734                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 735
 736                 if (!page)
 737                         return -ENOMEM;
 738
 739                 rx_buf->data = page;
 740                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 741         } else {
 742                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 743
 744                 if (!data)
 745                         return -ENOMEM;
 746
 747                 rx_buf->data = data;
 748                 rx_buf->data_ptr = data + bp->rx_offset;
 749         }
 750         rx_buf->mapping = mapping;
 751
 752         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 753         return 0;
 754 }
 755
 756 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 757 {
 758         u16 prod = rxr->rx_prod;
 759         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 760         struct rx_bd *cons_bd, *prod_bd;
 761
 762         prod_rx_buf = &rxr->rx_buf_ring[prod];
 763         cons_rx_buf = &rxr->rx_buf_ring[cons];
 764
 765         prod_rx_buf->data = data;
 766         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 767
 768         prod_rx_buf->mapping = cons_rx_buf->mapping;
 769
 770         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 771         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 772
 773         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 774 }
 775
 776 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 777 {
 778         u16 next, max = rxr->rx_agg_bmap_size;
 779
 780         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 781         if (next >= max)
 782                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 783         return next;
 784 }
 785
 786 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 787                                      struct bnxt_rx_ring_info *rxr,
 788                                      u16 prod, gfp_t gfp)
 789 {
 790         struct rx_bd *rxbd =
 791                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 792         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 793         struct pci_dev *pdev = bp->pdev;
 794         struct page *page;
 795         dma_addr_t mapping;
 796         u16 sw_prod = rxr->rx_sw_agg_prod;
 797         unsigned int offset = 0;
 798
 799         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 800                 page = rxr->rx_page;
 801                 if (!page) {
 802                         page = alloc_page(gfp);
 803                         if (!page)
 804                                 return -ENOMEM;
 805                         rxr->rx_page = page;
 806                         rxr->rx_page_offset = 0;
 807                 }
 808                 offset = rxr->rx_page_offset;
 809                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 810                 if (rxr->rx_page_offset == PAGE_SIZE)
 811                         rxr->rx_page = NULL;
 812                 else
 813                         get_page(page);
 814         } else {
 815                 page = alloc_page(gfp);
 816                 if (!page)
 817                         return -ENOMEM;
 818         }
 819
 820         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 821                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 822                                      DMA_ATTR_WEAK_ORDERING);
 823         if (dma_mapping_error(&pdev->dev, mapping)) {
 824                 __free_page(page);
 825                 return -EIO;
 826         }
 827
 828         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 829                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 830
 831         __set_bit(sw_prod, rxr->rx_agg_bmap);
 832         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 833         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 834
 835         rx_agg_buf->page = page;
 836         rx_agg_buf->offset = offset;
 837         rx_agg_buf->mapping = mapping;
 838         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 839         rxbd->rx_bd_opaque = sw_prod;
 840         return 0;
 841 }
 842
 843 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 844                                        struct bnxt_cp_ring_info *cpr,
 845                                        u16 cp_cons, u16 curr)
 846 {
 847         struct rx_agg_cmp *agg;
 848
 849         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 850         agg = (struct rx_agg_cmp *)
 851                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 852         return agg;
 853 }
 854
 855 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 856                                               struct bnxt_rx_ring_info *rxr,
 857                                               u16 agg_id, u16 curr)
 858 {
 859         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 860
 861         return &tpa_info->agg_arr[curr];
 862 }
 863
 864 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 865                                    u16 start, u32 agg_bufs, bool tpa)
 866 {
 867         struct bnxt_napi *bnapi = cpr->bnapi;
 868         struct bnxt *bp = bnapi->bp;
 869         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 870         u16 prod = rxr->rx_agg_prod;
 871         u16 sw_prod = rxr->rx_sw_agg_prod;
 872         bool p5_tpa = false;
 873         u32 i;
 874
 875         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 876                 p5_tpa = true;
 877
 878         for (i = 0; i < agg_bufs; i++) {
 879                 u16 cons;
 880                 struct rx_agg_cmp *agg;
 881                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 882                 struct rx_bd *prod_bd;
 883                 struct page *page;
 884
 885                 if (p5_tpa)
 886                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 887                 else
 888                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 889                 cons = agg->rx_agg_cmp_opaque;
 890                 __clear_bit(cons, rxr->rx_agg_bmap);
 891
 892                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 893                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 894
 895                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 896                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 897                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 898
 899                 /* It is possible for sw_prod to be equal to cons, so
 900                  * set cons_rx_buf->page to NULL first.
 901                  */
 902                 page = cons_rx_buf->page;
 903                 cons_rx_buf->page = NULL;
 904                 prod_rx_buf->page = page;
 905                 prod_rx_buf->offset = cons_rx_buf->offset;
 906
 907                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 908
 909                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 910
 911                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 912                 prod_bd->rx_bd_opaque = sw_prod;
 913
 914                 prod = NEXT_RX_AGG(prod);
 915                 sw_prod = NEXT_RX_AGG(sw_prod);
 916         }
 917         rxr->rx_agg_prod = prod;
 918         rxr->rx_sw_agg_prod = sw_prod;
 919 }
 920
 921 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 922                                         struct bnxt_rx_ring_info *rxr,
 923                                         u16 cons, void *data, u8 *data_ptr,
 924                                         dma_addr_t dma_addr,
 925                                         unsigned int offset_and_len)
 926 {
 927         unsigned int payload = offset_and_len >> 16;
 928         unsigned int len = offset_and_len & 0xffff;
 929         skb_frag_t *frag;
 930         struct page *page = data;
 931         u16 prod = rxr->rx_prod;
 932         struct sk_buff *skb;
 933         int off, err;
 934
 935         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 936         if (unlikely(err)) {
 937                 bnxt_reuse_rx_data(rxr, cons, data);
 938                 return NULL;
 939         }
 940         dma_addr -= bp->rx_dma_offset;
 941         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 942                              DMA_ATTR_WEAK_ORDERING);
 943
 944         if (unlikely(!payload))
 945                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 946
 947         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 948         if (!skb) {
 949                 __free_page(page);
 950                 return NULL;
 951         }
 952
 953         off = (void *)data_ptr - page_address(page);
 954         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 955         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 956                payload + NET_IP_ALIGN);
 957
 958         frag = &skb_shinfo(skb)->frags[0];
 959         skb_frag_size_sub(frag, payload);
 960         skb_frag_off_add(frag, payload);
 961         skb->data_len -= payload;
 962         skb->tail += payload;
 963
 964         return skb;
 965 }
 966
 967 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 968                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 969                                    void *data, u8 *data_ptr,
 970                                    dma_addr_t dma_addr,
 971                                    unsigned int offset_and_len)
 972 {
 973         u16 prod = rxr->rx_prod;
 974         struct sk_buff *skb;
 975         int err;
 976
 977         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 978         if (unlikely(err)) {
 979                 bnxt_reuse_rx_data(rxr, cons, data);
 980                 return NULL;
 981         }
 982
 983         skb = build_skb(data, 0);
 984         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 985                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 986         if (!skb) {
 987                 kfree(data);
 988                 return NULL;
 989         }
 990
 991         skb_reserve(skb, bp->rx_offset);
 992         skb_put(skb, offset_and_len & 0xffff);
 993         return skb;
 994 }
 995
 996 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
 997                                      struct bnxt_cp_ring_info *cpr,
 998                                      struct sk_buff *skb, u16 idx,
 999                                      u32 agg_bufs, bool tpa)
1000 {
1001         struct bnxt_napi *bnapi = cpr->bnapi;
1002         struct pci_dev *pdev = bp->pdev;
1003         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1004         u16 prod = rxr->rx_agg_prod;
1005         bool p5_tpa = false;
1006         u32 i;
1007
1008         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1009                 p5_tpa = true;
1010
1011         for (i = 0; i < agg_bufs; i++) {
1012                 u16 cons, frag_len;
1013                 struct rx_agg_cmp *agg;
1014                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1015                 struct page *page;
1016                 dma_addr_t mapping;
1017
1018                 if (p5_tpa)
1019                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1020                 else
1021                         agg = bnxt_get_agg(bp, cpr, idx, i);
1022                 cons = agg->rx_agg_cmp_opaque;
1023                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1024                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1025
1026                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1027                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1028                                    cons_rx_buf->offset, frag_len);
1029                 __clear_bit(cons, rxr->rx_agg_bmap);
1030
1031                 /* It is possible for bnxt_alloc_rx_page() to allocate
1032                  * a sw_prod index that equals the cons index, so we
1033                  * need to clear the cons entry now.
1034                  */
1035                 mapping = cons_rx_buf->mapping;
1036                 page = cons_rx_buf->page;
1037                 cons_rx_buf->page = NULL;
1038
1039                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1040                         struct skb_shared_info *shinfo;
1041                         unsigned int nr_frags;
1042
1043                         shinfo = skb_shinfo(skb);
1044                         nr_frags = --shinfo->nr_frags;
1045                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1046
1047                         dev_kfree_skb(skb);
1048
1049                         cons_rx_buf->page = page;
1050
1051                         /* Update prod since possibly some pages have been
1052                          * allocated already.
1053                          */
1054                         rxr->rx_agg_prod = prod;
1055                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1056                         return NULL;
1057                 }
1058
1059                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1060                                      PCI_DMA_FROMDEVICE,
1061                                      DMA_ATTR_WEAK_ORDERING);
1062
1063                 skb->data_len += frag_len;
1064                 skb->len += frag_len;
1065                 skb->truesize += PAGE_SIZE;
1066
1067                 prod = NEXT_RX_AGG(prod);
1068         }
1069         rxr->rx_agg_prod = prod;
1070         return skb;
1071 }
1072
1073 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1074                                u8 agg_bufs, u32 *raw_cons)
1075 {
1076         u16 last;
1077         struct rx_agg_cmp *agg;
1078
1079         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1080         last = RING_CMP(*raw_cons);
1081         agg = (struct rx_agg_cmp *)
1082                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1083         return RX_AGG_CMP_VALID(agg, *raw_cons);
1084 }
1085
1086 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1087                                             unsigned int len,
1088                                             dma_addr_t mapping)
1089 {
1090         struct bnxt *bp = bnapi->bp;
1091         struct pci_dev *pdev = bp->pdev;
1092         struct sk_buff *skb;
1093
1094         skb = napi_alloc_skb(&bnapi->napi, len);
1095         if (!skb)
1096                 return NULL;
1097
1098         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1099                                 bp->rx_dir);
1100
1101         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1102                len + NET_IP_ALIGN);
1103
1104         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1105                                    bp->rx_dir);
1106
1107         skb_put(skb, len);
1108         return skb;
1109 }
1110
1111 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1112                            u32 *raw_cons, void *cmp)
1113 {
1114         struct rx_cmp *rxcmp = cmp;
1115         u32 tmp_raw_cons = *raw_cons;
1116         u8 cmp_type, agg_bufs = 0;
1117
1118         cmp_type = RX_CMP_TYPE(rxcmp);
1119
1120         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1121                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1122                             RX_CMP_AGG_BUFS) >>
1123                            RX_CMP_AGG_BUFS_SHIFT;
1124         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1125                 struct rx_tpa_end_cmp *tpa_end = cmp;
1126
1127                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1128                         return 0;
1129
1130                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1131         }
1132
1133         if (agg_bufs) {
1134                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1135                         return -EBUSY;
1136         }
1137         *raw_cons = tmp_raw_cons;
1138         return 0;
1139 }
1140
1141 static void bnxt_queue_sp_work(struct bnxt *bp)
1142 {
1143         if (BNXT_PF(bp))
1144                 queue_work(bnxt_pf_wq, &bp->sp_task);
1145         else
1146                 schedule_work(&bp->sp_task);
1147 }
1148
1149 static void bnxt_cancel_sp_work(struct bnxt *bp)
1150 {
1151         if (BNXT_PF(bp))
1152                 flush_workqueue(bnxt_pf_wq);
1153         else
1154                 cancel_work_sync(&bp->sp_task);
1155 }
1156
1157 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1158 {
1159         if (!rxr->bnapi->in_reset) {
1160                 rxr->bnapi->in_reset = true;
1161                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1162                 bnxt_queue_sp_work(bp);
1163         }
1164         rxr->rx_next_cons = 0xffff;
1165 }
1166
1167 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1168 {
1169         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1170         u16 idx = agg_id & MAX_TPA_P5_MASK;
1171
1172         if (test_bit(idx, map->agg_idx_bmap))
1173                 idx = find_first_zero_bit(map->agg_idx_bmap,
1174                                           BNXT_AGG_IDX_BMAP_SIZE);
1175         __set_bit(idx, map->agg_idx_bmap);
1176         map->agg_id_tbl[agg_id] = idx;
1177         return idx;
1178 }
1179
1180 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1181 {
1182         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1183
1184         __clear_bit(idx, map->agg_idx_bmap);
1185 }
1186
1187 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1188 {
1189         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1190
1191         return map->agg_id_tbl[agg_id];
1192 }
1193
1194 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1195                            struct rx_tpa_start_cmp *tpa_start,
1196                            struct rx_tpa_start_cmp_ext *tpa_start1)
1197 {
1198         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1199         struct bnxt_tpa_info *tpa_info;
1200         u16 cons, prod, agg_id;
1201         struct rx_bd *prod_bd;
1202         dma_addr_t mapping;
1203
1204         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1205                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1206                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1207         } else {
1208                 agg_id = TPA_START_AGG_ID(tpa_start);
1209         }
1210         cons = tpa_start->rx_tpa_start_cmp_opaque;
1211         prod = rxr->rx_prod;
1212         cons_rx_buf = &rxr->rx_buf_ring[cons];
1213         prod_rx_buf = &rxr->rx_buf_ring[prod];
1214         tpa_info = &rxr->rx_tpa[agg_id];
1215
1216         if (unlikely(cons != rxr->rx_next_cons ||
1217                      TPA_START_ERROR(tpa_start))) {
1218                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1219                             cons, rxr->rx_next_cons,
1220                             TPA_START_ERROR_CODE(tpa_start1));
1221                 bnxt_sched_reset(bp, rxr);
1222                 return;
1223         }
1224         /* Store cfa_code in tpa_info to use in tpa_end
1225          * completion processing.
1226          */
1227         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1228         prod_rx_buf->data = tpa_info->data;
1229         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1230
1231         mapping = tpa_info->mapping;
1232         prod_rx_buf->mapping = mapping;
1233
1234         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1235
1236         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1237
1238         tpa_info->data = cons_rx_buf->data;
1239         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1240         cons_rx_buf->data = NULL;
1241         tpa_info->mapping = cons_rx_buf->mapping;
1242
1243         tpa_info->len =
1244                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1245                                 RX_TPA_START_CMP_LEN_SHIFT;
1246         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1247                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1248
1249                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1250                 tpa_info->gso_type = SKB_GSO_TCPV4;
1251                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1252                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1253                         tpa_info->gso_type = SKB_GSO_TCPV6;
1254                 tpa_info->rss_hash =
1255                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1256         } else {
1257                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1258                 tpa_info->gso_type = 0;
1259                 if (netif_msg_rx_err(bp))
1260                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1261         }
1262         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1263         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1264         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1265         tpa_info->agg_count = 0;
1266
1267         rxr->rx_prod = NEXT_RX(prod);
1268         cons = NEXT_RX(cons);
1269         rxr->rx_next_cons = NEXT_RX(cons);
1270         cons_rx_buf = &rxr->rx_buf_ring[cons];
1271
1272         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1273         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1274         cons_rx_buf->data = NULL;
1275 }
1276
1277 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1278 {
1279         if (agg_bufs)
1280                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1281 }
1282
1283 #ifdef CONFIG_INET
1284 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1285 {
1286         struct udphdr *uh = NULL;
1287
1288         if (ip_proto == htons(ETH_P_IP)) {
1289                 struct iphdr *iph = (struct iphdr *)skb->data;
1290
1291                 if (iph->protocol == IPPROTO_UDP)
1292                         uh = (struct udphdr *)(iph + 1);
1293         } else {
1294                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1295
1296                 if (iph->nexthdr == IPPROTO_UDP)
1297                         uh = (struct udphdr *)(iph + 1);
1298         }
1299         if (uh) {
1300                 if (uh->check)
1301                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1302                 else
1303                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1304         }
1305 }
1306 #endif
1307
1308 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1309                                            int payload_off, int tcp_ts,
1310                                            struct sk_buff *skb)
1311 {
1312 #ifdef CONFIG_INET
1313         struct tcphdr *th;
1314         int len, nw_off;
1315         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1316         u32 hdr_info = tpa_info->hdr_info;
1317         bool loopback = false;
1318
1319         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1320         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1321         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1322
1323         /* If the packet is an internal loopback packet, the offsets will
1324          * have an extra 4 bytes.
1325          */
1326         if (inner_mac_off == 4) {
1327                 loopback = true;
1328         } else if (inner_mac_off > 4) {
1329                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1330                                             ETH_HLEN - 2));
1331
1332                 /* We only support inner iPv4/ipv6.  If we don't see the
1333                  * correct protocol ID, it must be a loopback packet where
1334                  * the offsets are off by 4.
1335                  */
1336                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1337                         loopback = true;
1338         }
1339         if (loopback) {
1340                 /* internal loopback packet, subtract all offsets by 4 */
1341                 inner_ip_off -= 4;
1342                 inner_mac_off -= 4;
1343                 outer_ip_off -= 4;
1344         }
1345
1346         nw_off = inner_ip_off - ETH_HLEN;
1347         skb_set_network_header(skb, nw_off);
1348         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1349                 struct ipv6hdr *iph = ipv6_hdr(skb);
1350
1351                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1352                 len = skb->len - skb_transport_offset(skb);
1353                 th = tcp_hdr(skb);
1354                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1355         } else {
1356                 struct iphdr *iph = ip_hdr(skb);
1357
1358                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1359                 len = skb->len - skb_transport_offset(skb);
1360                 th = tcp_hdr(skb);
1361                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1362         }
1363
1364         if (inner_mac_off) { /* tunnel */
1365                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1366                                             ETH_HLEN - 2));
1367
1368                 bnxt_gro_tunnel(skb, proto);
1369         }
1370 #endif
1371         return skb;
1372 }
1373
1374 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1375                                            int payload_off, int tcp_ts,
1376                                            struct sk_buff *skb)
1377 {
1378 #ifdef CONFIG_INET
1379         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1380         u32 hdr_info = tpa_info->hdr_info;
1381         int iphdr_len, nw_off;
1382
1383         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1384         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1385         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1386
1387         nw_off = inner_ip_off - ETH_HLEN;
1388         skb_set_network_header(skb, nw_off);
1389         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1390                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1391         skb_set_transport_header(skb, nw_off + iphdr_len);
1392
1393         if (inner_mac_off) { /* tunnel */
1394                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1395                                             ETH_HLEN - 2));
1396
1397                 bnxt_gro_tunnel(skb, proto);
1398         }
1399 #endif
1400         return skb;
1401 }
1402
1403 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1404 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1405
1406 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1407                                            int payload_off, int tcp_ts,
1408                                            struct sk_buff *skb)
1409 {
1410 #ifdef CONFIG_INET
1411         struct tcphdr *th;
1412         int len, nw_off, tcp_opt_len = 0;
1413
1414         if (tcp_ts)
1415                 tcp_opt_len = 12;
1416
1417         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1418                 struct iphdr *iph;
1419
1420                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1421                          ETH_HLEN;
1422                 skb_set_network_header(skb, nw_off);
1423                 iph = ip_hdr(skb);
1424                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1425                 len = skb->len - skb_transport_offset(skb);
1426                 th = tcp_hdr(skb);
1427                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1428         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1429                 struct ipv6hdr *iph;
1430
1431                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1432                          ETH_HLEN;
1433                 skb_set_network_header(skb, nw_off);
1434                 iph = ipv6_hdr(skb);
1435                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1436                 len = skb->len - skb_transport_offset(skb);
1437                 th = tcp_hdr(skb);
1438                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1439         } else {
1440                 dev_kfree_skb_any(skb);
1441                 return NULL;
1442         }
1443
1444         if (nw_off) /* tunnel */
1445                 bnxt_gro_tunnel(skb, skb->protocol);
1446 #endif
1447         return skb;
1448 }
1449
1450 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1451                                            struct bnxt_tpa_info *tpa_info,
1452                                            struct rx_tpa_end_cmp *tpa_end,
1453                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1454                                            struct sk_buff *skb)
1455 {
1456 #ifdef CONFIG_INET
1457         int payload_off;
1458         u16 segs;
1459
1460         segs = TPA_END_TPA_SEGS(tpa_end);
1461         if (segs == 1)
1462                 return skb;
1463
1464         NAPI_GRO_CB(skb)->count = segs;
1465         skb_shinfo(skb)->gso_size =
1466                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1467         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1468         if (bp->flags & BNXT_FLAG_CHIP_P5)
1469                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1470         else
1471                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1472         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1473         if (likely(skb))
1474                 tcp_gro_complete(skb);
1475 #endif
1476         return skb;
1477 }
1478
1479 /* Given the cfa_code of a received packet determine which
1480  * netdev (vf-rep or PF) the packet is destined to.
1481  */
1482 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1483 {
1484         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1485
1486         /* if vf-rep dev is NULL, the must belongs to the PF */
1487         return dev ? dev : bp->dev;
1488 }
1489
1490 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1491                                            struct bnxt_cp_ring_info *cpr,
1492                                            u32 *raw_cons,
1493                                            struct rx_tpa_end_cmp *tpa_end,
1494                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1495                                            u8 *event)
1496 {
1497         struct bnxt_napi *bnapi = cpr->bnapi;
1498         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1499         u8 *data_ptr, agg_bufs;
1500         unsigned int len;
1501         struct bnxt_tpa_info *tpa_info;
1502         dma_addr_t mapping;
1503         struct sk_buff *skb;
1504         u16 idx = 0, agg_id;
1505         void *data;
1506         bool gro;
1507
1508         if (unlikely(bnapi->in_reset)) {
1509                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1510
1511                 if (rc < 0)
1512                         return ERR_PTR(-EBUSY);
1513                 return NULL;
1514         }
1515
1516         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1517                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1518                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1519                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1520                 tpa_info = &rxr->rx_tpa[agg_id];
1521                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1522                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1523                                     agg_bufs, tpa_info->agg_count);
1524                         agg_bufs = tpa_info->agg_count;
1525                 }
1526                 tpa_info->agg_count = 0;
1527                 *event |= BNXT_AGG_EVENT;
1528                 bnxt_free_agg_idx(rxr, agg_id);
1529                 idx = agg_id;
1530                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1531         } else {
1532                 agg_id = TPA_END_AGG_ID(tpa_end);
1533                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1534                 tpa_info = &rxr->rx_tpa[agg_id];
1535                 idx = RING_CMP(*raw_cons);
1536                 if (agg_bufs) {
1537                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1538                                 return ERR_PTR(-EBUSY);
1539
1540                         *event |= BNXT_AGG_EVENT;
1541                         idx = NEXT_CMP(idx);
1542                 }
1543                 gro = !!TPA_END_GRO(tpa_end);
1544         }
1545         data = tpa_info->data;
1546         data_ptr = tpa_info->data_ptr;
1547         prefetch(data_ptr);
1548         len = tpa_info->len;
1549         mapping = tpa_info->mapping;
1550
1551         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1552                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1553                 if (agg_bufs > MAX_SKB_FRAGS)
1554                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1555                                     agg_bufs, (int)MAX_SKB_FRAGS);
1556                 return NULL;
1557         }
1558
1559         if (len <= bp->rx_copy_thresh) {
1560                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1561                 if (!skb) {
1562                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1563                         return NULL;
1564                 }
1565         } else {
1566                 u8 *new_data;
1567                 dma_addr_t new_mapping;
1568
1569                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1570                 if (!new_data) {
1571                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1572                         return NULL;
1573                 }
1574
1575                 tpa_info->data = new_data;
1576                 tpa_info->data_ptr = new_data + bp->rx_offset;
1577                 tpa_info->mapping = new_mapping;
1578
1579                 skb = build_skb(data, 0);
1580                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1581                                        bp->rx_buf_use_size, bp->rx_dir,
1582                                        DMA_ATTR_WEAK_ORDERING);
1583
1584                 if (!skb) {
1585                         kfree(data);
1586                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1587                         return NULL;
1588                 }
1589                 skb_reserve(skb, bp->rx_offset);
1590                 skb_put(skb, len);
1591         }
1592
1593         if (agg_bufs) {
1594                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1595                 if (!skb) {
1596                         /* Page reuse already handled by bnxt_rx_pages(). */
1597                         return NULL;
1598                 }
1599         }
1600
1601         skb->protocol =
1602                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1603
1604         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1605                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1606
1607         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1608             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1609                 u16 vlan_proto = tpa_info->metadata >>
1610                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1611                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1612
1613                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1614         }
1615
1616         skb_checksum_none_assert(skb);
1617         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1618                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1619                 skb->csum_level =
1620                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1621         }
1622
1623         if (gro)
1624                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1625
1626         return skb;
1627 }
1628
1629 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1630                          struct rx_agg_cmp *rx_agg)
1631 {
1632         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1633         struct bnxt_tpa_info *tpa_info;
1634
1635         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1636         tpa_info = &rxr->rx_tpa[agg_id];
1637         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1638         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1639 }
1640
1641 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1642                              struct sk_buff *skb)
1643 {
1644         if (skb->dev != bp->dev) {
1645                 /* this packet belongs to a vf-rep */
1646                 bnxt_vf_rep_rx(bp, skb);
1647                 return;
1648         }
1649         skb_record_rx_queue(skb, bnapi->index);
1650         napi_gro_receive(&bnapi->napi, skb);
1651 }
1652
1653 /* returns the following:
1654  * 1       - 1 packet successfully received
1655  * 0       - successful TPA_START, packet not completed yet
1656  * -EBUSY  - completion ring does not have all the agg buffers yet
1657  * -ENOMEM - packet aborted due to out of memory
1658  * -EIO    - packet aborted due to hw error indicated in BD
1659  */
1660 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1661                        u32 *raw_cons, u8 *event)
1662 {
1663         struct bnxt_napi *bnapi = cpr->bnapi;
1664         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1665         struct net_device *dev = bp->dev;
1666         struct rx_cmp *rxcmp;
1667         struct rx_cmp_ext *rxcmp1;
1668         u32 tmp_raw_cons = *raw_cons;
1669         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1670         struct bnxt_sw_rx_bd *rx_buf;
1671         unsigned int len;
1672         u8 *data_ptr, agg_bufs, cmp_type;
1673         dma_addr_t dma_addr;
1674         struct sk_buff *skb;
1675         void *data;
1676         int rc = 0;
1677         u32 misc;
1678
1679         rxcmp = (struct rx_cmp *)
1680                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1681
1682         cmp_type = RX_CMP_TYPE(rxcmp);
1683
1684         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1685                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1686                 goto next_rx_no_prod_no_len;
1687         }
1688
1689         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1690         cp_cons = RING_CMP(tmp_raw_cons);
1691         rxcmp1 = (struct rx_cmp_ext *)
1692                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1693
1694         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1695                 return -EBUSY;
1696
1697         prod = rxr->rx_prod;
1698
1699         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1700                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1701                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1702
1703                 *event |= BNXT_RX_EVENT;
1704                 goto next_rx_no_prod_no_len;
1705
1706         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1707                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1708                                    (struct rx_tpa_end_cmp *)rxcmp,
1709                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1710
1711                 if (IS_ERR(skb))
1712                         return -EBUSY;
1713
1714                 rc = -ENOMEM;
1715                 if (likely(skb)) {
1716                         bnxt_deliver_skb(bp, bnapi, skb);
1717                         rc = 1;
1718                 }
1719                 *event |= BNXT_RX_EVENT;
1720                 goto next_rx_no_prod_no_len;
1721         }
1722
1723         cons = rxcmp->rx_cmp_opaque;
1724         if (unlikely(cons != rxr->rx_next_cons)) {
1725                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1726
1727                 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1728                             cons, rxr->rx_next_cons);
1729                 bnxt_sched_reset(bp, rxr);
1730                 return rc1;
1731         }
1732         rx_buf = &rxr->rx_buf_ring[cons];
1733         data = rx_buf->data;
1734         data_ptr = rx_buf->data_ptr;
1735         prefetch(data_ptr);
1736
1737         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1738         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1739
1740         if (agg_bufs) {
1741                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1742                         return -EBUSY;
1743
1744                 cp_cons = NEXT_CMP(cp_cons);
1745                 *event |= BNXT_AGG_EVENT;
1746         }
1747         *event |= BNXT_RX_EVENT;
1748
1749         rx_buf->data = NULL;
1750         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1751                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1752
1753                 bnxt_reuse_rx_data(rxr, cons, data);
1754                 if (agg_bufs)
1755                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1756                                                false);
1757
1758                 rc = -EIO;
1759                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1760                         netdev_warn(bp->dev, "RX buffer error %x\n", rx_err);
1761                         bnxt_sched_reset(bp, rxr);
1762                 }
1763                 goto next_rx_no_len;
1764         }
1765
1766         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1767         dma_addr = rx_buf->mapping;
1768
1769         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1770                 rc = 1;
1771                 goto next_rx;
1772         }
1773
1774         if (len <= bp->rx_copy_thresh) {
1775                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1776                 bnxt_reuse_rx_data(rxr, cons, data);
1777                 if (!skb) {
1778                         if (agg_bufs)
1779                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1780                                                        agg_bufs, false);
1781                         rc = -ENOMEM;
1782                         goto next_rx;
1783                 }
1784         } else {
1785                 u32 payload;
1786
1787                 if (rx_buf->data_ptr == data_ptr)
1788                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1789                 else
1790                         payload = 0;
1791                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1792                                       payload | len);
1793                 if (!skb) {
1794                         rc = -ENOMEM;
1795                         goto next_rx;
1796                 }
1797         }
1798
1799         if (agg_bufs) {
1800                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1801                 if (!skb) {
1802                         rc = -ENOMEM;
1803                         goto next_rx;
1804                 }
1805         }
1806
1807         if (RX_CMP_HASH_VALID(rxcmp)) {
1808                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1809                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1810
1811                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1812                 if (hash_type != 1 && hash_type != 3)
1813                         type = PKT_HASH_TYPE_L3;
1814                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1815         }
1816
1817         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1818         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1819
1820         if ((rxcmp1->rx_cmp_flags2 &
1821              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1822             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1823                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1824                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1825                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1826
1827                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1828         }
1829
1830         skb_checksum_none_assert(skb);
1831         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1832                 if (dev->features & NETIF_F_RXCSUM) {
1833                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1834                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1835                 }
1836         } else {
1837                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1838                         if (dev->features & NETIF_F_RXCSUM)
1839                                 bnapi->cp_ring.rx_l4_csum_errors++;
1840                 }
1841         }
1842
1843         bnxt_deliver_skb(bp, bnapi, skb);
1844         rc = 1;
1845
1846 next_rx:
1847         cpr->rx_packets += 1;
1848         cpr->rx_bytes += len;
1849
1850 next_rx_no_len:
1851         rxr->rx_prod = NEXT_RX(prod);
1852         rxr->rx_next_cons = NEXT_RX(cons);
1853
1854 next_rx_no_prod_no_len:
1855         *raw_cons = tmp_raw_cons;
1856
1857         return rc;
1858 }
1859
1860 /* In netpoll mode, if we are using a combined completion ring, we need to
1861  * discard the rx packets and recycle the buffers.
1862  */
1863 static int bnxt_force_rx_discard(struct bnxt *bp,
1864                                  struct bnxt_cp_ring_info *cpr,
1865                                  u32 *raw_cons, u8 *event)
1866 {
1867         u32 tmp_raw_cons = *raw_cons;
1868         struct rx_cmp_ext *rxcmp1;
1869         struct rx_cmp *rxcmp;
1870         u16 cp_cons;
1871         u8 cmp_type;
1872
1873         cp_cons = RING_CMP(tmp_raw_cons);
1874         rxcmp = (struct rx_cmp *)
1875                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1876
1877         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1878         cp_cons = RING_CMP(tmp_raw_cons);
1879         rxcmp1 = (struct rx_cmp_ext *)
1880                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1881
1882         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1883                 return -EBUSY;
1884
1885         cmp_type = RX_CMP_TYPE(rxcmp);
1886         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1887                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1888                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1889         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1890                 struct rx_tpa_end_cmp_ext *tpa_end1;
1891
1892                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1893                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1894                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1895         }
1896         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1897 }
1898
1899 #define BNXT_GET_EVENT_PORT(data)       \
1900         ((data) &                       \
1901          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1902
1903 static int bnxt_async_event_process(struct bnxt *bp,
1904                                     struct hwrm_async_event_cmpl *cmpl)
1905 {
1906         u16 event_id = le16_to_cpu(cmpl->event_id);
1907
1908         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1909         switch (event_id) {
1910         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1911                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1912                 struct bnxt_link_info *link_info = &bp->link_info;
1913
1914                 if (BNXT_VF(bp))
1915                         goto async_event_process_exit;
1916
1917                 /* print unsupported speed warning in forced speed mode only */
1918                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1919                     (data1 & 0x20000)) {
1920                         u16 fw_speed = link_info->force_link_speed;
1921                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1922
1923                         if (speed != SPEED_UNKNOWN)
1924                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1925                                             speed);
1926                 }
1927                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1928         }
1929         /* fall through */
1930         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1931                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1932                 break;
1933         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1934                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1935                 break;
1936         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1937                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1938                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1939
1940                 if (BNXT_VF(bp))
1941                         break;
1942
1943                 if (bp->pf.port_id != port_id)
1944                         break;
1945
1946                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1947                 break;
1948         }
1949         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1950                 if (BNXT_PF(bp))
1951                         goto async_event_process_exit;
1952                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1953                 break;
1954         default:
1955                 goto async_event_process_exit;
1956         }
1957         bnxt_queue_sp_work(bp);
1958 async_event_process_exit:
1959         bnxt_ulp_async_events(bp, cmpl);
1960         return 0;
1961 }
1962
1963 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1964 {
1965         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1966         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1967         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1968                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1969
1970         switch (cmpl_type) {
1971         case CMPL_BASE_TYPE_HWRM_DONE:
1972                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1973                 if (seq_id == bp->hwrm_intr_seq_id)
1974                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
1975                 else
1976                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1977                 break;
1978
1979         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1980                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1981
1982                 if ((vf_id < bp->pf.first_vf_id) ||
1983                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1984                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1985                                    vf_id);
1986                         return -EINVAL;
1987                 }
1988
1989                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1990                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1991                 bnxt_queue_sp_work(bp);
1992                 break;
1993
1994         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1995                 bnxt_async_event_process(bp,
1996                                          (struct hwrm_async_event_cmpl *)txcmp);
1997
1998         default:
1999                 break;
2000         }
2001
2002         return 0;
2003 }
2004
2005 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2006 {
2007         struct bnxt_napi *bnapi = dev_instance;
2008         struct bnxt *bp = bnapi->bp;
2009         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2010         u32 cons = RING_CMP(cpr->cp_raw_cons);
2011
2012         cpr->event_ctr++;
2013         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2014         napi_schedule(&bnapi->napi);
2015         return IRQ_HANDLED;
2016 }
2017
2018 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2019 {
2020         u32 raw_cons = cpr->cp_raw_cons;
2021         u16 cons = RING_CMP(raw_cons);
2022         struct tx_cmp *txcmp;
2023
2024         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2025
2026         return TX_CMP_VALID(txcmp, raw_cons);
2027 }
2028
2029 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2030 {
2031         struct bnxt_napi *bnapi = dev_instance;
2032         struct bnxt *bp = bnapi->bp;
2033         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2034         u32 cons = RING_CMP(cpr->cp_raw_cons);
2035         u32 int_status;
2036
2037         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2038
2039         if (!bnxt_has_work(bp, cpr)) {
2040                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2041                 /* return if erroneous interrupt */
2042                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2043                         return IRQ_NONE;
2044         }
2045
2046         /* disable ring IRQ */
2047         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2048
2049         /* Return here if interrupt is shared and is disabled. */
2050         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2051                 return IRQ_HANDLED;
2052
2053         napi_schedule(&bnapi->napi);
2054         return IRQ_HANDLED;
2055 }
2056
2057 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2058                             int budget)
2059 {
2060         struct bnxt_napi *bnapi = cpr->bnapi;
2061         u32 raw_cons = cpr->cp_raw_cons;
2062         u32 cons;
2063         int tx_pkts = 0;
2064         int rx_pkts = 0;
2065         u8 event = 0;
2066         struct tx_cmp *txcmp;
2067
2068         cpr->has_more_work = 0;
2069         while (1) {
2070                 int rc;
2071
2072                 cons = RING_CMP(raw_cons);
2073                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2074
2075                 if (!TX_CMP_VALID(txcmp, raw_cons))
2076                         break;
2077
2078                 /* The valid test of the entry must be done first before
2079                  * reading any further.
2080                  */
2081                 dma_rmb();
2082                 cpr->had_work_done = 1;
2083                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2084                         tx_pkts++;
2085                         /* return full budget so NAPI will complete. */
2086                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2087                                 rx_pkts = budget;
2088                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2089                                 if (budget)
2090                                         cpr->has_more_work = 1;
2091                                 break;
2092                         }
2093                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2094                         if (likely(budget))
2095                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2096                         else
2097                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2098                                                            &event);
2099                         if (likely(rc >= 0))
2100                                 rx_pkts += rc;
2101                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2102                          * the NAPI budget.  Otherwise, we may potentially loop
2103                          * here forever if we consistently cannot allocate
2104                          * buffers.
2105                          */
2106                         else if (rc == -ENOMEM && budget)
2107                                 rx_pkts++;
2108                         else if (rc == -EBUSY)  /* partial completion */
2109                                 break;
2110                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2111                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2112                                     (TX_CMP_TYPE(txcmp) ==
2113                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2114                                     (TX_CMP_TYPE(txcmp) ==
2115                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2116                         bnxt_hwrm_handler(bp, txcmp);
2117                 }
2118                 raw_cons = NEXT_RAW_CMP(raw_cons);
2119
2120                 if (rx_pkts && rx_pkts == budget) {
2121                         cpr->has_more_work = 1;
2122                         break;
2123                 }
2124         }
2125
2126         if (event & BNXT_REDIRECT_EVENT)
2127                 xdp_do_flush_map();
2128
2129         if (event & BNXT_TX_EVENT) {
2130                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2131                 u16 prod = txr->tx_prod;
2132
2133                 /* Sync BD data before updating doorbell */
2134                 wmb();
2135
2136                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2137         }
2138
2139         cpr->cp_raw_cons = raw_cons;
2140         bnapi->tx_pkts += tx_pkts;
2141         bnapi->events |= event;
2142         return rx_pkts;
2143 }
2144
2145 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2146 {
2147         if (bnapi->tx_pkts) {
2148                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2149                 bnapi->tx_pkts = 0;
2150         }
2151
2152         if (bnapi->events & BNXT_RX_EVENT) {
2153                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2154
2155                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2156                 if (bnapi->events & BNXT_AGG_EVENT)
2157                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2158         }
2159         bnapi->events = 0;
2160 }
2161
2162 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2163                           int budget)
2164 {
2165         struct bnxt_napi *bnapi = cpr->bnapi;
2166         int rx_pkts;
2167
2168         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2169
2170         /* ACK completion ring before freeing tx ring and producing new
2171          * buffers in rx/agg rings to prevent overflowing the completion
2172          * ring.
2173          */
2174         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2175
2176         __bnxt_poll_work_done(bp, bnapi);
2177         return rx_pkts;
2178 }
2179
2180 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2181 {
2182         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2183         struct bnxt *bp = bnapi->bp;
2184         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2185         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2186         struct tx_cmp *txcmp;
2187         struct rx_cmp_ext *rxcmp1;
2188         u32 cp_cons, tmp_raw_cons;
2189         u32 raw_cons = cpr->cp_raw_cons;
2190         u32 rx_pkts = 0;
2191         u8 event = 0;
2192
2193         while (1) {
2194                 int rc;
2195
2196                 cp_cons = RING_CMP(raw_cons);
2197                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2198
2199                 if (!TX_CMP_VALID(txcmp, raw_cons))
2200                         break;
2201
2202                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2203                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2204                         cp_cons = RING_CMP(tmp_raw_cons);
2205                         rxcmp1 = (struct rx_cmp_ext *)
2206                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2207
2208                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2209                                 break;
2210
2211                         /* force an error to recycle the buffer */
2212                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2213                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2214
2215                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2216                         if (likely(rc == -EIO) && budget)
2217                                 rx_pkts++;
2218                         else if (rc == -EBUSY)  /* partial completion */
2219                                 break;
2220                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2221                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2222                         bnxt_hwrm_handler(bp, txcmp);
2223                 } else {
2224                         netdev_err(bp->dev,
2225                                    "Invalid completion received on special ring\n");
2226                 }
2227                 raw_cons = NEXT_RAW_CMP(raw_cons);
2228
2229                 if (rx_pkts == budget)
2230                         break;
2231         }
2232
2233         cpr->cp_raw_cons = raw_cons;
2234         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2235         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2236
2237         if (event & BNXT_AGG_EVENT)
2238                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2239
2240         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2241                 napi_complete_done(napi, rx_pkts);
2242                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2243         }
2244         return rx_pkts;
2245 }
2246
2247 static int bnxt_poll(struct napi_struct *napi, int budget)
2248 {
2249         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2250         struct bnxt *bp = bnapi->bp;
2251         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2252         int work_done = 0;
2253
2254         while (1) {
2255                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2256
2257                 if (work_done >= budget) {
2258                         if (!budget)
2259                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2260                         break;
2261                 }
2262
2263                 if (!bnxt_has_work(bp, cpr)) {
2264                         if (napi_complete_done(napi, work_done))
2265                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2266                         break;
2267                 }
2268         }
2269         if (bp->flags & BNXT_FLAG_DIM) {
2270                 struct dim_sample dim_sample = {};
2271
2272                 dim_update_sample(cpr->event_ctr,
2273                                   cpr->rx_packets,
2274                                   cpr->rx_bytes,
2275                                   &dim_sample);
2276                 net_dim(&cpr->dim, dim_sample);
2277         }
2278         return work_done;
2279 }
2280
2281 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2282 {
2283         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2284         int i, work_done = 0;
2285
2286         for (i = 0; i < 2; i++) {
2287                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2288
2289                 if (cpr2) {
2290                         work_done += __bnxt_poll_work(bp, cpr2,
2291                                                       budget - work_done);
2292                         cpr->has_more_work |= cpr2->has_more_work;
2293                 }
2294         }
2295         return work_done;
2296 }
2297
2298 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2299                                  u64 dbr_type, bool all)
2300 {
2301         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2302         int i;
2303
2304         for (i = 0; i < 2; i++) {
2305                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2306                 struct bnxt_db_info *db;
2307
2308                 if (cpr2 && (all || cpr2->had_work_done)) {
2309                         db = &cpr2->cp_db;
2310                         writeq(db->db_key64 | dbr_type |
2311                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2312                         cpr2->had_work_done = 0;
2313                 }
2314         }
2315         __bnxt_poll_work_done(bp, bnapi);
2316 }
2317
2318 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2319 {
2320         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2321         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2322         u32 raw_cons = cpr->cp_raw_cons;
2323         struct bnxt *bp = bnapi->bp;
2324         struct nqe_cn *nqcmp;
2325         int work_done = 0;
2326         u32 cons;
2327
2328         if (cpr->has_more_work) {
2329                 cpr->has_more_work = 0;
2330                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2331                 if (cpr->has_more_work) {
2332                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2333                         return work_done;
2334                 }
2335                 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2336                 if (napi_complete_done(napi, work_done))
2337                         BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2338                 return work_done;
2339         }
2340         while (1) {
2341                 cons = RING_CMP(raw_cons);
2342                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2343
2344                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2345                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2346                                              false);
2347                         cpr->cp_raw_cons = raw_cons;
2348                         if (napi_complete_done(napi, work_done))
2349                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2350                                                   cpr->cp_raw_cons);
2351                         return work_done;
2352                 }
2353
2354                 /* The valid test of the entry must be done first before
2355                  * reading any further.
2356                  */
2357                 dma_rmb();
2358
2359                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2360                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2361                         struct bnxt_cp_ring_info *cpr2;
2362
2363                         cpr2 = cpr->cp_ring_arr[idx];
2364                         work_done += __bnxt_poll_work(bp, cpr2,
2365                                                       budget - work_done);
2366                         cpr->has_more_work = cpr2->has_more_work;
2367                 } else {
2368                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2369                 }
2370                 raw_cons = NEXT_RAW_CMP(raw_cons);
2371                 if (cpr->has_more_work)
2372                         break;
2373         }
2374         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2375         cpr->cp_raw_cons = raw_cons;
2376         return work_done;
2377 }
2378
2379 static void bnxt_free_tx_skbs(struct bnxt *bp)
2380 {
2381         int i, max_idx;
2382         struct pci_dev *pdev = bp->pdev;
2383
2384         if (!bp->tx_ring)
2385                 return;
2386
2387         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2388         for (i = 0; i < bp->tx_nr_rings; i++) {
2389                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2390                 int j;
2391
2392                 for (j = 0; j < max_idx;) {
2393                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2394                         struct sk_buff *skb;
2395                         int k, last;
2396
2397                         if (i < bp->tx_nr_rings_xdp &&
2398                             tx_buf->action == XDP_REDIRECT) {
2399                                 dma_unmap_single(&pdev->dev,
2400                                         dma_unmap_addr(tx_buf, mapping),
2401                                         dma_unmap_len(tx_buf, len),
2402                                         PCI_DMA_TODEVICE);
2403                                 xdp_return_frame(tx_buf->xdpf);
2404                                 tx_buf->action = 0;
2405                                 tx_buf->xdpf = NULL;
2406                                 j++;
2407                                 continue;
2408                         }
2409
2410                         skb = tx_buf->skb;
2411                         if (!skb) {
2412                                 j++;
2413                                 continue;
2414                         }
2415
2416                         tx_buf->skb = NULL;
2417
2418                         if (tx_buf->is_push) {
2419                                 dev_kfree_skb(skb);
2420                                 j += 2;
2421                                 continue;
2422                         }
2423
2424                         dma_unmap_single(&pdev->dev,
2425                                          dma_unmap_addr(tx_buf, mapping),
2426                                          skb_headlen(skb),
2427                                          PCI_DMA_TODEVICE);
2428
2429                         last = tx_buf->nr_frags;
2430                         j += 2;
2431                         for (k = 0; k < last; k++, j++) {
2432                                 int ring_idx = j & bp->tx_ring_mask;
2433                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2434
2435                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2436                                 dma_unmap_page(
2437                                         &pdev->dev,
2438                                         dma_unmap_addr(tx_buf, mapping),
2439                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2440                         }
2441                         dev_kfree_skb(skb);
2442                 }
2443                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2444         }
2445 }
2446
2447 static void bnxt_free_rx_skbs(struct bnxt *bp)
2448 {
2449         int i, max_idx, max_agg_idx;
2450         struct pci_dev *pdev = bp->pdev;
2451
2452         if (!bp->rx_ring)
2453                 return;
2454
2455         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2456         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2457         for (i = 0; i < bp->rx_nr_rings; i++) {
2458                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2459                 struct bnxt_tpa_idx_map *map;
2460                 int j;
2461
2462                 if (rxr->rx_tpa) {
2463                         for (j = 0; j < bp->max_tpa; j++) {
2464                                 struct bnxt_tpa_info *tpa_info =
2465                                                         &rxr->rx_tpa[j];
2466                                 u8 *data = tpa_info->data;
2467
2468                                 if (!data)
2469                                         continue;
2470
2471                                 dma_unmap_single_attrs(&pdev->dev,
2472                                                        tpa_info->mapping,
2473                                                        bp->rx_buf_use_size,
2474                                                        bp->rx_dir,
2475                                                        DMA_ATTR_WEAK_ORDERING);
2476
2477                                 tpa_info->data = NULL;
2478
2479                                 kfree(data);
2480                         }
2481                 }
2482
2483                 for (j = 0; j < max_idx; j++) {
2484                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2485                         dma_addr_t mapping = rx_buf->mapping;
2486                         void *data = rx_buf->data;
2487
2488                         if (!data)
2489                                 continue;
2490
2491                         rx_buf->data = NULL;
2492
2493                         if (BNXT_RX_PAGE_MODE(bp)) {
2494                                 mapping -= bp->rx_dma_offset;
2495                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2496                                                      PAGE_SIZE, bp->rx_dir,
2497                                                      DMA_ATTR_WEAK_ORDERING);
2498                                 page_pool_recycle_direct(rxr->page_pool, data);
2499                         } else {
2500                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2501                                                        bp->rx_buf_use_size,
2502                                                        bp->rx_dir,
2503                                                        DMA_ATTR_WEAK_ORDERING);
2504                                 kfree(data);
2505                         }
2506                 }
2507
2508                 for (j = 0; j < max_agg_idx; j++) {
2509                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2510                                 &rxr->rx_agg_ring[j];
2511                         struct page *page = rx_agg_buf->page;
2512
2513                         if (!page)
2514                                 continue;
2515
2516                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2517                                              BNXT_RX_PAGE_SIZE,
2518                                              PCI_DMA_FROMDEVICE,
2519                                              DMA_ATTR_WEAK_ORDERING);
2520
2521                         rx_agg_buf->page = NULL;
2522                         __clear_bit(j, rxr->rx_agg_bmap);
2523
2524                         __free_page(page);
2525                 }
2526                 if (rxr->rx_page) {
2527                         __free_page(rxr->rx_page);
2528                         rxr->rx_page = NULL;
2529                 }
2530                 map = rxr->rx_tpa_idx_map;
2531                 if (map)
2532                         memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2533         }
2534 }
2535
2536 static void bnxt_free_skbs(struct bnxt *bp)
2537 {
2538         bnxt_free_tx_skbs(bp);
2539         bnxt_free_rx_skbs(bp);
2540 }
2541
2542 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2543 {
2544         struct pci_dev *pdev = bp->pdev;
2545         int i;
2546
2547         for (i = 0; i < rmem->nr_pages; i++) {
2548                 if (!rmem->pg_arr[i])
2549                         continue;
2550
2551                 dma_free_coherent(&pdev->dev, rmem->page_size,
2552                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2553
2554                 rmem->pg_arr[i] = NULL;
2555         }
2556         if (rmem->pg_tbl) {
2557                 size_t pg_tbl_size = rmem->nr_pages * 8;
2558
2559                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2560                         pg_tbl_size = rmem->page_size;
2561                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2562                                   rmem->pg_tbl, rmem->pg_tbl_map);
2563                 rmem->pg_tbl = NULL;
2564         }
2565         if (rmem->vmem_size && *rmem->vmem) {
2566                 vfree(*rmem->vmem);
2567                 *rmem->vmem = NULL;
2568         }
2569 }
2570
2571 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2572 {
2573         struct pci_dev *pdev = bp->pdev;
2574         u64 valid_bit = 0;
2575         int i;
2576
2577         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2578                 valid_bit = PTU_PTE_VALID;
2579         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2580                 size_t pg_tbl_size = rmem->nr_pages * 8;
2581
2582                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2583                         pg_tbl_size = rmem->page_size;
2584                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2585                                                   &rmem->pg_tbl_map,
2586                                                   GFP_KERNEL);
2587                 if (!rmem->pg_tbl)
2588                         return -ENOMEM;
2589         }
2590
2591         for (i = 0; i < rmem->nr_pages; i++) {
2592                 u64 extra_bits = valid_bit;
2593
2594                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2595                                                      rmem->page_size,
2596                                                      &rmem->dma_arr[i],
2597                                                      GFP_KERNEL);
2598                 if (!rmem->pg_arr[i])
2599                         return -ENOMEM;
2600
2601                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2602                         if (i == rmem->nr_pages - 2 &&
2603                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2604                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2605                         else if (i == rmem->nr_pages - 1 &&
2606                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2607                                 extra_bits |= PTU_PTE_LAST;
2608                         rmem->pg_tbl[i] =
2609                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2610                 }
2611         }
2612
2613         if (rmem->vmem_size) {
2614                 *rmem->vmem = vzalloc(rmem->vmem_size);
2615                 if (!(*rmem->vmem))
2616                         return -ENOMEM;
2617         }
2618         return 0;
2619 }
2620
2621 static void bnxt_free_tpa_info(struct bnxt *bp)
2622 {
2623         int i;
2624
2625         for (i = 0; i < bp->rx_nr_rings; i++) {
2626                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2627
2628                 kfree(rxr->rx_tpa_idx_map);
2629                 rxr->rx_tpa_idx_map = NULL;
2630                 if (rxr->rx_tpa) {
2631                         kfree(rxr->rx_tpa[0].agg_arr);
2632                         rxr->rx_tpa[0].agg_arr = NULL;
2633                 }
2634                 kfree(rxr->rx_tpa);
2635                 rxr->rx_tpa = NULL;
2636         }
2637 }
2638
2639 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2640 {
2641         int i, j, total_aggs = 0;
2642
2643         bp->max_tpa = MAX_TPA;
2644         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2645                 if (!bp->max_tpa_v2)
2646                         return 0;
2647                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2648                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2649         }
2650
2651         for (i = 0; i < bp->rx_nr_rings; i++) {
2652                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2653                 struct rx_agg_cmp *agg;
2654
2655                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2656                                       GFP_KERNEL);
2657                 if (!rxr->rx_tpa)
2658                         return -ENOMEM;
2659
2660                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2661                         continue;
2662                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2663                 rxr->rx_tpa[0].agg_arr = agg;
2664                 if (!agg)
2665                         return -ENOMEM;
2666                 for (j = 1; j < bp->max_tpa; j++)
2667                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2668                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2669                                               GFP_KERNEL);
2670                 if (!rxr->rx_tpa_idx_map)
2671                         return -ENOMEM;
2672         }
2673         return 0;
2674 }
2675
2676 static void bnxt_free_rx_rings(struct bnxt *bp)
2677 {
2678         int i;
2679
2680         if (!bp->rx_ring)
2681                 return;
2682
2683         bnxt_free_tpa_info(bp);
2684         for (i = 0; i < bp->rx_nr_rings; i++) {
2685                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2686                 struct bnxt_ring_struct *ring;
2687
2688                 if (rxr->xdp_prog)
2689                         bpf_prog_put(rxr->xdp_prog);
2690
2691                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2692                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2693
2694                 page_pool_destroy(rxr->page_pool);
2695                 rxr->page_pool = NULL;
2696
2697                 kfree(rxr->rx_agg_bmap);
2698                 rxr->rx_agg_bmap = NULL;
2699
2700                 ring = &rxr->rx_ring_struct;
2701                 bnxt_free_ring(bp, &ring->ring_mem);
2702
2703                 ring = &rxr->rx_agg_ring_struct;
2704                 bnxt_free_ring(bp, &ring->ring_mem);
2705         }
2706 }
2707
2708 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2709                                    struct bnxt_rx_ring_info *rxr)
2710 {
2711         struct page_pool_params pp = { 0 };
2712
2713         pp.pool_size = bp->rx_ring_size;
2714         pp.nid = dev_to_node(&bp->pdev->dev);
2715         pp.dev = &bp->pdev->dev;
2716         pp.dma_dir = DMA_BIDIRECTIONAL;
2717
2718         rxr->page_pool = page_pool_create(&pp);
2719         if (IS_ERR(rxr->page_pool)) {
2720                 int err = PTR_ERR(rxr->page_pool);
2721
2722                 rxr->page_pool = NULL;
2723                 return err;
2724         }
2725         return 0;
2726 }
2727
2728 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2729 {
2730         int i, rc = 0, agg_rings = 0;
2731
2732         if (!bp->rx_ring)
2733                 return -ENOMEM;
2734
2735         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2736                 agg_rings = 1;
2737
2738         for (i = 0; i < bp->rx_nr_rings; i++) {
2739                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2740                 struct bnxt_ring_struct *ring;
2741
2742                 ring = &rxr->rx_ring_struct;
2743
2744                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2745                 if (rc)
2746                         return rc;
2747
2748                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2749                 if (rc < 0)
2750                         return rc;
2751
2752                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2753                                                 MEM_TYPE_PAGE_POOL,
2754                                                 rxr->page_pool);
2755                 if (rc) {
2756                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2757                         return rc;
2758                 }
2759
2760                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2761                 if (rc)
2762                         return rc;
2763
2764                 ring->grp_idx = i;
2765                 if (agg_rings) {
2766                         u16 mem_size;
2767
2768                         ring = &rxr->rx_agg_ring_struct;
2769                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2770                         if (rc)
2771                                 return rc;
2772
2773                         ring->grp_idx = i;
2774                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2775                         mem_size = rxr->rx_agg_bmap_size / 8;
2776                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2777                         if (!rxr->rx_agg_bmap)
2778                                 return -ENOMEM;
2779                 }
2780         }
2781         if (bp->flags & BNXT_FLAG_TPA)
2782                 rc = bnxt_alloc_tpa_info(bp);
2783         return rc;
2784 }
2785
2786 static void bnxt_free_tx_rings(struct bnxt *bp)
2787 {
2788         int i;
2789         struct pci_dev *pdev = bp->pdev;
2790
2791         if (!bp->tx_ring)
2792                 return;
2793
2794         for (i = 0; i < bp->tx_nr_rings; i++) {
2795                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2796                 struct bnxt_ring_struct *ring;
2797
2798                 if (txr->tx_push) {
2799                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2800                                           txr->tx_push, txr->tx_push_mapping);
2801                         txr->tx_push = NULL;
2802                 }
2803
2804                 ring = &txr->tx_ring_struct;
2805
2806                 bnxt_free_ring(bp, &ring->ring_mem);
2807         }
2808 }
2809
2810 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2811 {
2812         int i, j, rc;
2813         struct pci_dev *pdev = bp->pdev;
2814
2815         bp->tx_push_size = 0;
2816         if (bp->tx_push_thresh) {
2817                 int push_size;
2818
2819                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2820                                         bp->tx_push_thresh);
2821
2822                 if (push_size > 256) {
2823                         push_size = 0;
2824                         bp->tx_push_thresh = 0;
2825                 }
2826
2827                 bp->tx_push_size = push_size;
2828         }
2829
2830         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2831                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2832                 struct bnxt_ring_struct *ring;
2833                 u8 qidx;
2834
2835                 ring = &txr->tx_ring_struct;
2836
2837                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2838                 if (rc)
2839                         return rc;
2840
2841                 ring->grp_idx = txr->bnapi->index;
2842                 if (bp->tx_push_size) {
2843                         dma_addr_t mapping;
2844
2845                         /* One pre-allocated DMA buffer to backup
2846                          * TX push operation
2847                          */
2848                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2849                                                 bp->tx_push_size,
2850                                                 &txr->tx_push_mapping,
2851                                                 GFP_KERNEL);
2852
2853                         if (!txr->tx_push)
2854                                 return -ENOMEM;
2855
2856                         mapping = txr->tx_push_mapping +
2857                                 sizeof(struct tx_push_bd);
2858                         txr->data_mapping = cpu_to_le64(mapping);
2859                 }
2860                 qidx = bp->tc_to_qidx[j];
2861                 ring->queue_id = bp->q_info[qidx].queue_id;
2862                 if (i < bp->tx_nr_rings_xdp)
2863                         continue;
2864                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2865                         j++;
2866         }
2867         return 0;
2868 }
2869
2870 static void bnxt_free_cp_rings(struct bnxt *bp)
2871 {
2872         int i;
2873
2874         if (!bp->bnapi)
2875                 return;
2876
2877         for (i = 0; i < bp->cp_nr_rings; i++) {
2878                 struct bnxt_napi *bnapi = bp->bnapi[i];
2879                 struct bnxt_cp_ring_info *cpr;
2880                 struct bnxt_ring_struct *ring;
2881                 int j;
2882
2883                 if (!bnapi)
2884                         continue;
2885
2886                 cpr = &bnapi->cp_ring;
2887                 ring = &cpr->cp_ring_struct;
2888
2889                 bnxt_free_ring(bp, &ring->ring_mem);
2890
2891                 for (j = 0; j < 2; j++) {
2892                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2893
2894                         if (cpr2) {
2895                                 ring = &cpr2->cp_ring_struct;
2896                                 bnxt_free_ring(bp, &ring->ring_mem);
2897                                 kfree(cpr2);
2898                                 cpr->cp_ring_arr[j] = NULL;
2899                         }
2900                 }
2901         }
2902 }
2903
2904 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2905 {
2906         struct bnxt_ring_mem_info *rmem;
2907         struct bnxt_ring_struct *ring;
2908         struct bnxt_cp_ring_info *cpr;
2909         int rc;
2910
2911         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
2912         if (!cpr)
2913                 return NULL;
2914
2915         ring = &cpr->cp_ring_struct;
2916         rmem = &ring->ring_mem;
2917         rmem->nr_pages = bp->cp_nr_pages;
2918         rmem->page_size = HW_CMPD_RING_SIZE;
2919         rmem->pg_arr = (void **)cpr->cp_desc_ring;
2920         rmem->dma_arr = cpr->cp_desc_mapping;
2921         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
2922         rc = bnxt_alloc_ring(bp, rmem);
2923         if (rc) {
2924                 bnxt_free_ring(bp, rmem);
2925                 kfree(cpr);
2926                 cpr = NULL;
2927         }
2928         return cpr;
2929 }
2930
2931 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2932 {
2933         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
2934         int i, rc, ulp_base_vec, ulp_msix;
2935
2936         ulp_msix = bnxt_get_ulp_msix_num(bp);
2937         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
2938         for (i = 0; i < bp->cp_nr_rings; i++) {
2939                 struct bnxt_napi *bnapi = bp->bnapi[i];
2940                 struct bnxt_cp_ring_info *cpr;
2941                 struct bnxt_ring_struct *ring;
2942
2943                 if (!bnapi)
2944                         continue;
2945
2946                 cpr = &bnapi->cp_ring;
2947                 cpr->bnapi = bnapi;
2948                 ring = &cpr->cp_ring_struct;
2949
2950                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2951                 if (rc)
2952                         return rc;
2953
2954                 if (ulp_msix && i >= ulp_base_vec)
2955                         ring->map_idx = i + ulp_msix;
2956                 else
2957                         ring->map_idx = i;
2958
2959                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2960                         continue;
2961
2962                 if (i < bp->rx_nr_rings) {
2963                         struct bnxt_cp_ring_info *cpr2 =
2964                                 bnxt_alloc_cp_sub_ring(bp);
2965
2966                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
2967                         if (!cpr2)
2968                                 return -ENOMEM;
2969                         cpr2->bnapi = bnapi;
2970                 }
2971                 if ((sh && i < bp->tx_nr_rings) ||
2972                     (!sh && i >= bp->rx_nr_rings)) {
2973                         struct bnxt_cp_ring_info *cpr2 =
2974                                 bnxt_alloc_cp_sub_ring(bp);
2975
2976                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
2977                         if (!cpr2)
2978                                 return -ENOMEM;
2979                         cpr2->bnapi = bnapi;
2980                 }
2981         }
2982         return 0;
2983 }
2984
2985 static void bnxt_init_ring_struct(struct bnxt *bp)
2986 {
2987         int i;
2988
2989         for (i = 0; i < bp->cp_nr_rings; i++) {
2990                 struct bnxt_napi *bnapi = bp->bnapi[i];
2991                 struct bnxt_ring_mem_info *rmem;
2992                 struct bnxt_cp_ring_info *cpr;
2993                 struct bnxt_rx_ring_info *rxr;
2994                 struct bnxt_tx_ring_info *txr;
2995                 struct bnxt_ring_struct *ring;
2996
2997                 if (!bnapi)
2998                         continue;
2999
3000                 cpr = &bnapi->cp_ring;
3001                 ring = &cpr->cp_ring_struct;
3002                 rmem = &ring->ring_mem;
3003                 rmem->nr_pages = bp->cp_nr_pages;
3004                 rmem->page_size = HW_CMPD_RING_SIZE;
3005                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3006                 rmem->dma_arr = cpr->cp_desc_mapping;
3007                 rmem->vmem_size = 0;
3008
3009                 rxr = bnapi->rx_ring;
3010                 if (!rxr)
3011                         goto skip_rx;
3012
3013                 ring = &rxr->rx_ring_struct;
3014                 rmem = &ring->ring_mem;
3015                 rmem->nr_pages = bp->rx_nr_pages;
3016                 rmem->page_size = HW_RXBD_RING_SIZE;
3017                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3018                 rmem->dma_arr = rxr->rx_desc_mapping;
3019                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3020                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3021
3022                 ring = &rxr->rx_agg_ring_struct;
3023                 rmem = &ring->ring_mem;
3024                 rmem->nr_pages = bp->rx_agg_nr_pages;
3025                 rmem->page_size = HW_RXBD_RING_SIZE;
3026                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3027                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3028                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3029                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3030
3031 skip_rx:
3032                 txr = bnapi->tx_ring;
3033                 if (!txr)
3034                         continue;
3035
3036                 ring = &txr->tx_ring_struct;
3037                 rmem = &ring->ring_mem;
3038                 rmem->nr_pages = bp->tx_nr_pages;
3039                 rmem->page_size = HW_RXBD_RING_SIZE;
3040                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3041                 rmem->dma_arr = txr->tx_desc_mapping;
3042                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3043                 rmem->vmem = (void **)&txr->tx_buf_ring;
3044         }
3045 }
3046
3047 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3048 {
3049         int i;
3050         u32 prod;
3051         struct rx_bd **rx_buf_ring;
3052
3053         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3054         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3055                 int j;
3056                 struct rx_bd *rxbd;
3057
3058                 rxbd = rx_buf_ring[i];
3059                 if (!rxbd)
3060                         continue;
3061
3062                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3063                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3064                         rxbd->rx_bd_opaque = prod;
3065                 }
3066         }
3067 }
3068
3069 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3070 {
3071         struct net_device *dev = bp->dev;
3072         struct bnxt_rx_ring_info *rxr;
3073         struct bnxt_ring_struct *ring;
3074         u32 prod, type;
3075         int i;
3076
3077         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3078                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3079
3080         if (NET_IP_ALIGN == 2)
3081                 type |= RX_BD_FLAGS_SOP;
3082
3083         rxr = &bp->rx_ring[ring_nr];
3084         ring = &rxr->rx_ring_struct;
3085         bnxt_init_rxbd_pages(ring, type);
3086
3087         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3088                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
3089                 if (IS_ERR(rxr->xdp_prog)) {
3090                         int rc = PTR_ERR(rxr->xdp_prog);
3091
3092                         rxr->xdp_prog = NULL;
3093                         return rc;
3094                 }
3095         }
3096         prod = rxr->rx_prod;
3097         for (i = 0; i < bp->rx_ring_size; i++) {
3098                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3099                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3100                                     ring_nr, i, bp->rx_ring_size);
3101                         break;
3102                 }
3103                 prod = NEXT_RX(prod);
3104         }
3105         rxr->rx_prod = prod;
3106         ring->fw_ring_id = INVALID_HW_RING_ID;
3107
3108         ring = &rxr->rx_agg_ring_struct;
3109         ring->fw_ring_id = INVALID_HW_RING_ID;
3110
3111         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3112                 return 0;
3113
3114         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3115                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3116
3117         bnxt_init_rxbd_pages(ring, type);
3118
3119         prod = rxr->rx_agg_prod;
3120         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3121                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3122                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3123                                     ring_nr, i, bp->rx_ring_size);
3124                         break;
3125                 }
3126                 prod = NEXT_RX_AGG(prod);
3127         }
3128         rxr->rx_agg_prod = prod;
3129
3130         if (bp->flags & BNXT_FLAG_TPA) {
3131                 if (rxr->rx_tpa) {
3132                         u8 *data;
3133                         dma_addr_t mapping;
3134
3135                         for (i = 0; i < bp->max_tpa; i++) {
3136                                 data = __bnxt_alloc_rx_data(bp, &mapping,
3137                                                             GFP_KERNEL);
3138                                 if (!data)
3139                                         return -ENOMEM;
3140
3141                                 rxr->rx_tpa[i].data = data;
3142                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3143                                 rxr->rx_tpa[i].mapping = mapping;
3144                         }
3145                 } else {
3146                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3147                         return -ENOMEM;
3148                 }
3149         }
3150
3151         return 0;
3152 }
3153
3154 static void bnxt_init_cp_rings(struct bnxt *bp)
3155 {
3156         int i, j;
3157
3158         for (i = 0; i < bp->cp_nr_rings; i++) {
3159                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3160                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3161
3162                 ring->fw_ring_id = INVALID_HW_RING_ID;
3163                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3164                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3165                 for (j = 0; j < 2; j++) {
3166                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3167
3168                         if (!cpr2)
3169                                 continue;
3170
3171                         ring = &cpr2->cp_ring_struct;
3172                         ring->fw_ring_id = INVALID_HW_RING_ID;
3173                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3174                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3175                 }
3176         }
3177 }
3178
3179 static int bnxt_init_rx_rings(struct bnxt *bp)
3180 {
3181         int i, rc = 0;
3182
3183         if (BNXT_RX_PAGE_MODE(bp)) {
3184                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3185                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3186         } else {
3187                 bp->rx_offset = BNXT_RX_OFFSET;
3188                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3189         }
3190
3191         for (i = 0; i < bp->rx_nr_rings; i++) {
3192                 rc = bnxt_init_one_rx_ring(bp, i);
3193                 if (rc)
3194                         break;
3195         }
3196
3197         return rc;
3198 }
3199
3200 static int bnxt_init_tx_rings(struct bnxt *bp)
3201 {
3202         u16 i;
3203
3204         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3205                                    MAX_SKB_FRAGS + 1);
3206
3207         for (i = 0; i < bp->tx_nr_rings; i++) {
3208                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3209                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3210
3211                 ring->fw_ring_id = INVALID_HW_RING_ID;
3212         }
3213
3214         return 0;
3215 }
3216
3217 static void bnxt_free_ring_grps(struct bnxt *bp)
3218 {
3219         kfree(bp->grp_info);
3220         bp->grp_info = NULL;
3221 }
3222
3223 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3224 {
3225         int i;
3226
3227         if (irq_re_init) {
3228                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3229                                        sizeof(struct bnxt_ring_grp_info),
3230                                        GFP_KERNEL);
3231                 if (!bp->grp_info)
3232                         return -ENOMEM;
3233         }
3234         for (i = 0; i < bp->cp_nr_rings; i++) {
3235                 if (irq_re_init)
3236                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3237                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3238                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3239                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3240                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3241         }
3242         return 0;
3243 }
3244
3245 static void bnxt_free_vnics(struct bnxt *bp)
3246 {
3247         kfree(bp->vnic_info);
3248         bp->vnic_info = NULL;
3249         bp->nr_vnics = 0;
3250 }
3251
3252 static int bnxt_alloc_vnics(struct bnxt *bp)
3253 {
3254         int num_vnics = 1;
3255
3256 #ifdef CONFIG_RFS_ACCEL
3257         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3258                 num_vnics += bp->rx_nr_rings;
3259 #endif
3260
3261         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3262                 num_vnics++;
3263
3264         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3265                                 GFP_KERNEL);
3266         if (!bp->vnic_info)
3267                 return -ENOMEM;
3268
3269         bp->nr_vnics = num_vnics;
3270         return 0;
3271 }
3272
3273 static void bnxt_init_vnics(struct bnxt *bp)
3274 {
3275         int i;
3276
3277         for (i = 0; i < bp->nr_vnics; i++) {
3278                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3279                 int j;
3280
3281                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3282                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3283                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3284
3285                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3286
3287                 if (bp->vnic_info[i].rss_hash_key) {
3288                         if (i == 0)
3289                                 prandom_bytes(vnic->rss_hash_key,
3290                                               HW_HASH_KEY_SIZE);
3291                         else
3292                                 memcpy(vnic->rss_hash_key,
3293                                        bp->vnic_info[0].rss_hash_key,
3294                                        HW_HASH_KEY_SIZE);
3295                 }
3296         }
3297 }
3298
3299 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3300 {
3301         int pages;
3302
3303         pages = ring_size / desc_per_pg;
3304
3305         if (!pages)
3306                 return 1;
3307
3308         pages++;
3309
3310         while (pages & (pages - 1))
3311                 pages++;
3312
3313         return pages;
3314 }
3315
3316 void bnxt_set_tpa_flags(struct bnxt *bp)
3317 {
3318         bp->flags &= ~BNXT_FLAG_TPA;
3319         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3320                 return;
3321         if (bp->dev->features & NETIF_F_LRO)
3322                 bp->flags |= BNXT_FLAG_LRO;
3323         else if (bp->dev->features & NETIF_F_GRO_HW)
3324                 bp->flags |= BNXT_FLAG_GRO;
3325 }
3326
3327 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3328  * be set on entry.
3329  */
3330 void bnxt_set_ring_params(struct bnxt *bp)
3331 {
3332         u32 ring_size, rx_size, rx_space;
3333         u32 agg_factor = 0, agg_ring_size = 0;
3334
3335         /* 8 for CRC and VLAN */
3336         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3337
3338         rx_space = rx_size + NET_SKB_PAD +
3339                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3340
3341         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3342         ring_size = bp->rx_ring_size;
3343         bp->rx_agg_ring_size = 0;
3344         bp->rx_agg_nr_pages = 0;
3345
3346         if (bp->flags & BNXT_FLAG_TPA)
3347                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3348
3349         bp->flags &= ~BNXT_FLAG_JUMBO;
3350         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3351                 u32 jumbo_factor;
3352
3353                 bp->flags |= BNXT_FLAG_JUMBO;
3354                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3355                 if (jumbo_factor > agg_factor)
3356                         agg_factor = jumbo_factor;
3357         }
3358         agg_ring_size = ring_size * agg_factor;
3359
3360         if (agg_ring_size) {
3361                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3362                                                         RX_DESC_CNT);
3363                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3364                         u32 tmp = agg_ring_size;
3365
3366                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3367                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3368                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3369                                     tmp, agg_ring_size);
3370                 }
3371                 bp->rx_agg_ring_size = agg_ring_size;
3372                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3373                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3374                 rx_space = rx_size + NET_SKB_PAD +
3375                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3376         }
3377
3378         bp->rx_buf_use_size = rx_size;
3379         bp->rx_buf_size = rx_space;
3380
3381         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3382         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3383
3384         ring_size = bp->tx_ring_size;
3385         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3386         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3387
3388         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3389         bp->cp_ring_size = ring_size;
3390
3391         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3392         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3393                 bp->cp_nr_pages = MAX_CP_PAGES;
3394                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3395                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3396                             ring_size, bp->cp_ring_size);
3397         }
3398         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3399         bp->cp_ring_mask = bp->cp_bit - 1;
3400 }
3401
3402 /* Changing allocation mode of RX rings.
3403  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3404  */
3405 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3406 {
3407         if (page_mode) {
3408                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3409                         return -EOPNOTSUPP;
3410                 bp->dev->max_mtu =
3411                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3412                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3413                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3414                 bp->rx_dir = DMA_BIDIRECTIONAL;
3415                 bp->rx_skb_func = bnxt_rx_page_skb;
3416                 /* Disable LRO or GRO_HW */
3417                 netdev_update_features(bp->dev);
3418         } else {
3419                 bp->dev->max_mtu = bp->max_mtu;
3420                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3421                 bp->rx_dir = DMA_FROM_DEVICE;
3422                 bp->rx_skb_func = bnxt_rx_skb;
3423         }
3424         return 0;
3425 }
3426
3427 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3428 {
3429         int i;
3430         struct bnxt_vnic_info *vnic;
3431         struct pci_dev *pdev = bp->pdev;
3432
3433         if (!bp->vnic_info)
3434                 return;
3435
3436         for (i = 0; i < bp->nr_vnics; i++) {
3437                 vnic = &bp->vnic_info[i];
3438
3439                 kfree(vnic->fw_grp_ids);
3440                 vnic->fw_grp_ids = NULL;
3441
3442                 kfree(vnic->uc_list);
3443                 vnic->uc_list = NULL;
3444
3445                 if (vnic->mc_list) {
3446                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3447                                           vnic->mc_list, vnic->mc_list_mapping);
3448                         vnic->mc_list = NULL;
3449                 }
3450
3451                 if (vnic->rss_table) {
3452                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
3453                                           vnic->rss_table,
3454                                           vnic->rss_table_dma_addr);
3455                         vnic->rss_table = NULL;
3456                 }
3457
3458                 vnic->rss_hash_key = NULL;
3459                 vnic->flags = 0;
3460         }
3461 }
3462
3463 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3464 {
3465         int i, rc = 0, size;
3466         struct bnxt_vnic_info *vnic;
3467         struct pci_dev *pdev = bp->pdev;
3468         int max_rings;
3469
3470         for (i = 0; i < bp->nr_vnics; i++) {
3471                 vnic = &bp->vnic_info[i];
3472
3473                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3474                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3475
3476                         if (mem_size > 0) {
3477                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3478                                 if (!vnic->uc_list) {
3479                                         rc = -ENOMEM;
3480                                         goto out;
3481                                 }
3482                         }
3483                 }
3484
3485                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3486                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3487                         vnic->mc_list =
3488                                 dma_alloc_coherent(&pdev->dev,
3489                                                    vnic->mc_list_size,
3490                                                    &vnic->mc_list_mapping,
3491                                                    GFP_KERNEL);
3492                         if (!vnic->mc_list) {
3493                                 rc = -ENOMEM;
3494                                 goto out;
3495                         }
3496                 }
3497
3498                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3499                         goto vnic_skip_grps;
3500
3501                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3502                         max_rings = bp->rx_nr_rings;
3503                 else
3504                         max_rings = 1;
3505
3506                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3507                 if (!vnic->fw_grp_ids) {
3508                         rc = -ENOMEM;
3509                         goto out;
3510                 }
3511 vnic_skip_grps:
3512                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3513                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3514                         continue;
3515
3516                 /* Allocate rss table and hash key */
3517                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3518                                                      &vnic->rss_table_dma_addr,
3519                                                      GFP_KERNEL);
3520                 if (!vnic->rss_table) {
3521                         rc = -ENOMEM;
3522                         goto out;
3523                 }
3524
3525                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3526
3527                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3528                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3529         }
3530         return 0;
3531
3532 out:
3533         return rc;
3534 }
3535
3536 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3537 {
3538         struct pci_dev *pdev = bp->pdev;
3539
3540         if (bp->hwrm_cmd_resp_addr) {
3541                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3542                                   bp->hwrm_cmd_resp_dma_addr);
3543                 bp->hwrm_cmd_resp_addr = NULL;
3544         }
3545
3546         if (bp->hwrm_cmd_kong_resp_addr) {
3547                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3548                                   bp->hwrm_cmd_kong_resp_addr,
3549                                   bp->hwrm_cmd_kong_resp_dma_addr);
3550                 bp->hwrm_cmd_kong_resp_addr = NULL;
3551         }
3552 }
3553
3554 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3555 {
3556         struct pci_dev *pdev = bp->pdev;
3557
3558         bp->hwrm_cmd_kong_resp_addr =
3559                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3560                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3561                                    GFP_KERNEL);
3562         if (!bp->hwrm_cmd_kong_resp_addr)
3563                 return -ENOMEM;
3564
3565         return 0;
3566 }
3567
3568 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3569 {
3570         struct pci_dev *pdev = bp->pdev;
3571
3572         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3573                                                    &bp->hwrm_cmd_resp_dma_addr,
3574                                                    GFP_KERNEL);
3575         if (!bp->hwrm_cmd_resp_addr)
3576                 return -ENOMEM;
3577
3578         return 0;
3579 }
3580
3581 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3582 {
3583         if (bp->hwrm_short_cmd_req_addr) {
3584                 struct pci_dev *pdev = bp->pdev;
3585
3586                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3587                                   bp->hwrm_short_cmd_req_addr,
3588                                   bp->hwrm_short_cmd_req_dma_addr);
3589                 bp->hwrm_short_cmd_req_addr = NULL;
3590         }
3591 }
3592
3593 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3594 {
3595         struct pci_dev *pdev = bp->pdev;
3596
3597         bp->hwrm_short_cmd_req_addr =
3598                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3599                                    &bp->hwrm_short_cmd_req_dma_addr,
3600                                    GFP_KERNEL);
3601         if (!bp->hwrm_short_cmd_req_addr)
3602                 return -ENOMEM;
3603
3604         return 0;
3605 }
3606
3607 static void bnxt_free_port_stats(struct bnxt *bp)
3608 {
3609         struct pci_dev *pdev = bp->pdev;
3610
3611         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3612         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3613
3614         if (bp->hw_rx_port_stats) {
3615                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3616                                   bp->hw_rx_port_stats,
3617                                   bp->hw_rx_port_stats_map);
3618                 bp->hw_rx_port_stats = NULL;
3619         }
3620
3621         if (bp->hw_tx_port_stats_ext) {
3622                 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3623                                   bp->hw_tx_port_stats_ext,
3624                                   bp->hw_tx_port_stats_ext_map);
3625                 bp->hw_tx_port_stats_ext = NULL;
3626         }
3627
3628         if (bp->hw_rx_port_stats_ext) {
3629                 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3630                                   bp->hw_rx_port_stats_ext,
3631                                   bp->hw_rx_port_stats_ext_map);
3632                 bp->hw_rx_port_stats_ext = NULL;
3633         }
3634
3635         if (bp->hw_pcie_stats) {
3636                 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3637                                   bp->hw_pcie_stats, bp->hw_pcie_stats_map);
3638                 bp->hw_pcie_stats = NULL;
3639         }
3640 }
3641
3642 static void bnxt_free_ring_stats(struct bnxt *bp)
3643 {
3644         struct pci_dev *pdev = bp->pdev;
3645         int size, i;
3646
3647         if (!bp->bnapi)
3648                 return;
3649
3650         size = bp->hw_ring_stats_size;
3651
3652         for (i = 0; i < bp->cp_nr_rings; i++) {
3653                 struct bnxt_napi *bnapi = bp->bnapi[i];
3654                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3655
3656                 if (cpr->hw_stats) {
3657                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3658                                           cpr->hw_stats_map);
3659                         cpr->hw_stats = NULL;
3660                 }
3661         }
3662 }
3663
3664 static int bnxt_alloc_stats(struct bnxt *bp)
3665 {
3666         u32 size, i;
3667         struct pci_dev *pdev = bp->pdev;
3668
3669         size = bp->hw_ring_stats_size;
3670
3671         for (i = 0; i < bp->cp_nr_rings; i++) {
3672                 struct bnxt_napi *bnapi = bp->bnapi[i];
3673                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3674
3675                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3676                                                    &cpr->hw_stats_map,
3677                                                    GFP_KERNEL);
3678                 if (!cpr->hw_stats)
3679                         return -ENOMEM;
3680
3681                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3682         }
3683
3684         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3685                 return 0;
3686
3687         if (bp->hw_rx_port_stats)
3688                 goto alloc_ext_stats;
3689
3690         bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3691                                  sizeof(struct tx_port_stats) + 1024;
3692
3693         bp->hw_rx_port_stats =
3694                 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3695                                    &bp->hw_rx_port_stats_map,
3696                                    GFP_KERNEL);
3697         if (!bp->hw_rx_port_stats)
3698                 return -ENOMEM;
3699
3700         bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512;
3701         bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3702                                    sizeof(struct rx_port_stats) + 512;
3703         bp->flags |= BNXT_FLAG_PORT_STATS;
3704
3705 alloc_ext_stats:
3706         /* Display extended statistics only if FW supports it */
3707         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3708                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3709                         return 0;
3710
3711         if (bp->hw_rx_port_stats_ext)
3712                 goto alloc_tx_ext_stats;
3713
3714         bp->hw_rx_port_stats_ext =
3715                 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3716                                    &bp->hw_rx_port_stats_ext_map, GFP_KERNEL);
3717         if (!bp->hw_rx_port_stats_ext)
3718                 return 0;
3719
3720 alloc_tx_ext_stats:
3721         if (bp->hw_tx_port_stats_ext)
3722                 goto alloc_pcie_stats;
3723
3724         if (bp->hwrm_spec_code >= 0x10902 ||
3725             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3726                 bp->hw_tx_port_stats_ext =
3727                         dma_alloc_coherent(&pdev->dev,
3728                                            sizeof(struct tx_port_stats_ext),
3729                                            &bp->hw_tx_port_stats_ext_map,
3730                                            GFP_KERNEL);
3731         }
3732         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3733
3734 alloc_pcie_stats:
3735         if (bp->hw_pcie_stats ||
3736             !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED))
3737                 return 0;
3738
3739         bp->hw_pcie_stats =
3740                 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3741                                    &bp->hw_pcie_stats_map, GFP_KERNEL);
3742         if (!bp->hw_pcie_stats)
3743                 return 0;
3744
3745         bp->flags |= BNXT_FLAG_PCIE_STATS;
3746         return 0;
3747 }
3748
3749 static void bnxt_clear_ring_indices(struct bnxt *bp)
3750 {
3751         int i;
3752
3753         if (!bp->bnapi)
3754                 return;
3755
3756         for (i = 0; i < bp->cp_nr_rings; i++) {
3757                 struct bnxt_napi *bnapi = bp->bnapi[i];
3758                 struct bnxt_cp_ring_info *cpr;
3759                 struct bnxt_rx_ring_info *rxr;
3760                 struct bnxt_tx_ring_info *txr;
3761
3762                 if (!bnapi)
3763                         continue;
3764
3765                 cpr = &bnapi->cp_ring;
3766                 cpr->cp_raw_cons = 0;
3767
3768                 txr = bnapi->tx_ring;
3769                 if (txr) {
3770                         txr->tx_prod = 0;
3771                         txr->tx_cons = 0;
3772                 }
3773
3774                 rxr = bnapi->rx_ring;
3775                 if (rxr) {
3776                         rxr->rx_prod = 0;
3777                         rxr->rx_agg_prod = 0;
3778                         rxr->rx_sw_agg_prod = 0;
3779                         rxr->rx_next_cons = 0;
3780                 }
3781         }
3782 }
3783
3784 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3785 {
3786 #ifdef CONFIG_RFS_ACCEL
3787         int i;
3788
3789         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3790          * safe to delete the hash table.
3791          */
3792         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3793                 struct hlist_head *head;
3794                 struct hlist_node *tmp;
3795                 struct bnxt_ntuple_filter *fltr;
3796
3797                 head = &bp->ntp_fltr_hash_tbl[i];
3798                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3799                         hlist_del(&fltr->hash);
3800                         kfree(fltr);
3801                 }
3802         }
3803         if (irq_reinit) {
3804                 kfree(bp->ntp_fltr_bmap);
3805                 bp->ntp_fltr_bmap = NULL;
3806         }
3807         bp->ntp_fltr_count = 0;
3808 #endif
3809 }
3810
3811 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3812 {
3813 #ifdef CONFIG_RFS_ACCEL
3814         int i, rc = 0;
3815
3816         if (!(bp->flags & BNXT_FLAG_RFS))
3817                 return 0;
3818
3819         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3820                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3821
3822         bp->ntp_fltr_count = 0;
3823         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3824                                     sizeof(long),
3825                                     GFP_KERNEL);
3826
3827         if (!bp->ntp_fltr_bmap)
3828                 rc = -ENOMEM;
3829
3830         return rc;
3831 #else
3832         return 0;
3833 #endif
3834 }
3835
3836 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3837 {
3838         bnxt_free_vnic_attributes(bp);
3839         bnxt_free_tx_rings(bp);
3840         bnxt_free_rx_rings(bp);
3841         bnxt_free_cp_rings(bp);
3842         bnxt_free_ntp_fltrs(bp, irq_re_init);
3843         if (irq_re_init) {
3844                 bnxt_free_ring_stats(bp);
3845                 bnxt_free_ring_grps(bp);
3846                 bnxt_free_vnics(bp);
3847                 kfree(bp->tx_ring_map);
3848                 bp->tx_ring_map = NULL;
3849                 kfree(bp->tx_ring);
3850                 bp->tx_ring = NULL;
3851                 kfree(bp->rx_ring);
3852                 bp->rx_ring = NULL;
3853                 kfree(bp->bnapi);
3854                 bp->bnapi = NULL;
3855         } else {
3856                 bnxt_clear_ring_indices(bp);
3857         }
3858 }
3859
3860 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3861 {
3862         int i, j, rc, size, arr_size;
3863         void *bnapi;
3864
3865         if (irq_re_init) {
3866                 /* Allocate bnapi mem pointer array and mem block for
3867                  * all queues
3868                  */
3869                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3870                                 bp->cp_nr_rings);
3871                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3872                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3873                 if (!bnapi)
3874                         return -ENOMEM;
3875
3876                 bp->bnapi = bnapi;
3877                 bnapi += arr_size;
3878                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3879                         bp->bnapi[i] = bnapi;
3880                         bp->bnapi[i]->index = i;
3881                         bp->bnapi[i]->bp = bp;
3882                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3883                                 struct bnxt_cp_ring_info *cpr =
3884                                         &bp->bnapi[i]->cp_ring;
3885
3886                                 cpr->cp_ring_struct.ring_mem.flags =
3887                                         BNXT_RMEM_RING_PTE_FLAG;
3888                         }
3889                 }
3890
3891                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3892                                       sizeof(struct bnxt_rx_ring_info),
3893                                       GFP_KERNEL);
3894                 if (!bp->rx_ring)
3895                         return -ENOMEM;
3896
3897                 for (i = 0; i < bp->rx_nr_rings; i++) {
3898                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3899
3900                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3901                                 rxr->rx_ring_struct.ring_mem.flags =
3902                                         BNXT_RMEM_RING_PTE_FLAG;
3903                                 rxr->rx_agg_ring_struct.ring_mem.flags =
3904                                         BNXT_RMEM_RING_PTE_FLAG;
3905                         }
3906                         rxr->bnapi = bp->bnapi[i];
3907                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3908                 }
3909
3910                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3911                                       sizeof(struct bnxt_tx_ring_info),
3912                                       GFP_KERNEL);
3913                 if (!bp->tx_ring)
3914                         return -ENOMEM;
3915
3916                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3917                                           GFP_KERNEL);
3918
3919                 if (!bp->tx_ring_map)
3920                         return -ENOMEM;
3921
3922                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3923                         j = 0;
3924                 else
3925                         j = bp->rx_nr_rings;
3926
3927                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3928                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3929
3930                         if (bp->flags & BNXT_FLAG_CHIP_P5)
3931                                 txr->tx_ring_struct.ring_mem.flags =
3932                                         BNXT_RMEM_RING_PTE_FLAG;
3933                         txr->bnapi = bp->bnapi[j];
3934                         bp->bnapi[j]->tx_ring = txr;
3935                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3936                         if (i >= bp->tx_nr_rings_xdp) {
3937                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
3938                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3939                         } else {
3940                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3941                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3942                         }
3943                 }
3944
3945                 rc = bnxt_alloc_stats(bp);
3946                 if (rc)
3947                         goto alloc_mem_err;
3948
3949                 rc = bnxt_alloc_ntp_fltrs(bp);
3950                 if (rc)
3951                         goto alloc_mem_err;
3952
3953                 rc = bnxt_alloc_vnics(bp);
3954                 if (rc)
3955                         goto alloc_mem_err;
3956         }
3957
3958         bnxt_init_ring_struct(bp);
3959
3960         rc = bnxt_alloc_rx_rings(bp);
3961         if (rc)
3962                 goto alloc_mem_err;
3963
3964         rc = bnxt_alloc_tx_rings(bp);
3965         if (rc)
3966                 goto alloc_mem_err;
3967
3968         rc = bnxt_alloc_cp_rings(bp);
3969         if (rc)
3970                 goto alloc_mem_err;
3971
3972         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3973                                   BNXT_VNIC_UCAST_FLAG;
3974         rc = bnxt_alloc_vnic_attributes(bp);
3975         if (rc)
3976                 goto alloc_mem_err;
3977         return 0;
3978
3979 alloc_mem_err:
3980         bnxt_free_mem(bp, true);
3981         return rc;
3982 }
3983
3984 static void bnxt_disable_int(struct bnxt *bp)
3985 {
3986         int i;
3987
3988         if (!bp->bnapi)
3989                 return;
3990
3991         for (i = 0; i < bp->cp_nr_rings; i++) {
3992                 struct bnxt_napi *bnapi = bp->bnapi[i];
3993                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3994                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3995
3996                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3997                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
3998         }
3999 }
4000
4001 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4002 {
4003         struct bnxt_napi *bnapi = bp->bnapi[n];
4004         struct bnxt_cp_ring_info *cpr;
4005
4006         cpr = &bnapi->cp_ring;
4007         return cpr->cp_ring_struct.map_idx;
4008 }
4009
4010 static void bnxt_disable_int_sync(struct bnxt *bp)
4011 {
4012         int i;
4013
4014         atomic_inc(&bp->intr_sem);
4015
4016         bnxt_disable_int(bp);
4017         for (i = 0; i < bp->cp_nr_rings; i++) {
4018                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4019
4020                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4021         }
4022 }
4023
4024 static void bnxt_enable_int(struct bnxt *bp)
4025 {
4026         int i;
4027
4028         atomic_set(&bp->intr_sem, 0);
4029         for (i = 0; i < bp->cp_nr_rings; i++) {
4030                 struct bnxt_napi *bnapi = bp->bnapi[i];
4031                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4032
4033                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4034         }
4035 }
4036
4037 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4038                             u16 cmpl_ring, u16 target_id)
4039 {
4040         struct input *req = request;
4041
4042         req->req_type = cpu_to_le16(req_type);
4043         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4044         req->target_id = cpu_to_le16(target_id);
4045         if (bnxt_kong_hwrm_message(bp, req))
4046                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4047         else
4048                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4049 }
4050
4051 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4052                                  int timeout, bool silent)
4053 {
4054         int i, intr_process, rc, tmo_count;
4055         struct input *req = msg;
4056         u32 *data = msg;
4057         __le32 *resp_len;
4058         u8 *valid;
4059         u16 cp_ring_id, len = 0;
4060         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4061         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4062         struct hwrm_short_input short_input = {0};
4063         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4064         u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
4065         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4066         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4067
4068         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4069                 if (msg_len > bp->hwrm_max_ext_req_len ||
4070                     !bp->hwrm_short_cmd_req_addr)
4071                         return -EINVAL;
4072         }
4073
4074         if (bnxt_hwrm_kong_chnl(bp, req)) {
4075                 dst = BNXT_HWRM_CHNL_KONG;
4076                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4077                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4078                 resp = bp->hwrm_cmd_kong_resp_addr;
4079                 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
4080         }
4081
4082         memset(resp, 0, PAGE_SIZE);
4083         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4084         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4085
4086         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4087         /* currently supports only one outstanding message */
4088         if (intr_process)
4089                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4090
4091         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4092             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4093                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4094                 u16 max_msg_len;
4095
4096                 /* Set boundary for maximum extended request length for short
4097                  * cmd format. If passed up from device use the max supported
4098                  * internal req length.
4099                  */
4100                 max_msg_len = bp->hwrm_max_ext_req_len;
4101
4102                 memcpy(short_cmd_req, req, msg_len);
4103                 if (msg_len < max_msg_len)
4104                         memset(short_cmd_req + msg_len, 0,
4105                                max_msg_len - msg_len);
4106
4107                 short_input.req_type = req->req_type;
4108                 short_input.signature =
4109                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4110                 short_input.size = cpu_to_le16(msg_len);
4111                 short_input.req_addr =
4112                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4113
4114                 data = (u32 *)&short_input;
4115                 msg_len = sizeof(short_input);
4116
4117                 /* Sync memory write before updating doorbell */
4118                 wmb();
4119
4120                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4121         }
4122
4123         /* Write request msg to hwrm channel */
4124         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4125
4126         for (i = msg_len; i < max_req_len; i += 4)
4127                 writel(0, bp->bar0 + bar_offset + i);
4128
4129         /* Ring channel doorbell */
4130         writel(1, bp->bar0 + doorbell_offset);
4131
4132         if (!timeout)
4133                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4134         /* convert timeout to usec */
4135         timeout *= 1000;
4136
4137         i = 0;
4138         /* Short timeout for the first few iterations:
4139          * number of loops = number of loops for short timeout +
4140          * number of loops for standard timeout.
4141          */
4142         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4143         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4144         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4145         resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
4146
4147         if (intr_process) {
4148                 u16 seq_id = bp->hwrm_intr_seq_id;
4149
4150                 /* Wait until hwrm response cmpl interrupt is processed */
4151                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4152                        i++ < tmo_count) {
4153                         /* on first few passes, just barely sleep */
4154                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4155                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4156                                              HWRM_SHORT_MAX_TIMEOUT);
4157                         else
4158                                 usleep_range(HWRM_MIN_TIMEOUT,
4159                                              HWRM_MAX_TIMEOUT);
4160                 }
4161
4162                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4163                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4164                                    le16_to_cpu(req->req_type));
4165                         return -1;
4166                 }
4167                 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4168                       HWRM_RESP_LEN_SFT;
4169                 valid = resp_addr + len - 1;
4170         } else {
4171                 int j;
4172
4173                 /* Check if response len is updated */
4174                 for (i = 0; i < tmo_count; i++) {
4175                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4176                               HWRM_RESP_LEN_SFT;
4177                         if (len)
4178                                 break;
4179                         /* on first few passes, just barely sleep */
4180                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4181                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4182                                              HWRM_SHORT_MAX_TIMEOUT);
4183                         else
4184                                 usleep_range(HWRM_MIN_TIMEOUT,
4185                                              HWRM_MAX_TIMEOUT);
4186                 }
4187
4188                 if (i >= tmo_count) {
4189                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4190                                    HWRM_TOTAL_TIMEOUT(i),
4191                                    le16_to_cpu(req->req_type),
4192                                    le16_to_cpu(req->seq_id), len);
4193                         return -1;
4194                 }
4195
4196                 /* Last byte of resp contains valid bit */
4197                 valid = resp_addr + len - 1;
4198                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4199                         /* make sure we read from updated DMA memory */
4200                         dma_rmb();
4201                         if (*valid)
4202                                 break;
4203                         usleep_range(1, 5);
4204                 }
4205
4206                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4207                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4208                                    HWRM_TOTAL_TIMEOUT(i),
4209                                    le16_to_cpu(req->req_type),
4210                                    le16_to_cpu(req->seq_id), len, *valid);
4211                         return -1;
4212                 }
4213         }
4214
4215         /* Zero valid bit for compatibility.  Valid bit in an older spec
4216          * may become a new field in a newer spec.  We must make sure that
4217          * a new field not implemented by old spec will read zero.
4218          */
4219         *valid = 0;
4220         rc = le16_to_cpu(resp->error_code);
4221         if (rc && !silent)
4222                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4223                            le16_to_cpu(resp->req_type),
4224                            le16_to_cpu(resp->seq_id), rc);
4225         return rc;
4226 }
4227
4228 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4229 {
4230         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4231 }
4232
4233 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4234                               int timeout)
4235 {
4236         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4237 }
4238
4239 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4240 {
4241         int rc;
4242
4243         mutex_lock(&bp->hwrm_cmd_lock);
4244         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4245         mutex_unlock(&bp->hwrm_cmd_lock);
4246         return rc;
4247 }
4248
4249 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4250                              int timeout)
4251 {
4252         int rc;
4253
4254         mutex_lock(&bp->hwrm_cmd_lock);
4255         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4256         mutex_unlock(&bp->hwrm_cmd_lock);
4257         return rc;
4258 }
4259
4260 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
4261                                      int bmap_size)
4262 {
4263         struct hwrm_func_drv_rgtr_input req = {0};
4264         DECLARE_BITMAP(async_events_bmap, 256);
4265         u32 *events = (u32 *)async_events_bmap;
4266         int i;
4267
4268         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4269
4270         req.enables =
4271                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4272
4273         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4274         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
4275                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4276
4277         if (bmap && bmap_size) {
4278                 for (i = 0; i < bmap_size; i++) {
4279                         if (test_bit(i, bmap))
4280                                 __set_bit(i, async_events_bmap);
4281                 }
4282         }
4283
4284         for (i = 0; i < 8; i++)
4285                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4286
4287         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4288 }
4289
4290 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
4291 {
4292         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4293         struct hwrm_func_drv_rgtr_input req = {0};
4294         int rc;
4295
4296         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4297
4298         req.enables =
4299                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4300                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
4301
4302         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4303         req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
4304         req.ver_maj_8b = DRV_VER_MAJ;
4305         req.ver_min_8b = DRV_VER_MIN;
4306         req.ver_upd_8b = DRV_VER_UPD;
4307         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4308         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4309         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4310
4311         if (BNXT_PF(bp)) {
4312                 u32 data[8];
4313                 int i;
4314
4315                 memset(data, 0, sizeof(data));
4316                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4317                         u16 cmd = bnxt_vf_req_snif[i];
4318                         unsigned int bit, idx;
4319
4320                         idx = cmd / 32;
4321                         bit = cmd % 32;
4322                         data[idx] |= 1 << bit;
4323                 }
4324
4325                 for (i = 0; i < 8; i++)
4326                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4327
4328                 req.enables |=
4329                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4330         }
4331
4332         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4333                 req.flags |= cpu_to_le32(
4334                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4335
4336         mutex_lock(&bp->hwrm_cmd_lock);
4337         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4338         if (rc)
4339                 rc = -EIO;
4340         else if (resp->flags &
4341                  cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4342                 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4343         mutex_unlock(&bp->hwrm_cmd_lock);
4344         return rc;
4345 }
4346
4347 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4348 {
4349         struct hwrm_func_drv_unrgtr_input req = {0};
4350
4351         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4352         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4353 }
4354
4355 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4356 {
4357         u32 rc = 0;
4358         struct hwrm_tunnel_dst_port_free_input req = {0};
4359
4360         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4361         req.tunnel_type = tunnel_type;
4362
4363         switch (tunnel_type) {
4364         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4365                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4366                 break;
4367         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4368                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4369                 break;
4370         default:
4371                 break;
4372         }
4373
4374         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4375         if (rc)
4376                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4377                            rc);
4378         return rc;
4379 }
4380
4381 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4382                                            u8 tunnel_type)
4383 {
4384         u32 rc = 0;
4385         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4386         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4387
4388         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4389
4390         req.tunnel_type = tunnel_type;
4391         req.tunnel_dst_port_val = port;
4392
4393         mutex_lock(&bp->hwrm_cmd_lock);
4394         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4395         if (rc) {
4396                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4397                            rc);
4398                 goto err_out;
4399         }
4400
4401         switch (tunnel_type) {
4402         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4403                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
4404                 break;
4405         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4406                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
4407                 break;
4408         default:
4409                 break;
4410         }
4411
4412 err_out:
4413         mutex_unlock(&bp->hwrm_cmd_lock);
4414         return rc;
4415 }
4416
4417 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4418 {
4419         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4420         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4421
4422         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4423         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4424
4425         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4426         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4427         req.mask = cpu_to_le32(vnic->rx_mask);
4428         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4429 }
4430
4431 #ifdef CONFIG_RFS_ACCEL
4432 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4433                                             struct bnxt_ntuple_filter *fltr)
4434 {
4435         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4436
4437         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4438         req.ntuple_filter_id = fltr->filter_id;
4439         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4440 }
4441
4442 #define BNXT_NTP_FLTR_FLAGS                                     \
4443         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4444          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4445          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4446          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4447          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4448          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4449          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4450          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4451          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4452          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4453          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4454          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4455          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4456          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4457
4458 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4459                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4460
4461 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4462                                              struct bnxt_ntuple_filter *fltr)
4463 {
4464         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4465         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4466         struct flow_keys *keys = &fltr->fkeys;
4467         struct bnxt_vnic_info *vnic;
4468         u32 dst_ena = 0;
4469         int rc = 0;
4470
4471         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4472         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4473
4474         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX) {
4475                 dst_ena = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
4476                 req.rfs_ring_tbl_idx = cpu_to_le16(fltr->rxq);
4477                 vnic = &bp->vnic_info[0];
4478         } else {
4479                 vnic = &bp->vnic_info[fltr->rxq + 1];
4480         }
4481         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4482         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS | dst_ena);
4483
4484         req.ethertype = htons(ETH_P_IP);
4485         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4486         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4487         req.ip_protocol = keys->basic.ip_proto;
4488
4489         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4490                 int i;
4491
4492                 req.ethertype = htons(ETH_P_IPV6);
4493                 req.ip_addr_type =
4494                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4495                 *(struct in6_addr *)&req.src_ipaddr[0] =
4496                         keys->addrs.v6addrs.src;
4497                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4498                         keys->addrs.v6addrs.dst;
4499                 for (i = 0; i < 4; i++) {
4500                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4501                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4502                 }
4503         } else {
4504                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4505                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4506                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4507                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4508         }
4509         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4510                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4511                 req.tunnel_type =
4512                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4513         }
4514
4515         req.src_port = keys->ports.src;
4516         req.src_port_mask = cpu_to_be16(0xffff);
4517         req.dst_port = keys->ports.dst;
4518         req.dst_port_mask = cpu_to_be16(0xffff);
4519
4520         mutex_lock(&bp->hwrm_cmd_lock);
4521         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4522         if (!rc) {
4523                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4524                 fltr->filter_id = resp->ntuple_filter_id;
4525         }
4526         mutex_unlock(&bp->hwrm_cmd_lock);
4527         return rc;
4528 }
4529 #endif
4530
4531 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4532                                      u8 *mac_addr)
4533 {
4534         u32 rc = 0;
4535         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4536         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4537
4538         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4539         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4540         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4541                 req.flags |=
4542                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4543         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4544         req.enables =
4545                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4546                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4547                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4548         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4549         req.l2_addr_mask[0] = 0xff;
4550         req.l2_addr_mask[1] = 0xff;
4551         req.l2_addr_mask[2] = 0xff;
4552         req.l2_addr_mask[3] = 0xff;
4553         req.l2_addr_mask[4] = 0xff;
4554         req.l2_addr_mask[5] = 0xff;
4555
4556         mutex_lock(&bp->hwrm_cmd_lock);
4557         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4558         if (!rc)
4559                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4560                                                         resp->l2_filter_id;
4561         mutex_unlock(&bp->hwrm_cmd_lock);
4562         return rc;
4563 }
4564
4565 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4566 {
4567         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4568         int rc = 0;
4569
4570         /* Any associated ntuple filters will also be cleared by firmware. */
4571         mutex_lock(&bp->hwrm_cmd_lock);
4572         for (i = 0; i < num_of_vnics; i++) {
4573                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4574
4575                 for (j = 0; j < vnic->uc_filter_count; j++) {
4576                         struct hwrm_cfa_l2_filter_free_input req = {0};
4577
4578                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4579                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4580
4581                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4582
4583                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4584                                                 HWRM_CMD_TIMEOUT);
4585                 }
4586                 vnic->uc_filter_count = 0;
4587         }
4588         mutex_unlock(&bp->hwrm_cmd_lock);
4589
4590         return rc;
4591 }
4592
4593 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4594 {
4595         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4596         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4597         struct hwrm_vnic_tpa_cfg_input req = {0};
4598
4599         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4600                 return 0;
4601
4602         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4603
4604         if (tpa_flags) {
4605                 u16 mss = bp->dev->mtu - 40;
4606                 u32 nsegs, n, segs = 0, flags;
4607
4608                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4609                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4610                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4611                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4612                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4613                 if (tpa_flags & BNXT_FLAG_GRO)
4614                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4615
4616                 req.flags = cpu_to_le32(flags);
4617
4618                 req.enables =
4619                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4620                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4621                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4622
4623                 /* Number of segs are log2 units, and first packet is not
4624                  * included as part of this units.
4625                  */
4626                 if (mss <= BNXT_RX_PAGE_SIZE) {
4627                         n = BNXT_RX_PAGE_SIZE / mss;
4628                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4629                 } else {
4630                         n = mss / BNXT_RX_PAGE_SIZE;
4631                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4632                                 n++;
4633                         nsegs = (MAX_SKB_FRAGS - n) / n;
4634                 }
4635
4636                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4637                         segs = MAX_TPA_SEGS_P5;
4638                         max_aggs = bp->max_tpa;
4639                 } else {
4640                         segs = ilog2(nsegs);
4641                 }
4642                 req.max_agg_segs = cpu_to_le16(segs);
4643                 req.max_aggs = cpu_to_le16(max_aggs);
4644
4645                 req.min_agg_len = cpu_to_le32(512);
4646         }
4647         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4648
4649         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4650 }
4651
4652 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4653 {
4654         struct bnxt_ring_grp_info *grp_info;
4655
4656         grp_info = &bp->grp_info[ring->grp_idx];
4657         return grp_info->cp_fw_ring_id;
4658 }
4659
4660 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4661 {
4662         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4663                 struct bnxt_napi *bnapi = rxr->bnapi;
4664                 struct bnxt_cp_ring_info *cpr;
4665
4666                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4667                 return cpr->cp_ring_struct.fw_ring_id;
4668         } else {
4669                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4670         }
4671 }
4672
4673 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4674 {
4675         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4676                 struct bnxt_napi *bnapi = txr->bnapi;
4677                 struct bnxt_cp_ring_info *cpr;
4678
4679                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4680                 return cpr->cp_ring_struct.fw_ring_id;
4681         } else {
4682                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4683         }
4684 }
4685
4686 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4687 {
4688         u32 i, j, max_rings;
4689         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4690         struct hwrm_vnic_rss_cfg_input req = {0};
4691
4692         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4693             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
4694                 return 0;
4695
4696         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4697         if (set_rss) {
4698                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4699                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4700                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4701                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4702                                 max_rings = bp->rx_nr_rings - 1;
4703                         else
4704                                 max_rings = bp->rx_nr_rings;
4705                 } else {
4706                         max_rings = 1;
4707                 }
4708
4709                 /* Fill the RSS indirection table with ring group ids */
4710                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4711                         if (j == max_rings)
4712                                 j = 0;
4713                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4714                 }
4715
4716                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4717                 req.hash_key_tbl_addr =
4718                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
4719         }
4720         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4721         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4722 }
4723
4724 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4725 {
4726         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4727         u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4728         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4729         struct hwrm_vnic_rss_cfg_input req = {0};
4730
4731         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4732         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4733         if (!set_rss) {
4734                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4735                 return 0;
4736         }
4737         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4738         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4739         req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4740         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4741         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4742         for (i = 0, k = 0; i < nr_ctxs; i++) {
4743                 __le16 *ring_tbl = vnic->rss_table;
4744                 int rc;
4745
4746                 req.ring_table_pair_index = i;
4747                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4748                 for (j = 0; j < 64; j++) {
4749                         u16 ring_id;
4750
4751                         ring_id = rxr->rx_ring_struct.fw_ring_id;
4752                         *ring_tbl++ = cpu_to_le16(ring_id);
4753                         ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4754                         *ring_tbl++ = cpu_to_le16(ring_id);
4755                         rxr++;
4756                         k++;
4757                         if (k == max_rings) {
4758                                 k = 0;
4759                                 rxr = &bp->rx_ring[0];
4760                         }
4761                 }
4762                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4763                 if (rc)
4764                         return -EIO;
4765         }
4766         return 0;
4767 }
4768
4769 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4770 {
4771         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4772         struct hwrm_vnic_plcmodes_cfg_input req = {0};
4773
4774         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4775         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4776                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4777                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4778         req.enables =
4779                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4780                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4781         /* thresholds not implemented in firmware yet */
4782         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4783         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4784         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4785         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4786 }
4787
4788 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4789                                         u16 ctx_idx)
4790 {
4791         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4792
4793         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4794         req.rss_cos_lb_ctx_id =
4795                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4796
4797         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4798         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4799 }
4800
4801 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4802 {
4803         int i, j;
4804
4805         for (i = 0; i < bp->nr_vnics; i++) {
4806                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4807
4808                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4809                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4810                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4811                 }
4812         }
4813         bp->rsscos_nr_ctxs = 0;
4814 }
4815
4816 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4817 {
4818         int rc;
4819         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4820         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4821                                                 bp->hwrm_cmd_resp_addr;
4822
4823         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4824                                -1);
4825
4826         mutex_lock(&bp->hwrm_cmd_lock);
4827         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4828         if (!rc)
4829                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4830                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
4831         mutex_unlock(&bp->hwrm_cmd_lock);
4832
4833         return rc;
4834 }
4835
4836 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4837 {
4838         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4839                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4840         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4841 }
4842
4843 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
4844 {
4845         unsigned int ring = 0, grp_idx;
4846         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4847         struct hwrm_vnic_cfg_input req = {0};
4848         u16 def_vlan = 0;
4849
4850         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
4851
4852         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4853                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4854
4855                 req.default_rx_ring_id =
4856                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
4857                 req.default_cmpl_ring_id =
4858                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
4859                 req.enables =
4860                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
4861                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
4862                 goto vnic_mru;
4863         }
4864         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
4865         /* Only RSS support for now TBD: COS & LB */
4866         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4867                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4868                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4869                                            VNIC_CFG_REQ_ENABLES_MRU);
4870         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4871                 req.rss_rule =
4872                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4873                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4874                                            VNIC_CFG_REQ_ENABLES_MRU);
4875                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
4876         } else {
4877                 req.rss_rule = cpu_to_le16(0xffff);
4878         }
4879
4880         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4881             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
4882                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4883                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4884         } else {
4885                 req.cos_rule = cpu_to_le16(0xffff);
4886         }
4887
4888         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4889                 ring = 0;
4890         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
4891                 ring = vnic_id - 1;
4892         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4893                 ring = bp->rx_nr_rings - 1;
4894
4895         grp_idx = bp->rx_ring[ring].bnapi->index;
4896         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
4897         req.lb_rule = cpu_to_le16(0xffff);
4898 vnic_mru:
4899         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4900                               VLAN_HLEN);
4901
4902         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4903 #ifdef CONFIG_BNXT_SRIOV
4904         if (BNXT_VF(bp))
4905                 def_vlan = bp->vf.vlan;
4906 #endif
4907         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4908                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4909         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4910                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
4911
4912         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4913 }
4914
4915 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4916 {
4917         u32 rc = 0;
4918
4919         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4920                 struct hwrm_vnic_free_input req = {0};
4921
4922                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4923                 req.vnic_id =
4924                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4925
4926                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4927                 if (rc)
4928                         return rc;
4929                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4930         }
4931         return rc;
4932 }
4933
4934 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4935 {
4936         u16 i;
4937
4938         for (i = 0; i < bp->nr_vnics; i++)
4939                 bnxt_hwrm_vnic_free_one(bp, i);
4940 }
4941
4942 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4943                                 unsigned int start_rx_ring_idx,
4944                                 unsigned int nr_rings)
4945 {
4946         int rc = 0;
4947         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4948         struct hwrm_vnic_alloc_input req = {0};
4949         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4950         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4951
4952         if (bp->flags & BNXT_FLAG_CHIP_P5)
4953                 goto vnic_no_ring_grps;
4954
4955         /* map ring groups to this vnic */
4956         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4957                 grp_idx = bp->rx_ring[i].bnapi->index;
4958                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4959                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4960                                    j, nr_rings);
4961                         break;
4962                 }
4963                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
4964         }
4965
4966 vnic_no_ring_grps:
4967         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
4968                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
4969         if (vnic_id == 0)
4970                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4971
4972         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4973
4974         mutex_lock(&bp->hwrm_cmd_lock);
4975         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4976         if (!rc)
4977                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
4978         mutex_unlock(&bp->hwrm_cmd_lock);
4979         return rc;
4980 }
4981
4982 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4983 {
4984         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4985         struct hwrm_vnic_qcaps_input req = {0};
4986         int rc;
4987
4988         if (bp->hwrm_spec_code < 0x10600)
4989                 return 0;
4990
4991         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4992         mutex_lock(&bp->hwrm_cmd_lock);
4993         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4994         if (!rc) {
4995                 u32 flags = le32_to_cpu(resp->flags);
4996
4997                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
4998                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4999                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5000                 if (flags &
5001                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5002                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5003                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5004                 if (bp->max_tpa_v2)
5005                         bp->hw_ring_stats_size =
5006                                 sizeof(struct ctx_hw_stats_ext);
5007                 else
5008                         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5009         }
5010         mutex_unlock(&bp->hwrm_cmd_lock);
5011         return rc;
5012 }
5013
5014 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5015 {
5016         u16 i;
5017         u32 rc = 0;
5018
5019         if (bp->flags & BNXT_FLAG_CHIP_P5)
5020                 return 0;
5021
5022         mutex_lock(&bp->hwrm_cmd_lock);
5023         for (i = 0; i < bp->rx_nr_rings; i++) {
5024                 struct hwrm_ring_grp_alloc_input req = {0};
5025                 struct hwrm_ring_grp_alloc_output *resp =
5026                                         bp->hwrm_cmd_resp_addr;
5027                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5028
5029                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5030
5031                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5032                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5033                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5034                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5035
5036                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5037                                         HWRM_CMD_TIMEOUT);
5038                 if (rc)
5039                         break;
5040
5041                 bp->grp_info[grp_idx].fw_grp_id =
5042                         le32_to_cpu(resp->ring_group_id);
5043         }
5044         mutex_unlock(&bp->hwrm_cmd_lock);
5045         return rc;
5046 }
5047
5048 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5049 {
5050         u16 i;
5051         u32 rc = 0;
5052         struct hwrm_ring_grp_free_input req = {0};
5053
5054         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5055                 return 0;
5056
5057         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5058
5059         mutex_lock(&bp->hwrm_cmd_lock);
5060         for (i = 0; i < bp->cp_nr_rings; i++) {
5061                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5062                         continue;
5063                 req.ring_group_id =
5064                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5065
5066                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5067                                         HWRM_CMD_TIMEOUT);
5068                 if (rc)
5069                         break;
5070                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5071         }
5072         mutex_unlock(&bp->hwrm_cmd_lock);
5073         return rc;
5074 }
5075
5076 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5077                                     struct bnxt_ring_struct *ring,
5078                                     u32 ring_type, u32 map_index)
5079 {
5080         int rc = 0, err = 0;
5081         struct hwrm_ring_alloc_input req = {0};
5082         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5083         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5084         struct bnxt_ring_grp_info *grp_info;
5085         u16 ring_id;
5086
5087         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5088
5089         req.enables = 0;
5090         if (rmem->nr_pages > 1) {
5091                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5092                 /* Page size is in log2 units */
5093                 req.page_size = BNXT_PAGE_SHIFT;
5094                 req.page_tbl_depth = 1;
5095         } else {
5096                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5097         }
5098         req.fbo = 0;
5099         /* Association of ring index with doorbell index and MSIX number */
5100         req.logical_id = cpu_to_le16(map_index);
5101
5102         switch (ring_type) {
5103         case HWRM_RING_ALLOC_TX: {
5104                 struct bnxt_tx_ring_info *txr;
5105
5106                 txr = container_of(ring, struct bnxt_tx_ring_info,
5107                                    tx_ring_struct);
5108                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5109                 /* Association of transmit ring with completion ring */
5110                 grp_info = &bp->grp_info[ring->grp_idx];
5111                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5112                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5113                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5114                 req.queue_id = cpu_to_le16(ring->queue_id);
5115                 break;
5116         }
5117         case HWRM_RING_ALLOC_RX:
5118                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5119                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5120                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5121                         u16 flags = 0;
5122
5123                         /* Association of rx ring with stats context */
5124                         grp_info = &bp->grp_info[ring->grp_idx];
5125                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5126                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5127                         req.enables |= cpu_to_le32(
5128                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5129                         if (NET_IP_ALIGN == 2)
5130                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5131                         req.flags = cpu_to_le16(flags);
5132                 }
5133                 break;
5134         case HWRM_RING_ALLOC_AGG:
5135                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5136                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5137                         /* Association of agg ring with rx ring */
5138                         grp_info = &bp->grp_info[ring->grp_idx];
5139                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5140                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5141                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5142                         req.enables |= cpu_to_le32(
5143                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5144                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5145                 } else {
5146                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5147                 }
5148                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5149                 break;
5150         case HWRM_RING_ALLOC_CMPL:
5151                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5152                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5153                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5154                         /* Association of cp ring with nq */
5155                         grp_info = &bp->grp_info[map_index];
5156                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5157                         req.cq_handle = cpu_to_le64(ring->handle);
5158                         req.enables |= cpu_to_le32(
5159                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5160                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5161                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5162                 }
5163                 break;
5164         case HWRM_RING_ALLOC_NQ:
5165                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5166                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5167                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5168                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5169                 break;
5170         default:
5171                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5172                            ring_type);
5173                 return -1;
5174         }
5175
5176         mutex_lock(&bp->hwrm_cmd_lock);
5177         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5178         err = le16_to_cpu(resp->error_code);
5179         ring_id = le16_to_cpu(resp->ring_id);
5180         mutex_unlock(&bp->hwrm_cmd_lock);
5181
5182         if (rc || err) {
5183                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5184                            ring_type, rc, err);
5185                 return -EIO;
5186         }
5187         ring->fw_ring_id = ring_id;
5188         return rc;
5189 }
5190
5191 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5192 {
5193         int rc;
5194
5195         if (BNXT_PF(bp)) {
5196                 struct hwrm_func_cfg_input req = {0};
5197
5198                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5199                 req.fid = cpu_to_le16(0xffff);
5200                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5201                 req.async_event_cr = cpu_to_le16(idx);
5202                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5203         } else {
5204                 struct hwrm_func_vf_cfg_input req = {0};
5205
5206                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5207                 req.enables =
5208                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5209                 req.async_event_cr = cpu_to_le16(idx);
5210                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5211         }
5212         return rc;
5213 }
5214
5215 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5216                         u32 map_idx, u32 xid)
5217 {
5218         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5219                 if (BNXT_PF(bp))
5220                         db->doorbell = bp->bar1 + 0x10000;
5221                 else
5222                         db->doorbell = bp->bar1 + 0x4000;
5223                 switch (ring_type) {
5224                 case HWRM_RING_ALLOC_TX:
5225                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5226                         break;
5227                 case HWRM_RING_ALLOC_RX:
5228                 case HWRM_RING_ALLOC_AGG:
5229                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5230                         break;
5231                 case HWRM_RING_ALLOC_CMPL:
5232                         db->db_key64 = DBR_PATH_L2;
5233                         break;
5234                 case HWRM_RING_ALLOC_NQ:
5235                         db->db_key64 = DBR_PATH_L2;
5236                         break;
5237                 }
5238                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5239         } else {
5240                 db->doorbell = bp->bar1 + map_idx * 0x80;
5241                 switch (ring_type) {
5242                 case HWRM_RING_ALLOC_TX:
5243                         db->db_key32 = DB_KEY_TX;
5244                         break;
5245                 case HWRM_RING_ALLOC_RX:
5246                 case HWRM_RING_ALLOC_AGG:
5247                         db->db_key32 = DB_KEY_RX;
5248                         break;
5249                 case HWRM_RING_ALLOC_CMPL:
5250                         db->db_key32 = DB_KEY_CP;
5251                         break;
5252                 }
5253         }
5254 }
5255
5256 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5257 {
5258         int i, rc = 0;
5259         u32 type;
5260
5261         if (bp->flags & BNXT_FLAG_CHIP_P5)
5262                 type = HWRM_RING_ALLOC_NQ;
5263         else
5264                 type = HWRM_RING_ALLOC_CMPL;
5265         for (i = 0; i < bp->cp_nr_rings; i++) {
5266                 struct bnxt_napi *bnapi = bp->bnapi[i];
5267                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5268                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5269                 u32 map_idx = ring->map_idx;
5270                 unsigned int vector;
5271
5272                 vector = bp->irq_tbl[map_idx].vector;
5273                 disable_irq_nosync(vector);
5274                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5275                 if (rc) {
5276                         enable_irq(vector);
5277                         goto err_out;
5278                 }
5279                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5280                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5281                 enable_irq(vector);
5282                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5283
5284                 if (!i) {
5285                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5286                         if (rc)
5287                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5288                 }
5289         }
5290
5291         type = HWRM_RING_ALLOC_TX;
5292         for (i = 0; i < bp->tx_nr_rings; i++) {
5293                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5294                 struct bnxt_ring_struct *ring;
5295                 u32 map_idx;
5296
5297                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5298                         struct bnxt_napi *bnapi = txr->bnapi;
5299                         struct bnxt_cp_ring_info *cpr, *cpr2;
5300                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5301
5302                         cpr = &bnapi->cp_ring;
5303                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5304                         ring = &cpr2->cp_ring_struct;
5305                         ring->handle = BNXT_TX_HDL;
5306                         map_idx = bnapi->index;
5307                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5308                         if (rc)
5309                                 goto err_out;
5310                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5311                                     ring->fw_ring_id);
5312                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5313                 }
5314                 ring = &txr->tx_ring_struct;
5315                 map_idx = i;
5316                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5317                 if (rc)
5318                         goto err_out;
5319                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5320         }
5321
5322         type = HWRM_RING_ALLOC_RX;
5323         for (i = 0; i < bp->rx_nr_rings; i++) {
5324                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5325                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5326                 struct bnxt_napi *bnapi = rxr->bnapi;
5327                 u32 map_idx = bnapi->index;
5328
5329                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5330                 if (rc)
5331                         goto err_out;
5332                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5333                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5334                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5335                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5336                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5337                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5338                         struct bnxt_cp_ring_info *cpr2;
5339
5340                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5341                         ring = &cpr2->cp_ring_struct;
5342                         ring->handle = BNXT_RX_HDL;
5343                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5344                         if (rc)
5345                                 goto err_out;
5346                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5347                                     ring->fw_ring_id);
5348                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5349                 }
5350         }
5351
5352         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5353                 type = HWRM_RING_ALLOC_AGG;
5354                 for (i = 0; i < bp->rx_nr_rings; i++) {
5355                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5356                         struct bnxt_ring_struct *ring =
5357                                                 &rxr->rx_agg_ring_struct;
5358                         u32 grp_idx = ring->grp_idx;
5359                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5360
5361                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5362                         if (rc)
5363                                 goto err_out;
5364
5365                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5366                                     ring->fw_ring_id);
5367                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5368                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5369                 }
5370         }
5371 err_out:
5372         return rc;
5373 }
5374
5375 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5376                                    struct bnxt_ring_struct *ring,
5377                                    u32 ring_type, int cmpl_ring_id)
5378 {
5379         int rc;
5380         struct hwrm_ring_free_input req = {0};
5381         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5382         u16 error_code;
5383
5384         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5385         req.ring_type = ring_type;
5386         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5387
5388         mutex_lock(&bp->hwrm_cmd_lock);
5389         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5390         error_code = le16_to_cpu(resp->error_code);
5391         mutex_unlock(&bp->hwrm_cmd_lock);
5392
5393         if (rc || error_code) {
5394                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5395                            ring_type, rc, error_code);
5396                 return -EIO;
5397         }
5398         return 0;
5399 }
5400
5401 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5402 {
5403         u32 type;
5404         int i;
5405
5406         if (!bp->bnapi)
5407                 return;
5408
5409         for (i = 0; i < bp->tx_nr_rings; i++) {
5410                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5411                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5412
5413                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5414                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5415
5416                         hwrm_ring_free_send_msg(bp, ring,
5417                                                 RING_FREE_REQ_RING_TYPE_TX,
5418                                                 close_path ? cmpl_ring_id :
5419                                                 INVALID_HW_RING_ID);
5420                         ring->fw_ring_id = INVALID_HW_RING_ID;
5421                 }
5422         }
5423
5424         for (i = 0; i < bp->rx_nr_rings; i++) {
5425                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5426                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5427                 u32 grp_idx = rxr->bnapi->index;
5428
5429                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5430                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5431
5432                         hwrm_ring_free_send_msg(bp, ring,
5433                                                 RING_FREE_REQ_RING_TYPE_RX,
5434                                                 close_path ? cmpl_ring_id :
5435                                                 INVALID_HW_RING_ID);
5436                         ring->fw_ring_id = INVALID_HW_RING_ID;
5437                         bp->grp_info[grp_idx].rx_fw_ring_id =
5438                                 INVALID_HW_RING_ID;
5439                 }
5440         }
5441
5442         if (bp->flags & BNXT_FLAG_CHIP_P5)
5443                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5444         else
5445                 type = RING_FREE_REQ_RING_TYPE_RX;
5446         for (i = 0; i < bp->rx_nr_rings; i++) {
5447                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5448                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5449                 u32 grp_idx = rxr->bnapi->index;
5450
5451                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5452                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5453
5454                         hwrm_ring_free_send_msg(bp, ring, type,
5455                                                 close_path ? cmpl_ring_id :
5456                                                 INVALID_HW_RING_ID);
5457                         ring->fw_ring_id = INVALID_HW_RING_ID;
5458                         bp->grp_info[grp_idx].agg_fw_ring_id =
5459                                 INVALID_HW_RING_ID;
5460                 }
5461         }
5462
5463         /* The completion rings are about to be freed.  After that the
5464          * IRQ doorbell will not work anymore.  So we need to disable
5465          * IRQ here.
5466          */
5467         bnxt_disable_int_sync(bp);
5468
5469         if (bp->flags & BNXT_FLAG_CHIP_P5)
5470                 type = RING_FREE_REQ_RING_TYPE_NQ;
5471         else
5472                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5473         for (i = 0; i < bp->cp_nr_rings; i++) {
5474                 struct bnxt_napi *bnapi = bp->bnapi[i];
5475                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5476                 struct bnxt_ring_struct *ring;
5477                 int j;
5478
5479                 for (j = 0; j < 2; j++) {
5480                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5481
5482                         if (cpr2) {
5483                                 ring = &cpr2->cp_ring_struct;
5484                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5485                                         continue;
5486                                 hwrm_ring_free_send_msg(bp, ring,
5487                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5488                                         INVALID_HW_RING_ID);
5489                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5490                         }
5491                 }
5492                 ring = &cpr->cp_ring_struct;
5493                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5494                         hwrm_ring_free_send_msg(bp, ring, type,
5495                                                 INVALID_HW_RING_ID);
5496                         ring->fw_ring_id = INVALID_HW_RING_ID;
5497                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5498                 }
5499         }
5500 }
5501
5502 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5503                            bool shared);
5504
5505 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5506 {
5507         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5508         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5509         struct hwrm_func_qcfg_input req = {0};
5510         int rc;
5511
5512         if (bp->hwrm_spec_code < 0x10601)
5513                 return 0;
5514
5515         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5516         req.fid = cpu_to_le16(0xffff);
5517         mutex_lock(&bp->hwrm_cmd_lock);
5518         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5519         if (rc) {
5520                 mutex_unlock(&bp->hwrm_cmd_lock);
5521                 return -EIO;
5522         }
5523
5524         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5525         if (BNXT_NEW_RM(bp)) {
5526                 u16 cp, stats;
5527
5528                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5529                 hw_resc->resv_hw_ring_grps =
5530                         le32_to_cpu(resp->alloc_hw_ring_grps);
5531                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5532                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5533                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5534                 hw_resc->resv_irqs = cp;
5535                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5536                         int rx = hw_resc->resv_rx_rings;
5537                         int tx = hw_resc->resv_tx_rings;
5538
5539                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5540                                 rx >>= 1;
5541                         if (cp < (rx + tx)) {
5542                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5543                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5544                                         rx <<= 1;
5545                                 hw_resc->resv_rx_rings = rx;
5546                                 hw_resc->resv_tx_rings = tx;
5547                         }
5548                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5549                         hw_resc->resv_hw_ring_grps = rx;
5550                 }
5551                 hw_resc->resv_cp_rings = cp;
5552                 hw_resc->resv_stat_ctxs = stats;
5553         }
5554         mutex_unlock(&bp->hwrm_cmd_lock);
5555         return 0;
5556 }
5557
5558 /* Caller must hold bp->hwrm_cmd_lock */
5559 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5560 {
5561         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5562         struct hwrm_func_qcfg_input req = {0};
5563         int rc;
5564
5565         if (bp->hwrm_spec_code < 0x10601)
5566                 return 0;
5567
5568         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5569         req.fid = cpu_to_le16(fid);
5570         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5571         if (!rc)
5572                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5573
5574         return rc;
5575 }
5576
5577 static bool bnxt_rfs_supported(struct bnxt *bp);
5578
5579 static void
5580 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5581                              int tx_rings, int rx_rings, int ring_grps,
5582                              int cp_rings, int stats, int vnics)
5583 {
5584         u32 enables = 0;
5585
5586         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5587         req->fid = cpu_to_le16(0xffff);
5588         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5589         req->num_tx_rings = cpu_to_le16(tx_rings);
5590         if (BNXT_NEW_RM(bp)) {
5591                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5592                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5593                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5594                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5595                         enables |= tx_rings + ring_grps ?
5596                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5597                         enables |= rx_rings ?
5598                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5599                 } else {
5600                         enables |= cp_rings ?
5601                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5602                         enables |= ring_grps ?
5603                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5604                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5605                 }
5606                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5607
5608                 req->num_rx_rings = cpu_to_le16(rx_rings);
5609                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5610                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5611                         req->num_msix = cpu_to_le16(cp_rings);
5612                         req->num_rsscos_ctxs =
5613                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5614                 } else {
5615                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
5616                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5617                         req->num_rsscos_ctxs = cpu_to_le16(1);
5618                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5619                             bnxt_rfs_supported(bp))
5620                                 req->num_rsscos_ctxs =
5621                                         cpu_to_le16(ring_grps + 1);
5622                 }
5623                 req->num_stat_ctxs = cpu_to_le16(stats);
5624                 req->num_vnics = cpu_to_le16(vnics);
5625         }
5626         req->enables = cpu_to_le32(enables);
5627 }
5628
5629 static void
5630 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5631                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
5632                              int rx_rings, int ring_grps, int cp_rings,
5633                              int stats, int vnics)
5634 {
5635         u32 enables = 0;
5636
5637         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5638         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5639         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5640                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5641         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5642         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5643                 enables |= tx_rings + ring_grps ?
5644                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5645         } else {
5646                 enables |= cp_rings ?
5647                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5648                 enables |= ring_grps ?
5649                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5650         }
5651         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5652         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5653
5654         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
5655         req->num_tx_rings = cpu_to_le16(tx_rings);
5656         req->num_rx_rings = cpu_to_le16(rx_rings);
5657         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5658                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5659                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5660         } else {
5661                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5662                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5663                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5664         }
5665         req->num_stat_ctxs = cpu_to_le16(stats);
5666         req->num_vnics = cpu_to_le16(vnics);
5667
5668         req->enables = cpu_to_le32(enables);
5669 }
5670
5671 static int
5672 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5673                            int ring_grps, int cp_rings, int stats, int vnics)
5674 {
5675         struct hwrm_func_cfg_input req = {0};
5676         int rc;
5677
5678         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5679                                      cp_rings, stats, vnics);
5680         if (!req.enables)
5681                 return 0;
5682
5683         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5684         if (rc)
5685                 return -ENOMEM;
5686
5687         if (bp->hwrm_spec_code < 0x10601)
5688                 bp->hw_resc.resv_tx_rings = tx_rings;
5689
5690         rc = bnxt_hwrm_get_rings(bp);
5691         return rc;
5692 }
5693
5694 static int
5695 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5696                            int ring_grps, int cp_rings, int stats, int vnics)
5697 {
5698         struct hwrm_func_vf_cfg_input req = {0};
5699         int rc;
5700
5701         if (!BNXT_NEW_RM(bp)) {
5702                 bp->hw_resc.resv_tx_rings = tx_rings;
5703                 return 0;
5704         }
5705
5706         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5707                                      cp_rings, stats, vnics);
5708         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5709         if (rc)
5710                 return -ENOMEM;
5711
5712         rc = bnxt_hwrm_get_rings(bp);
5713         return rc;
5714 }
5715
5716 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
5717                                    int cp, int stat, int vnic)
5718 {
5719         if (BNXT_PF(bp))
5720                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5721                                                   vnic);
5722         else
5723                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5724                                                   vnic);
5725 }
5726
5727 int bnxt_nq_rings_in_use(struct bnxt *bp)
5728 {
5729         int cp = bp->cp_nr_rings;
5730         int ulp_msix, ulp_base;
5731
5732         ulp_msix = bnxt_get_ulp_msix_num(bp);
5733         if (ulp_msix) {
5734                 ulp_base = bnxt_get_ulp_msix_base(bp);
5735                 cp += ulp_msix;
5736                 if ((ulp_base + ulp_msix) > cp)
5737                         cp = ulp_base + ulp_msix;
5738         }
5739         return cp;
5740 }
5741
5742 static int bnxt_cp_rings_in_use(struct bnxt *bp)
5743 {
5744         int cp;
5745
5746         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5747                 return bnxt_nq_rings_in_use(bp);
5748
5749         cp = bp->tx_nr_rings + bp->rx_nr_rings;
5750         return cp;
5751 }
5752
5753 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5754 {
5755         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
5756         int cp = bp->cp_nr_rings;
5757
5758         if (!ulp_stat)
5759                 return cp;
5760
5761         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
5762                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
5763
5764         return cp + ulp_stat;
5765 }
5766
5767 static bool bnxt_need_reserve_rings(struct bnxt *bp)
5768 {
5769         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5770         int cp = bnxt_cp_rings_in_use(bp);
5771         int nq = bnxt_nq_rings_in_use(bp);
5772         int rx = bp->rx_nr_rings, stat;
5773         int vnic = 1, grp = rx;
5774
5775         if (bp->hwrm_spec_code < 0x10601)
5776                 return false;
5777
5778         if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5779                 return true;
5780
5781         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5782                 vnic = rx + 1;
5783         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5784                 rx <<= 1;
5785         stat = bnxt_get_func_stat_ctxs(bp);
5786         if (BNXT_NEW_RM(bp) &&
5787             (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
5788              hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
5789              (hw_resc->resv_hw_ring_grps != grp &&
5790               !(bp->flags & BNXT_FLAG_CHIP_P5))))
5791                 return true;
5792         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
5793             hw_resc->resv_irqs != nq)
5794                 return true;
5795         return false;
5796 }
5797
5798 static int __bnxt_reserve_rings(struct bnxt *bp)
5799 {
5800         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5801         int cp = bnxt_nq_rings_in_use(bp);
5802         int tx = bp->tx_nr_rings;
5803         int rx = bp->rx_nr_rings;
5804         int grp, rx_rings, rc;
5805         int vnic = 1, stat;
5806         bool sh = false;
5807
5808         if (!bnxt_need_reserve_rings(bp))
5809                 return 0;
5810
5811         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5812                 sh = true;
5813         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5814                 vnic = rx + 1;
5815         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5816                 rx <<= 1;
5817         grp = bp->rx_nr_rings;
5818         stat = bnxt_get_func_stat_ctxs(bp);
5819
5820         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
5821         if (rc)
5822                 return rc;
5823
5824         tx = hw_resc->resv_tx_rings;
5825         if (BNXT_NEW_RM(bp)) {
5826                 rx = hw_resc->resv_rx_rings;
5827                 cp = hw_resc->resv_irqs;
5828                 grp = hw_resc->resv_hw_ring_grps;
5829                 vnic = hw_resc->resv_vnics;
5830                 stat = hw_resc->resv_stat_ctxs;
5831         }
5832
5833         rx_rings = rx;
5834         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5835                 if (rx >= 2) {
5836                         rx_rings = rx >> 1;
5837                 } else {
5838                         if (netif_running(bp->dev))
5839                                 return -ENOMEM;
5840
5841                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5842                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5843                         bp->dev->hw_features &= ~NETIF_F_LRO;
5844                         bp->dev->features &= ~NETIF_F_LRO;
5845                         bnxt_set_ring_params(bp);
5846                 }
5847         }
5848         rx_rings = min_t(int, rx_rings, grp);
5849         cp = min_t(int, cp, bp->cp_nr_rings);
5850         if (stat > bnxt_get_ulp_stat_ctxs(bp))
5851                 stat -= bnxt_get_ulp_stat_ctxs(bp);
5852         cp = min_t(int, cp, stat);
5853         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
5854         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5855                 rx = rx_rings << 1;
5856         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
5857         bp->tx_nr_rings = tx;
5858         bp->rx_nr_rings = rx_rings;
5859         bp->cp_nr_rings = cp;
5860
5861         if (!tx || !rx || !cp || !grp || !vnic || !stat)
5862                 return -ENOMEM;
5863
5864         return rc;
5865 }
5866
5867 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5868                                     int ring_grps, int cp_rings, int stats,
5869                                     int vnics)
5870 {
5871         struct hwrm_func_vf_cfg_input req = {0};
5872         u32 flags;
5873         int rc;
5874
5875         if (!BNXT_NEW_RM(bp))
5876                 return 0;
5877
5878         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5879                                      cp_rings, stats, vnics);
5880         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
5881                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5882                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5883                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5884                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
5885                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5886         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5887                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5888
5889         req.flags = cpu_to_le32(flags);
5890         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5891         if (rc)
5892                 return -ENOMEM;
5893         return 0;
5894 }
5895
5896 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5897                                     int ring_grps, int cp_rings, int stats,
5898                                     int vnics)
5899 {
5900         struct hwrm_func_cfg_input req = {0};
5901         u32 flags;
5902         int rc;
5903
5904         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5905                                      cp_rings, stats, vnics);
5906         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
5907         if (BNXT_NEW_RM(bp)) {
5908                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5909                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5910                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5911                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
5912                 if (bp->flags & BNXT_FLAG_CHIP_P5)
5913                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
5914                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
5915                 else
5916                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5917         }
5918
5919         req.flags = cpu_to_le32(flags);
5920         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5921         if (rc)
5922                 return -ENOMEM;
5923         return 0;
5924 }
5925
5926 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5927                                  int ring_grps, int cp_rings, int stats,
5928                                  int vnics)
5929 {
5930         if (bp->hwrm_spec_code < 0x10801)
5931                 return 0;
5932
5933         if (BNXT_PF(bp))
5934                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
5935                                                 ring_grps, cp_rings, stats,
5936                                                 vnics);
5937
5938         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
5939                                         cp_rings, stats, vnics);
5940 }
5941
5942 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
5943 {
5944         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5945         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5946         struct hwrm_ring_aggint_qcaps_input req = {0};
5947         int rc;
5948
5949         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
5950         coal_cap->num_cmpl_dma_aggr_max = 63;
5951         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
5952         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
5953         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
5954         coal_cap->int_lat_tmr_min_max = 65535;
5955         coal_cap->int_lat_tmr_max_max = 65535;
5956         coal_cap->num_cmpl_aggr_int_max = 65535;
5957         coal_cap->timer_units = 80;
5958
5959         if (bp->hwrm_spec_code < 0x10902)
5960                 return;
5961
5962         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
5963         mutex_lock(&bp->hwrm_cmd_lock);
5964         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5965         if (!rc) {
5966                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
5967                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
5968                 coal_cap->num_cmpl_dma_aggr_max =
5969                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
5970                 coal_cap->num_cmpl_dma_aggr_during_int_max =
5971                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
5972                 coal_cap->cmpl_aggr_dma_tmr_max =
5973                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
5974                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
5975                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
5976                 coal_cap->int_lat_tmr_min_max =
5977                         le16_to_cpu(resp->int_lat_tmr_min_max);
5978                 coal_cap->int_lat_tmr_max_max =
5979                         le16_to_cpu(resp->int_lat_tmr_max_max);
5980                 coal_cap->num_cmpl_aggr_int_max =
5981                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
5982                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
5983         }
5984         mutex_unlock(&bp->hwrm_cmd_lock);
5985 }
5986
5987 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
5988 {
5989         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5990
5991         return usec * 1000 / coal_cap->timer_units;
5992 }
5993
5994 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
5995         struct bnxt_coal *hw_coal,
5996         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
5997 {
5998         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5999         u32 cmpl_params = coal_cap->cmpl_params;
6000         u16 val, tmr, max, flags = 0;
6001
6002         max = hw_coal->bufs_per_record * 128;
6003         if (hw_coal->budget)
6004                 max = hw_coal->bufs_per_record * hw_coal->budget;
6005         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6006
6007         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6008         req->num_cmpl_aggr_int = cpu_to_le16(val);
6009
6010         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6011         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6012
6013         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6014                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6015         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6016
6017         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6018         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6019         req->int_lat_tmr_max = cpu_to_le16(tmr);
6020
6021         /* min timer set to 1/2 of interrupt timer */
6022         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6023                 val = tmr / 2;
6024                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6025                 req->int_lat_tmr_min = cpu_to_le16(val);
6026                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6027         }
6028
6029         /* buf timer set to 1/4 of interrupt timer */
6030         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6031         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6032
6033         if (cmpl_params &
6034             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6035                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6036                 val = clamp_t(u16, tmr, 1,
6037                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6038                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
6039                 req->enables |=
6040                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6041         }
6042
6043         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6044                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6045         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6046             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6047                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6048         req->flags = cpu_to_le16(flags);
6049         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6050 }
6051
6052 /* Caller holds bp->hwrm_cmd_lock */
6053 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6054                                    struct bnxt_coal *hw_coal)
6055 {
6056         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6057         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6058         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6059         u32 nq_params = coal_cap->nq_params;
6060         u16 tmr;
6061
6062         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6063                 return 0;
6064
6065         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6066                                -1, -1);
6067         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6068         req.flags =
6069                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6070
6071         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6072         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6073         req.int_lat_tmr_min = cpu_to_le16(tmr);
6074         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6075         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6076 }
6077
6078 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6079 {
6080         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6081         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6082         struct bnxt_coal coal;
6083
6084         /* Tick values in micro seconds.
6085          * 1 coal_buf x bufs_per_record = 1 completion record.
6086          */
6087         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6088
6089         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6090         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6091
6092         if (!bnapi->rx_ring)
6093                 return -ENODEV;
6094
6095         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6096                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6097
6098         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6099
6100         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6101
6102         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6103                                  HWRM_CMD_TIMEOUT);
6104 }
6105
6106 int bnxt_hwrm_set_coal(struct bnxt *bp)
6107 {
6108         int i, rc = 0;
6109         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6110                                                            req_tx = {0}, *req;
6111
6112         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6113                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6114         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6115                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6116
6117         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6118         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6119
6120         mutex_lock(&bp->hwrm_cmd_lock);
6121         for (i = 0; i < bp->cp_nr_rings; i++) {
6122                 struct bnxt_napi *bnapi = bp->bnapi[i];
6123                 struct bnxt_coal *hw_coal;
6124                 u16 ring_id;
6125
6126                 req = &req_rx;
6127                 if (!bnapi->rx_ring) {
6128                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6129                         req = &req_tx;
6130                 } else {
6131                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6132                 }
6133                 req->ring_id = cpu_to_le16(ring_id);
6134
6135                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6136                                         HWRM_CMD_TIMEOUT);
6137                 if (rc)
6138                         break;
6139
6140                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6141                         continue;
6142
6143                 if (bnapi->rx_ring && bnapi->tx_ring) {
6144                         req = &req_tx;
6145                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6146                         req->ring_id = cpu_to_le16(ring_id);
6147                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6148                                                 HWRM_CMD_TIMEOUT);
6149                         if (rc)
6150                                 break;
6151                 }
6152                 if (bnapi->rx_ring)
6153                         hw_coal = &bp->rx_coal;
6154                 else
6155                         hw_coal = &bp->tx_coal;
6156                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6157         }
6158         mutex_unlock(&bp->hwrm_cmd_lock);
6159         return rc;
6160 }
6161
6162 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6163 {
6164         int rc = 0, i;
6165         struct hwrm_stat_ctx_free_input req = {0};
6166
6167         if (!bp->bnapi)
6168                 return 0;
6169
6170         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6171                 return 0;
6172
6173         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6174
6175         mutex_lock(&bp->hwrm_cmd_lock);
6176         for (i = 0; i < bp->cp_nr_rings; i++) {
6177                 struct bnxt_napi *bnapi = bp->bnapi[i];
6178                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6179
6180                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6181                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6182
6183                         rc = _hwrm_send_message(bp, &req, sizeof(req),
6184                                                 HWRM_CMD_TIMEOUT);
6185                         if (rc)
6186                                 break;
6187
6188                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6189                 }
6190         }
6191         mutex_unlock(&bp->hwrm_cmd_lock);
6192         return rc;
6193 }
6194
6195 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6196 {
6197         int rc = 0, i;
6198         struct hwrm_stat_ctx_alloc_input req = {0};
6199         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6200
6201         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6202                 return 0;
6203
6204         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6205
6206         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6207         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6208
6209         mutex_lock(&bp->hwrm_cmd_lock);
6210         for (i = 0; i < bp->cp_nr_rings; i++) {
6211                 struct bnxt_napi *bnapi = bp->bnapi[i];
6212                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6213
6214                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
6215
6216                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6217                                         HWRM_CMD_TIMEOUT);
6218                 if (rc)
6219                         break;
6220
6221                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6222
6223                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6224         }
6225         mutex_unlock(&bp->hwrm_cmd_lock);
6226         return rc;
6227 }
6228
6229 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6230 {
6231         struct hwrm_func_qcfg_input req = {0};
6232         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6233         u16 flags;
6234         int rc;
6235
6236         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6237         req.fid = cpu_to_le16(0xffff);
6238         mutex_lock(&bp->hwrm_cmd_lock);
6239         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6240         if (rc)
6241                 goto func_qcfg_exit;
6242
6243 #ifdef CONFIG_BNXT_SRIOV
6244         if (BNXT_VF(bp)) {
6245                 struct bnxt_vf_info *vf = &bp->vf;
6246
6247                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6248         }
6249 #endif
6250         flags = le16_to_cpu(resp->flags);
6251         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6252                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6253                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6254                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6255                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6256         }
6257         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6258                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6259
6260         switch (resp->port_partition_type) {
6261         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6262         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6263         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6264                 bp->port_partition_type = resp->port_partition_type;
6265                 break;
6266         }
6267         if (bp->hwrm_spec_code < 0x10707 ||
6268             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6269                 bp->br_mode = BRIDGE_MODE_VEB;
6270         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6271                 bp->br_mode = BRIDGE_MODE_VEPA;
6272         else
6273                 bp->br_mode = BRIDGE_MODE_UNDEF;
6274
6275         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6276         if (!bp->max_mtu)
6277                 bp->max_mtu = BNXT_MAX_MTU;
6278
6279 func_qcfg_exit:
6280         mutex_unlock(&bp->hwrm_cmd_lock);
6281         return rc;
6282 }
6283
6284 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6285 {
6286         struct hwrm_func_backing_store_qcaps_input req = {0};
6287         struct hwrm_func_backing_store_qcaps_output *resp =
6288                 bp->hwrm_cmd_resp_addr;
6289         int rc;
6290
6291         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6292                 return 0;
6293
6294         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6295         mutex_lock(&bp->hwrm_cmd_lock);
6296         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6297         if (!rc) {
6298                 struct bnxt_ctx_pg_info *ctx_pg;
6299                 struct bnxt_ctx_mem_info *ctx;
6300                 int i;
6301
6302                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6303                 if (!ctx) {
6304                         rc = -ENOMEM;
6305                         goto ctx_err;
6306                 }
6307                 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6308                 if (!ctx_pg) {
6309                         kfree(ctx);
6310                         rc = -ENOMEM;
6311                         goto ctx_err;
6312                 }
6313                 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6314                         ctx->tqm_mem[i] = ctx_pg;
6315
6316                 bp->ctx = ctx;
6317                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6318                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6319                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6320                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6321                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6322                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6323                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6324                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6325                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6326                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6327                 ctx->vnic_max_vnic_entries =
6328                         le16_to_cpu(resp->vnic_max_vnic_entries);
6329                 ctx->vnic_max_ring_table_entries =
6330                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6331                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6332                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6333                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6334                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6335                 ctx->tqm_min_entries_per_ring =
6336                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6337                 ctx->tqm_max_entries_per_ring =
6338                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6339                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6340                 if (!ctx->tqm_entries_multiple)
6341                         ctx->tqm_entries_multiple = 1;
6342                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6343                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6344                 ctx->mrav_num_entries_units =
6345                         le16_to_cpu(resp->mrav_num_entries_units);
6346                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6347                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6348         } else {
6349                 rc = 0;
6350         }
6351 ctx_err:
6352         mutex_unlock(&bp->hwrm_cmd_lock);
6353         return rc;
6354 }
6355
6356 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6357                                   __le64 *pg_dir)
6358 {
6359         u8 pg_size = 0;
6360
6361         if (BNXT_PAGE_SHIFT == 13)
6362                 pg_size = 1 << 4;
6363         else if (BNXT_PAGE_SIZE == 16)
6364                 pg_size = 2 << 4;
6365
6366         *pg_attr = pg_size;
6367         if (rmem->depth >= 1) {
6368                 if (rmem->depth == 2)
6369                         *pg_attr |= 2;
6370                 else
6371                         *pg_attr |= 1;
6372                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6373         } else {
6374                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6375         }
6376 }
6377
6378 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6379         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6380          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6381          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6382          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6383          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6384
6385 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6386 {
6387         struct hwrm_func_backing_store_cfg_input req = {0};
6388         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6389         struct bnxt_ctx_pg_info *ctx_pg;
6390         __le32 *num_entries;
6391         __le64 *pg_dir;
6392         u32 flags = 0;
6393         u8 *pg_attr;
6394         int i, rc;
6395         u32 ena;
6396
6397         if (!ctx)
6398                 return 0;
6399
6400         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6401         req.enables = cpu_to_le32(enables);
6402
6403         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6404                 ctx_pg = &ctx->qp_mem;
6405                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6406                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6407                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6408                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6409                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6410                                       &req.qpc_pg_size_qpc_lvl,
6411                                       &req.qpc_page_dir);
6412         }
6413         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6414                 ctx_pg = &ctx->srq_mem;
6415                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6416                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6417                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6418                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6419                                       &req.srq_pg_size_srq_lvl,
6420                                       &req.srq_page_dir);
6421         }
6422         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6423                 ctx_pg = &ctx->cq_mem;
6424                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6425                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6426                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6427                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6428                                       &req.cq_page_dir);
6429         }
6430         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6431                 ctx_pg = &ctx->vnic_mem;
6432                 req.vnic_num_vnic_entries =
6433                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6434                 req.vnic_num_ring_table_entries =
6435                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6436                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6437                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6438                                       &req.vnic_pg_size_vnic_lvl,
6439                                       &req.vnic_page_dir);
6440         }
6441         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6442                 ctx_pg = &ctx->stat_mem;
6443                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6444                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6445                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6446                                       &req.stat_pg_size_stat_lvl,
6447                                       &req.stat_page_dir);
6448         }
6449         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6450                 ctx_pg = &ctx->mrav_mem;
6451                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6452                 if (ctx->mrav_num_entries_units)
6453                         flags |=
6454                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6455                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6456                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6457                                       &req.mrav_pg_size_mrav_lvl,
6458                                       &req.mrav_page_dir);
6459         }
6460         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6461                 ctx_pg = &ctx->tim_mem;
6462                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6463                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6464                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6465                                       &req.tim_pg_size_tim_lvl,
6466                                       &req.tim_page_dir);
6467         }
6468         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6469              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6470              pg_dir = &req.tqm_sp_page_dir,
6471              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6472              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6473                 if (!(enables & ena))
6474                         continue;
6475
6476                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6477                 ctx_pg = ctx->tqm_mem[i];
6478                 *num_entries = cpu_to_le32(ctx_pg->entries);
6479                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6480         }
6481         req.flags = cpu_to_le32(flags);
6482         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6483         if (rc)
6484                 rc = -EIO;
6485         return rc;
6486 }
6487
6488 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6489                                   struct bnxt_ctx_pg_info *ctx_pg)
6490 {
6491         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6492
6493         rmem->page_size = BNXT_PAGE_SIZE;
6494         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6495         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6496         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6497         if (rmem->depth >= 1)
6498                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6499         return bnxt_alloc_ring(bp, rmem);
6500 }
6501
6502 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6503                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6504                                   u8 depth)
6505 {
6506         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6507         int rc;
6508
6509         if (!mem_size)
6510                 return 0;
6511
6512         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6513         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6514                 ctx_pg->nr_pages = 0;
6515                 return -EINVAL;
6516         }
6517         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6518                 int nr_tbls, i;
6519
6520                 rmem->depth = 2;
6521                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6522                                              GFP_KERNEL);
6523                 if (!ctx_pg->ctx_pg_tbl)
6524                         return -ENOMEM;
6525                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6526                 rmem->nr_pages = nr_tbls;
6527                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6528                 if (rc)
6529                         return rc;
6530                 for (i = 0; i < nr_tbls; i++) {
6531                         struct bnxt_ctx_pg_info *pg_tbl;
6532
6533                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6534                         if (!pg_tbl)
6535                                 return -ENOMEM;
6536                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6537                         rmem = &pg_tbl->ring_mem;
6538                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6539                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6540                         rmem->depth = 1;
6541                         rmem->nr_pages = MAX_CTX_PAGES;
6542                         if (i == (nr_tbls - 1)) {
6543                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6544
6545                                 if (rem)
6546                                         rmem->nr_pages = rem;
6547                         }
6548                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6549                         if (rc)
6550                                 break;
6551                 }
6552         } else {
6553                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6554                 if (rmem->nr_pages > 1 || depth)
6555                         rmem->depth = 1;
6556                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6557         }
6558         return rc;
6559 }
6560
6561 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6562                                   struct bnxt_ctx_pg_info *ctx_pg)
6563 {
6564         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6565
6566         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6567             ctx_pg->ctx_pg_tbl) {
6568                 int i, nr_tbls = rmem->nr_pages;
6569
6570                 for (i = 0; i < nr_tbls; i++) {
6571                         struct bnxt_ctx_pg_info *pg_tbl;
6572                         struct bnxt_ring_mem_info *rmem2;
6573
6574                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
6575                         if (!pg_tbl)
6576                                 continue;
6577                         rmem2 = &pg_tbl->ring_mem;
6578                         bnxt_free_ring(bp, rmem2);
6579                         ctx_pg->ctx_pg_arr[i] = NULL;
6580                         kfree(pg_tbl);
6581                         ctx_pg->ctx_pg_tbl[i] = NULL;
6582                 }
6583                 kfree(ctx_pg->ctx_pg_tbl);
6584                 ctx_pg->ctx_pg_tbl = NULL;
6585         }
6586         bnxt_free_ring(bp, rmem);
6587         ctx_pg->nr_pages = 0;
6588 }
6589
6590 static void bnxt_free_ctx_mem(struct bnxt *bp)
6591 {
6592         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6593         int i;
6594
6595         if (!ctx)
6596                 return;
6597
6598         if (ctx->tqm_mem[0]) {
6599                 for (i = 0; i < bp->max_q + 1; i++)
6600                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
6601                 kfree(ctx->tqm_mem[0]);
6602                 ctx->tqm_mem[0] = NULL;
6603         }
6604
6605         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6606         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
6607         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6608         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6609         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6610         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6611         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
6612         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6613 }
6614
6615 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6616 {
6617         struct bnxt_ctx_pg_info *ctx_pg;
6618         struct bnxt_ctx_mem_info *ctx;
6619         u32 mem_size, ena, entries;
6620         u32 num_mr, num_ah;
6621         u32 extra_srqs = 0;
6622         u32 extra_qps = 0;
6623         u8 pg_lvl = 1;
6624         int i, rc;
6625
6626         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6627         if (rc) {
6628                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6629                            rc);
6630                 return rc;
6631         }
6632         ctx = bp->ctx;
6633         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6634                 return 0;
6635
6636         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
6637                 pg_lvl = 2;
6638                 extra_qps = 65536;
6639                 extra_srqs = 8192;
6640         }
6641
6642         ctx_pg = &ctx->qp_mem;
6643         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6644                           extra_qps;
6645         mem_size = ctx->qp_entry_size * ctx_pg->entries;
6646         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6647         if (rc)
6648                 return rc;
6649
6650         ctx_pg = &ctx->srq_mem;
6651         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
6652         mem_size = ctx->srq_entry_size * ctx_pg->entries;
6653         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6654         if (rc)
6655                 return rc;
6656
6657         ctx_pg = &ctx->cq_mem;
6658         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
6659         mem_size = ctx->cq_entry_size * ctx_pg->entries;
6660         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6661         if (rc)
6662                 return rc;
6663
6664         ctx_pg = &ctx->vnic_mem;
6665         ctx_pg->entries = ctx->vnic_max_vnic_entries +
6666                           ctx->vnic_max_ring_table_entries;
6667         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
6668         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6669         if (rc)
6670                 return rc;
6671
6672         ctx_pg = &ctx->stat_mem;
6673         ctx_pg->entries = ctx->stat_max_entries;
6674         mem_size = ctx->stat_entry_size * ctx_pg->entries;
6675         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6676         if (rc)
6677                 return rc;
6678
6679         ena = 0;
6680         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6681                 goto skip_rdma;
6682
6683         ctx_pg = &ctx->mrav_mem;
6684         /* 128K extra is needed to accommodate static AH context
6685          * allocation by f/w.
6686          */
6687         num_mr = 1024 * 256;
6688         num_ah = 1024 * 128;
6689         ctx_pg->entries = num_mr + num_ah;
6690         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6691         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2);
6692         if (rc)
6693                 return rc;
6694         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
6695         if (ctx->mrav_num_entries_units)
6696                 ctx_pg->entries =
6697                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
6698                          (num_ah / ctx->mrav_num_entries_units);
6699
6700         ctx_pg = &ctx->tim_mem;
6701         ctx_pg->entries = ctx->qp_mem.entries;
6702         mem_size = ctx->tim_entry_size * ctx_pg->entries;
6703         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6704         if (rc)
6705                 return rc;
6706         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6707
6708 skip_rdma:
6709         entries = ctx->qp_max_l2_entries + extra_qps;
6710         entries = roundup(entries, ctx->tqm_entries_multiple);
6711         entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6712                           ctx->tqm_max_entries_per_ring);
6713         for (i = 0; i < bp->max_q + 1; i++) {
6714                 ctx_pg = ctx->tqm_mem[i];
6715                 ctx_pg->entries = entries;
6716                 mem_size = ctx->tqm_entry_size * entries;
6717                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6718                 if (rc)
6719                         return rc;
6720                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
6721         }
6722         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6723         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6724         if (rc)
6725                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6726                            rc);
6727         else
6728                 ctx->flags |= BNXT_CTX_FLAG_INITED;
6729
6730         return 0;
6731 }
6732
6733 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
6734 {
6735         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6736         struct hwrm_func_resource_qcaps_input req = {0};
6737         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6738         int rc;
6739
6740         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6741         req.fid = cpu_to_le16(0xffff);
6742
6743         mutex_lock(&bp->hwrm_cmd_lock);
6744         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6745                                        HWRM_CMD_TIMEOUT);
6746         if (rc) {
6747                 rc = -EIO;
6748                 goto hwrm_func_resc_qcaps_exit;
6749         }
6750
6751         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6752         if (!all)
6753                 goto hwrm_func_resc_qcaps_exit;
6754
6755         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6756         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6757         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6758         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6759         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6760         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6761         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6762         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6763         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6764         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6765         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6766         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6767         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6768         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6769         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6770         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6771
6772         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6773                 u16 max_msix = le16_to_cpu(resp->max_msix);
6774
6775                 hw_resc->max_nqs = max_msix;
6776                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6777         }
6778
6779         if (BNXT_PF(bp)) {
6780                 struct bnxt_pf_info *pf = &bp->pf;
6781
6782                 pf->vf_resv_strategy =
6783                         le16_to_cpu(resp->vf_reservation_strategy);
6784                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
6785                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6786         }
6787 hwrm_func_resc_qcaps_exit:
6788         mutex_unlock(&bp->hwrm_cmd_lock);
6789         return rc;
6790 }
6791
6792 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
6793 {
6794         int rc = 0;
6795         struct hwrm_func_qcaps_input req = {0};
6796         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6797         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6798         u32 flags;
6799
6800         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6801         req.fid = cpu_to_le16(0xffff);
6802
6803         mutex_lock(&bp->hwrm_cmd_lock);
6804         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6805         if (rc)
6806                 goto hwrm_func_qcaps_exit;
6807
6808         flags = le32_to_cpu(resp->flags);
6809         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
6810                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6811         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
6812                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
6813         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
6814                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
6815         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
6816                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
6817
6818         bp->tx_push_thresh = 0;
6819         if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
6820                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6821
6822         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6823         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6824         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6825         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6826         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6827         if (!hw_resc->max_hw_ring_grps)
6828                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6829         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6830         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6831         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6832
6833         if (BNXT_PF(bp)) {
6834                 struct bnxt_pf_info *pf = &bp->pf;
6835
6836                 pf->fw_fid = le16_to_cpu(resp->fid);
6837                 pf->port_id = le16_to_cpu(resp->port_id);
6838                 bp->dev->dev_port = pf->port_id;
6839                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
6840                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6841                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6842                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6843                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6844                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6845                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6846                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6847                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
6848                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
6849                         bp->flags |= BNXT_FLAG_WOL_CAP;
6850         } else {
6851 #ifdef CONFIG_BNXT_SRIOV
6852                 struct bnxt_vf_info *vf = &bp->vf;
6853
6854                 vf->fw_fid = le16_to_cpu(resp->fid);
6855                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
6856 #endif
6857         }
6858
6859 hwrm_func_qcaps_exit:
6860         mutex_unlock(&bp->hwrm_cmd_lock);
6861         return rc;
6862 }
6863
6864 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
6865
6866 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
6867 {
6868         int rc;
6869
6870         rc = __bnxt_hwrm_func_qcaps(bp);
6871         if (rc)
6872                 return rc;
6873         rc = bnxt_hwrm_queue_qportcfg(bp);
6874         if (rc) {
6875                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
6876                 return rc;
6877         }
6878         if (bp->hwrm_spec_code >= 0x10803) {
6879                 rc = bnxt_alloc_ctx_mem(bp);
6880                 if (rc)
6881                         return rc;
6882                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
6883                 if (!rc)
6884                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
6885         }
6886         return 0;
6887 }
6888
6889 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
6890 {
6891         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
6892         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
6893         int rc = 0;
6894         u32 flags;
6895
6896         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
6897                 return 0;
6898
6899         resp = bp->hwrm_cmd_resp_addr;
6900         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
6901
6902         mutex_lock(&bp->hwrm_cmd_lock);
6903         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6904         if (rc)
6905                 goto hwrm_cfa_adv_qcaps_exit;
6906
6907         flags = le32_to_cpu(resp->flags);
6908         if (flags &
6909             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED)
6910                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX;
6911
6912 hwrm_cfa_adv_qcaps_exit:
6913         mutex_unlock(&bp->hwrm_cmd_lock);
6914         return rc;
6915 }
6916
6917 static int bnxt_hwrm_func_reset(struct bnxt *bp)
6918 {
6919         struct hwrm_func_reset_input req = {0};
6920
6921         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
6922         req.enables = 0;
6923
6924         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
6925 }
6926
6927 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
6928 {
6929         int rc = 0;
6930         struct hwrm_queue_qportcfg_input req = {0};
6931         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
6932         u8 i, j, *qptr;
6933         bool no_rdma;
6934
6935         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
6936
6937         mutex_lock(&bp->hwrm_cmd_lock);
6938         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6939         if (rc)
6940                 goto qportcfg_exit;
6941
6942         if (!resp->max_configurable_queues) {
6943                 rc = -EINVAL;
6944                 goto qportcfg_exit;
6945         }
6946         bp->max_tc = resp->max_configurable_queues;
6947         bp->max_lltc = resp->max_configurable_lossless_queues;
6948         if (bp->max_tc > BNXT_MAX_QUEUE)
6949                 bp->max_tc = BNXT_MAX_QUEUE;
6950
6951         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
6952         qptr = &resp->queue_id0;
6953         for (i = 0, j = 0; i < bp->max_tc; i++) {
6954                 bp->q_info[j].queue_id = *qptr;
6955                 bp->q_ids[i] = *qptr++;
6956                 bp->q_info[j].queue_profile = *qptr++;
6957                 bp->tc_to_qidx[j] = j;
6958                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
6959                     (no_rdma && BNXT_PF(bp)))
6960                         j++;
6961         }
6962         bp->max_q = bp->max_tc;
6963         bp->max_tc = max_t(u8, j, 1);
6964
6965         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
6966                 bp->max_tc = 1;
6967
6968         if (bp->max_lltc > bp->max_tc)
6969                 bp->max_lltc = bp->max_tc;
6970
6971 qportcfg_exit:
6972         mutex_unlock(&bp->hwrm_cmd_lock);
6973         return rc;
6974 }
6975
6976 static int bnxt_hwrm_ver_get(struct bnxt *bp)
6977 {
6978         int rc;
6979         struct hwrm_ver_get_input req = {0};
6980         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
6981         u32 dev_caps_cfg;
6982
6983         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
6984         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
6985         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
6986         req.hwrm_intf_min = HWRM_VERSION_MINOR;
6987         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
6988         mutex_lock(&bp->hwrm_cmd_lock);
6989         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6990         if (rc)
6991                 goto hwrm_ver_get_exit;
6992
6993         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
6994
6995         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
6996                              resp->hwrm_intf_min_8b << 8 |
6997                              resp->hwrm_intf_upd_8b;
6998         if (resp->hwrm_intf_maj_8b < 1) {
6999                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7000                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7001                             resp->hwrm_intf_upd_8b);
7002                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7003         }
7004         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
7005                  resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
7006                  resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
7007
7008         if (strlen(resp->active_pkg_name)) {
7009                 int fw_ver_len = strlen(bp->fw_ver_str);
7010
7011                 snprintf(bp->fw_ver_str + fw_ver_len,
7012                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7013                          resp->active_pkg_name);
7014                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7015         }
7016
7017         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7018         if (!bp->hwrm_cmd_timeout)
7019                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7020
7021         if (resp->hwrm_intf_maj_8b >= 1) {
7022                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7023                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7024         }
7025         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7026                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7027
7028         bp->chip_num = le16_to_cpu(resp->chip_num);
7029         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7030             !resp->chip_metal)
7031                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7032
7033         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7034         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7035             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7036                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7037
7038         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7039                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7040
7041         if (dev_caps_cfg &
7042             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7043                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7044
7045         if (dev_caps_cfg &
7046             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7047                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7048
7049         if (dev_caps_cfg &
7050             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7051                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7052
7053 hwrm_ver_get_exit:
7054         mutex_unlock(&bp->hwrm_cmd_lock);
7055         return rc;
7056 }
7057
7058 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7059 {
7060         struct hwrm_fw_set_time_input req = {0};
7061         struct tm tm;
7062         time64_t now = ktime_get_real_seconds();
7063
7064         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7065             bp->hwrm_spec_code < 0x10400)
7066                 return -EOPNOTSUPP;
7067
7068         time64_to_tm(now, 0, &tm);
7069         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7070         req.year = cpu_to_le16(1900 + tm.tm_year);
7071         req.month = 1 + tm.tm_mon;
7072         req.day = tm.tm_mday;
7073         req.hour = tm.tm_hour;
7074         req.minute = tm.tm_min;
7075         req.second = tm.tm_sec;
7076         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7077 }
7078
7079 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
7080 {
7081         int rc;
7082         struct bnxt_pf_info *pf = &bp->pf;
7083         struct hwrm_port_qstats_input req = {0};
7084
7085         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7086                 return 0;
7087
7088         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7089         req.port_id = cpu_to_le16(pf->port_id);
7090         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
7091         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
7092         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7093         return rc;
7094 }
7095
7096 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
7097 {
7098         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7099         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7100         struct hwrm_port_qstats_ext_input req = {0};
7101         struct bnxt_pf_info *pf = &bp->pf;
7102         u32 tx_stat_size;
7103         int rc;
7104
7105         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7106                 return 0;
7107
7108         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7109         req.port_id = cpu_to_le16(pf->port_id);
7110         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7111         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
7112         tx_stat_size = bp->hw_tx_port_stats_ext ?
7113                        sizeof(*bp->hw_tx_port_stats_ext) : 0;
7114         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7115         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
7116         mutex_lock(&bp->hwrm_cmd_lock);
7117         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7118         if (!rc) {
7119                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7120                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7121                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7122         } else {
7123                 bp->fw_rx_stats_ext_size = 0;
7124                 bp->fw_tx_stats_ext_size = 0;
7125         }
7126         if (bp->fw_tx_stats_ext_size <=
7127             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7128                 mutex_unlock(&bp->hwrm_cmd_lock);
7129                 bp->pri2cos_valid = 0;
7130                 return rc;
7131         }
7132
7133         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7134         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7135
7136         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7137         if (!rc) {
7138                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7139                 u8 *pri2cos;
7140                 int i, j;
7141
7142                 resp2 = bp->hwrm_cmd_resp_addr;
7143                 pri2cos = &resp2->pri0_cos_queue_id;
7144                 for (i = 0; i < 8; i++) {
7145                         u8 queue_id = pri2cos[i];
7146
7147                         for (j = 0; j < bp->max_q; j++) {
7148                                 if (bp->q_ids[j] == queue_id)
7149                                         bp->pri2cos[i] = j;
7150                         }
7151                 }
7152                 bp->pri2cos_valid = 1;
7153         }
7154         mutex_unlock(&bp->hwrm_cmd_lock);
7155         return rc;
7156 }
7157
7158 static int bnxt_hwrm_pcie_qstats(struct bnxt *bp)
7159 {
7160         struct hwrm_pcie_qstats_input req = {0};
7161
7162         if (!(bp->flags & BNXT_FLAG_PCIE_STATS))
7163                 return 0;
7164
7165         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1);
7166         req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats));
7167         req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map);
7168         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7169 }
7170
7171 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7172 {
7173         if (bp->vxlan_port_cnt) {
7174                 bnxt_hwrm_tunnel_dst_port_free(
7175                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7176         }
7177         bp->vxlan_port_cnt = 0;
7178         if (bp->nge_port_cnt) {
7179                 bnxt_hwrm_tunnel_dst_port_free(
7180                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7181         }
7182         bp->nge_port_cnt = 0;
7183 }
7184
7185 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7186 {
7187         int rc, i;
7188         u32 tpa_flags = 0;
7189
7190         if (set_tpa)
7191                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7192         for (i = 0; i < bp->nr_vnics; i++) {
7193                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7194                 if (rc) {
7195                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7196                                    i, rc);
7197                         return rc;
7198                 }
7199         }
7200         return 0;
7201 }
7202
7203 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7204 {
7205         int i;
7206
7207         for (i = 0; i < bp->nr_vnics; i++)
7208                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7209 }
7210
7211 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7212                                     bool irq_re_init)
7213 {
7214         if (bp->vnic_info) {
7215                 bnxt_hwrm_clear_vnic_filter(bp);
7216                 /* clear all RSS setting before free vnic ctx */
7217                 bnxt_hwrm_clear_vnic_rss(bp);
7218                 bnxt_hwrm_vnic_ctx_free(bp);
7219                 /* before free the vnic, undo the vnic tpa settings */
7220                 if (bp->flags & BNXT_FLAG_TPA)
7221                         bnxt_set_tpa(bp, false);
7222                 bnxt_hwrm_vnic_free(bp);
7223         }
7224         bnxt_hwrm_ring_free(bp, close_path);
7225         bnxt_hwrm_ring_grp_free(bp);
7226         if (irq_re_init) {
7227                 bnxt_hwrm_stat_ctx_free(bp);
7228                 bnxt_hwrm_free_tunnel_ports(bp);
7229         }
7230 }
7231
7232 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7233 {
7234         struct hwrm_func_cfg_input req = {0};
7235         int rc;
7236
7237         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7238         req.fid = cpu_to_le16(0xffff);
7239         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7240         if (br_mode == BRIDGE_MODE_VEB)
7241                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7242         else if (br_mode == BRIDGE_MODE_VEPA)
7243                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7244         else
7245                 return -EINVAL;
7246         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7247         if (rc)
7248                 rc = -EIO;
7249         return rc;
7250 }
7251
7252 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7253 {
7254         struct hwrm_func_cfg_input req = {0};
7255         int rc;
7256
7257         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7258                 return 0;
7259
7260         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7261         req.fid = cpu_to_le16(0xffff);
7262         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7263         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7264         if (size == 128)
7265                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7266
7267         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7268         if (rc)
7269                 rc = -EIO;
7270         return rc;
7271 }
7272
7273 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7274 {
7275         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7276         int rc;
7277
7278         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7279                 goto skip_rss_ctx;
7280
7281         /* allocate context for vnic */
7282         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7283         if (rc) {
7284                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7285                            vnic_id, rc);
7286                 goto vnic_setup_err;
7287         }
7288         bp->rsscos_nr_ctxs++;
7289
7290         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7291                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7292                 if (rc) {
7293                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7294                                    vnic_id, rc);
7295                         goto vnic_setup_err;
7296                 }
7297                 bp->rsscos_nr_ctxs++;
7298         }
7299
7300 skip_rss_ctx:
7301         /* configure default vnic, ring grp */
7302         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7303         if (rc) {
7304                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7305                            vnic_id, rc);
7306                 goto vnic_setup_err;
7307         }
7308
7309         /* Enable RSS hashing on vnic */
7310         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7311         if (rc) {
7312                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7313                            vnic_id, rc);
7314                 goto vnic_setup_err;
7315         }
7316
7317         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7318                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7319                 if (rc) {
7320                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7321                                    vnic_id, rc);
7322                 }
7323         }
7324
7325 vnic_setup_err:
7326         return rc;
7327 }
7328
7329 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7330 {
7331         int rc, i, nr_ctxs;
7332
7333         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
7334         for (i = 0; i < nr_ctxs; i++) {
7335                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7336                 if (rc) {
7337                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7338                                    vnic_id, i, rc);
7339                         break;
7340                 }
7341                 bp->rsscos_nr_ctxs++;
7342         }
7343         if (i < nr_ctxs)
7344                 return -ENOMEM;
7345
7346         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7347         if (rc) {
7348                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7349                            vnic_id, rc);
7350                 return rc;
7351         }
7352         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7353         if (rc) {
7354                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7355                            vnic_id, rc);
7356                 return rc;
7357         }
7358         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7359                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7360                 if (rc) {
7361                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7362                                    vnic_id, rc);
7363                 }
7364         }
7365         return rc;
7366 }
7367
7368 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7369 {
7370         if (bp->flags & BNXT_FLAG_CHIP_P5)
7371                 return __bnxt_setup_vnic_p5(bp, vnic_id);
7372         else
7373                 return __bnxt_setup_vnic(bp, vnic_id);
7374 }
7375
7376 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
7377 {
7378 #ifdef CONFIG_RFS_ACCEL
7379         int i, rc = 0;
7380
7381         if (bp->flags & BNXT_FLAG_CHIP_P5)
7382                 return 0;
7383
7384         for (i = 0; i < bp->rx_nr_rings; i++) {
7385                 struct bnxt_vnic_info *vnic;
7386                 u16 vnic_id = i + 1;
7387                 u16 ring_id = i;
7388
7389                 if (vnic_id >= bp->nr_vnics)
7390                         break;
7391
7392                 vnic = &bp->vnic_info[vnic_id];
7393                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7394                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7395                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
7396                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
7397                 if (rc) {
7398                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7399                                    vnic_id, rc);
7400                         break;
7401                 }
7402                 rc = bnxt_setup_vnic(bp, vnic_id);
7403                 if (rc)
7404                         break;
7405         }
7406         return rc;
7407 #else
7408         return 0;
7409 #endif
7410 }
7411
7412 /* Allow PF and VF with default VLAN to be in promiscuous mode */
7413 static bool bnxt_promisc_ok(struct bnxt *bp)
7414 {
7415 #ifdef CONFIG_BNXT_SRIOV
7416         if (BNXT_VF(bp) && !bp->vf.vlan)
7417                 return false;
7418 #endif
7419         return true;
7420 }
7421
7422 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7423 {
7424         unsigned int rc = 0;
7425
7426         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7427         if (rc) {
7428                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7429                            rc);
7430                 return rc;
7431         }
7432
7433         rc = bnxt_hwrm_vnic_cfg(bp, 1);
7434         if (rc) {
7435                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7436                            rc);
7437                 return rc;
7438         }
7439         return rc;
7440 }
7441
7442 static int bnxt_cfg_rx_mode(struct bnxt *);
7443 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
7444
7445 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7446 {
7447         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7448         int rc = 0;
7449         unsigned int rx_nr_rings = bp->rx_nr_rings;
7450
7451         if (irq_re_init) {
7452                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7453                 if (rc) {
7454                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7455                                    rc);
7456                         goto err_out;
7457                 }
7458         }
7459
7460         rc = bnxt_hwrm_ring_alloc(bp);
7461         if (rc) {
7462                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7463                 goto err_out;
7464         }
7465
7466         rc = bnxt_hwrm_ring_grp_alloc(bp);
7467         if (rc) {
7468                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7469                 goto err_out;
7470         }
7471
7472         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7473                 rx_nr_rings--;
7474
7475         /* default vnic 0 */
7476         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
7477         if (rc) {
7478                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7479                 goto err_out;
7480         }
7481
7482         rc = bnxt_setup_vnic(bp, 0);
7483         if (rc)
7484                 goto err_out;
7485
7486         if (bp->flags & BNXT_FLAG_RFS) {
7487                 rc = bnxt_alloc_rfs_vnics(bp);
7488                 if (rc)
7489                         goto err_out;
7490         }
7491
7492         if (bp->flags & BNXT_FLAG_TPA) {
7493                 rc = bnxt_set_tpa(bp, true);
7494                 if (rc)
7495                         goto err_out;
7496         }
7497
7498         if (BNXT_VF(bp))
7499                 bnxt_update_vf_mac(bp);
7500
7501         /* Filter for default vnic 0 */
7502         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7503         if (rc) {
7504                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7505                 goto err_out;
7506         }
7507         vnic->uc_filter_count = 1;
7508
7509         vnic->rx_mask = 0;
7510         if (bp->dev->flags & IFF_BROADCAST)
7511                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
7512
7513         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7514                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7515
7516         if (bp->dev->flags & IFF_ALLMULTI) {
7517                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7518                 vnic->mc_list_count = 0;
7519         } else {
7520                 u32 mask = 0;
7521
7522                 bnxt_mc_list_updated(bp, &mask);
7523                 vnic->rx_mask |= mask;
7524         }
7525
7526         rc = bnxt_cfg_rx_mode(bp);
7527         if (rc)
7528                 goto err_out;
7529
7530         rc = bnxt_hwrm_set_coal(bp);
7531         if (rc)
7532                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
7533                                 rc);
7534
7535         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7536                 rc = bnxt_setup_nitroa0_vnic(bp);
7537                 if (rc)
7538                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7539                                    rc);
7540         }
7541
7542         if (BNXT_VF(bp)) {
7543                 bnxt_hwrm_func_qcfg(bp);
7544                 netdev_update_features(bp->dev);
7545         }
7546
7547         return 0;
7548
7549 err_out:
7550         bnxt_hwrm_resource_free(bp, 0, true);
7551
7552         return rc;
7553 }
7554
7555 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7556 {
7557         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7558         return 0;
7559 }
7560
7561 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7562 {
7563         bnxt_init_cp_rings(bp);
7564         bnxt_init_rx_rings(bp);
7565         bnxt_init_tx_rings(bp);
7566         bnxt_init_ring_grps(bp, irq_re_init);
7567         bnxt_init_vnics(bp);
7568
7569         return bnxt_init_chip(bp, irq_re_init);
7570 }
7571
7572 static int bnxt_set_real_num_queues(struct bnxt *bp)
7573 {
7574         int rc;
7575         struct net_device *dev = bp->dev;
7576
7577         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7578                                           bp->tx_nr_rings_xdp);
7579         if (rc)
7580                 return rc;
7581
7582         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7583         if (rc)
7584                 return rc;
7585
7586 #ifdef CONFIG_RFS_ACCEL
7587         if (bp->flags & BNXT_FLAG_RFS)
7588                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
7589 #endif
7590
7591         return rc;
7592 }
7593
7594 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7595                            bool shared)
7596 {
7597         int _rx = *rx, _tx = *tx;
7598
7599         if (shared) {
7600                 *rx = min_t(int, _rx, max);
7601                 *tx = min_t(int, _tx, max);
7602         } else {
7603                 if (max < 2)
7604                         return -ENOMEM;
7605
7606                 while (_rx + _tx > max) {
7607                         if (_rx > _tx && _rx > 1)
7608                                 _rx--;
7609                         else if (_tx > 1)
7610                                 _tx--;
7611                 }
7612                 *rx = _rx;
7613                 *tx = _tx;
7614         }
7615         return 0;
7616 }
7617
7618 static void bnxt_setup_msix(struct bnxt *bp)
7619 {
7620         const int len = sizeof(bp->irq_tbl[0].name);
7621         struct net_device *dev = bp->dev;
7622         int tcs, i;
7623
7624         tcs = netdev_get_num_tc(dev);
7625         if (tcs > 1) {
7626                 int i, off, count;
7627
7628                 for (i = 0; i < tcs; i++) {
7629                         count = bp->tx_nr_rings_per_tc;
7630                         off = i * count;
7631                         netdev_set_tc_queue(dev, i, count, off);
7632                 }
7633         }
7634
7635         for (i = 0; i < bp->cp_nr_rings; i++) {
7636                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7637                 char *attr;
7638
7639                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7640                         attr = "TxRx";
7641                 else if (i < bp->rx_nr_rings)
7642                         attr = "rx";
7643                 else
7644                         attr = "tx";
7645
7646                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7647                          attr, i);
7648                 bp->irq_tbl[map_idx].handler = bnxt_msix;
7649         }
7650 }
7651
7652 static void bnxt_setup_inta(struct bnxt *bp)
7653 {
7654         const int len = sizeof(bp->irq_tbl[0].name);
7655
7656         if (netdev_get_num_tc(bp->dev))
7657                 netdev_reset_tc(bp->dev);
7658
7659         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7660                  0);
7661         bp->irq_tbl[0].handler = bnxt_inta;
7662 }
7663
7664 static int bnxt_setup_int_mode(struct bnxt *bp)
7665 {
7666         int rc;
7667
7668         if (bp->flags & BNXT_FLAG_USING_MSIX)
7669                 bnxt_setup_msix(bp);
7670         else
7671                 bnxt_setup_inta(bp);
7672
7673         rc = bnxt_set_real_num_queues(bp);
7674         return rc;
7675 }
7676
7677 #ifdef CONFIG_RFS_ACCEL
7678 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7679 {
7680         return bp->hw_resc.max_rsscos_ctxs;
7681 }
7682
7683 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7684 {
7685         return bp->hw_resc.max_vnics;
7686 }
7687 #endif
7688
7689 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7690 {
7691         return bp->hw_resc.max_stat_ctxs;
7692 }
7693
7694 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7695 {
7696         return bp->hw_resc.max_cp_rings;
7697 }
7698
7699 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
7700 {
7701         unsigned int cp = bp->hw_resc.max_cp_rings;
7702
7703         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7704                 cp -= bnxt_get_ulp_msix_num(bp);
7705
7706         return cp;
7707 }
7708
7709 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7710 {
7711         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7712
7713         if (bp->flags & BNXT_FLAG_CHIP_P5)
7714                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7715
7716         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7717 }
7718
7719 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
7720 {
7721         bp->hw_resc.max_irqs = max_irqs;
7722 }
7723
7724 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7725 {
7726         unsigned int cp;
7727
7728         cp = bnxt_get_max_func_cp_rings_for_en(bp);
7729         if (bp->flags & BNXT_FLAG_CHIP_P5)
7730                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
7731         else
7732                 return cp - bp->cp_nr_rings;
7733 }
7734
7735 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
7736 {
7737         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
7738 }
7739
7740 int bnxt_get_avail_msix(struct bnxt *bp, int num)
7741 {
7742         int max_cp = bnxt_get_max_func_cp_rings(bp);
7743         int max_irq = bnxt_get_max_func_irqs(bp);
7744         int total_req = bp->cp_nr_rings + num;
7745         int max_idx, avail_msix;
7746
7747         max_idx = bp->total_irqs;
7748         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7749                 max_idx = min_t(int, bp->total_irqs, max_cp);
7750         avail_msix = max_idx - bp->cp_nr_rings;
7751         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
7752                 return avail_msix;
7753
7754         if (max_irq < total_req) {
7755                 num = max_irq - bp->cp_nr_rings;
7756                 if (num <= 0)
7757                         return 0;
7758         }
7759         return num;
7760 }
7761
7762 static int bnxt_get_num_msix(struct bnxt *bp)
7763 {
7764         if (!BNXT_NEW_RM(bp))
7765                 return bnxt_get_max_func_irqs(bp);
7766
7767         return bnxt_nq_rings_in_use(bp);
7768 }
7769
7770 static int bnxt_init_msix(struct bnxt *bp)
7771 {
7772         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
7773         struct msix_entry *msix_ent;
7774
7775         total_vecs = bnxt_get_num_msix(bp);
7776         max = bnxt_get_max_func_irqs(bp);
7777         if (total_vecs > max)
7778                 total_vecs = max;
7779
7780         if (!total_vecs)
7781                 return 0;
7782
7783         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
7784         if (!msix_ent)
7785                 return -ENOMEM;
7786
7787         for (i = 0; i < total_vecs; i++) {
7788                 msix_ent[i].entry = i;
7789                 msix_ent[i].vector = 0;
7790         }
7791
7792         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
7793                 min = 2;
7794
7795         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
7796         ulp_msix = bnxt_get_ulp_msix_num(bp);
7797         if (total_vecs < 0 || total_vecs < ulp_msix) {
7798                 rc = -ENODEV;
7799                 goto msix_setup_exit;
7800         }
7801
7802         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
7803         if (bp->irq_tbl) {
7804                 for (i = 0; i < total_vecs; i++)
7805                         bp->irq_tbl[i].vector = msix_ent[i].vector;
7806
7807                 bp->total_irqs = total_vecs;
7808                 /* Trim rings based upon num of vectors allocated */
7809                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
7810                                      total_vecs - ulp_msix, min == 1);
7811                 if (rc)
7812                         goto msix_setup_exit;
7813
7814                 bp->cp_nr_rings = (min == 1) ?
7815                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7816                                   bp->tx_nr_rings + bp->rx_nr_rings;
7817
7818         } else {
7819                 rc = -ENOMEM;
7820                 goto msix_setup_exit;
7821         }
7822         bp->flags |= BNXT_FLAG_USING_MSIX;
7823         kfree(msix_ent);
7824         return 0;
7825
7826 msix_setup_exit:
7827         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
7828         kfree(bp->irq_tbl);
7829         bp->irq_tbl = NULL;
7830         pci_disable_msix(bp->pdev);
7831         kfree(msix_ent);
7832         return rc;
7833 }
7834
7835 static int bnxt_init_inta(struct bnxt *bp)
7836 {
7837         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7838         if (!bp->irq_tbl)
7839                 return -ENOMEM;
7840
7841         bp->total_irqs = 1;
7842         bp->rx_nr_rings = 1;
7843         bp->tx_nr_rings = 1;
7844         bp->cp_nr_rings = 1;
7845         bp->flags |= BNXT_FLAG_SHARED_RINGS;
7846         bp->irq_tbl[0].vector = bp->pdev->irq;
7847         return 0;
7848 }
7849
7850 static int bnxt_init_int_mode(struct bnxt *bp)
7851 {
7852         int rc = 0;
7853
7854         if (bp->flags & BNXT_FLAG_MSIX_CAP)
7855                 rc = bnxt_init_msix(bp);
7856
7857         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
7858                 /* fallback to INTA */
7859                 rc = bnxt_init_inta(bp);
7860         }
7861         return rc;
7862 }
7863
7864 static void bnxt_clear_int_mode(struct bnxt *bp)
7865 {
7866         if (bp->flags & BNXT_FLAG_USING_MSIX)
7867                 pci_disable_msix(bp->pdev);
7868
7869         kfree(bp->irq_tbl);
7870         bp->irq_tbl = NULL;
7871         bp->flags &= ~BNXT_FLAG_USING_MSIX;
7872 }
7873
7874 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
7875 {
7876         int tcs = netdev_get_num_tc(bp->dev);
7877         bool irq_cleared = false;
7878         int rc;
7879
7880         if (!bnxt_need_reserve_rings(bp))
7881                 return 0;
7882
7883         if (irq_re_init && BNXT_NEW_RM(bp) &&
7884             bnxt_get_num_msix(bp) != bp->total_irqs) {
7885                 bnxt_ulp_irq_stop(bp);
7886                 bnxt_clear_int_mode(bp);
7887                 irq_cleared = true;
7888         }
7889         rc = __bnxt_reserve_rings(bp);
7890         if (irq_cleared) {
7891                 if (!rc)
7892                         rc = bnxt_init_int_mode(bp);
7893                 bnxt_ulp_irq_restart(bp, rc);
7894         }
7895         if (rc) {
7896                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
7897                 return rc;
7898         }
7899         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
7900                 netdev_err(bp->dev, "tx ring reservation failure\n");
7901                 netdev_reset_tc(bp->dev);
7902                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
7903                 return -ENOMEM;
7904         }
7905         return 0;
7906 }
7907
7908 static void bnxt_free_irq(struct bnxt *bp)
7909 {
7910         struct bnxt_irq *irq;
7911         int i;
7912
7913 #ifdef CONFIG_RFS_ACCEL
7914         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
7915         bp->dev->rx_cpu_rmap = NULL;
7916 #endif
7917         if (!bp->irq_tbl || !bp->bnapi)
7918                 return;
7919
7920         for (i = 0; i < bp->cp_nr_rings; i++) {
7921                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7922
7923                 irq = &bp->irq_tbl[map_idx];
7924                 if (irq->requested) {
7925                         if (irq->have_cpumask) {
7926                                 irq_set_affinity_hint(irq->vector, NULL);
7927                                 free_cpumask_var(irq->cpu_mask);
7928                                 irq->have_cpumask = 0;
7929                         }
7930                         free_irq(irq->vector, bp->bnapi[i]);
7931                 }
7932
7933                 irq->requested = 0;
7934         }
7935 }
7936
7937 static int bnxt_request_irq(struct bnxt *bp)
7938 {
7939         int i, j, rc = 0;
7940         unsigned long flags = 0;
7941 #ifdef CONFIG_RFS_ACCEL
7942         struct cpu_rmap *rmap;
7943 #endif
7944
7945         rc = bnxt_setup_int_mode(bp);
7946         if (rc) {
7947                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
7948                            rc);
7949                 return rc;
7950         }
7951 #ifdef CONFIG_RFS_ACCEL
7952         rmap = bp->dev->rx_cpu_rmap;
7953 #endif
7954         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
7955                 flags = IRQF_SHARED;
7956
7957         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
7958                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7959                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
7960
7961 #ifdef CONFIG_RFS_ACCEL
7962                 if (rmap && bp->bnapi[i]->rx_ring) {
7963                         rc = irq_cpu_rmap_add(rmap, irq->vector);
7964                         if (rc)
7965                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
7966                                             j);
7967                         j++;
7968                 }
7969 #endif
7970                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
7971                                  bp->bnapi[i]);
7972                 if (rc)
7973                         break;
7974
7975                 irq->requested = 1;
7976
7977                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
7978                         int numa_node = dev_to_node(&bp->pdev->dev);
7979
7980                         irq->have_cpumask = 1;
7981                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
7982                                         irq->cpu_mask);
7983                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
7984                         if (rc) {
7985                                 netdev_warn(bp->dev,
7986                                             "Set affinity failed, IRQ = %d\n",
7987                                             irq->vector);
7988                                 break;
7989                         }
7990                 }
7991         }
7992         return rc;
7993 }
7994
7995 static void bnxt_del_napi(struct bnxt *bp)
7996 {
7997         int i;
7998
7999         if (!bp->bnapi)
8000                 return;
8001
8002         for (i = 0; i < bp->cp_nr_rings; i++) {
8003                 struct bnxt_napi *bnapi = bp->bnapi[i];
8004
8005                 napi_hash_del(&bnapi->napi);
8006                 netif_napi_del(&bnapi->napi);
8007         }
8008         /* We called napi_hash_del() before netif_napi_del(), we need
8009          * to respect an RCU grace period before freeing napi structures.
8010          */
8011         synchronize_net();
8012 }
8013
8014 static void bnxt_init_napi(struct bnxt *bp)
8015 {
8016         int i;
8017         unsigned int cp_nr_rings = bp->cp_nr_rings;
8018         struct bnxt_napi *bnapi;
8019
8020         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8021                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8022
8023                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8024                         poll_fn = bnxt_poll_p5;
8025                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8026                         cp_nr_rings--;
8027                 for (i = 0; i < cp_nr_rings; i++) {
8028                         bnapi = bp->bnapi[i];
8029                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8030                 }
8031                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8032                         bnapi = bp->bnapi[cp_nr_rings];
8033                         netif_napi_add(bp->dev, &bnapi->napi,
8034                                        bnxt_poll_nitroa0, 64);
8035                 }
8036         } else {
8037                 bnapi = bp->bnapi[0];
8038                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8039         }
8040 }
8041
8042 static void bnxt_disable_napi(struct bnxt *bp)
8043 {
8044         int i;
8045
8046         if (!bp->bnapi)
8047                 return;
8048
8049         for (i = 0; i < bp->cp_nr_rings; i++) {
8050                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8051
8052                 if (bp->bnapi[i]->rx_ring)
8053                         cancel_work_sync(&cpr->dim.work);
8054
8055                 napi_disable(&bp->bnapi[i]->napi);
8056         }
8057 }
8058
8059 static void bnxt_enable_napi(struct bnxt *bp)
8060 {
8061         int i;
8062
8063         for (i = 0; i < bp->cp_nr_rings; i++) {
8064                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8065                 bp->bnapi[i]->in_reset = false;
8066
8067                 if (bp->bnapi[i]->rx_ring) {
8068                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8069                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8070                 }
8071                 napi_enable(&bp->bnapi[i]->napi);
8072         }
8073 }
8074
8075 void bnxt_tx_disable(struct bnxt *bp)
8076 {
8077         int i;
8078         struct bnxt_tx_ring_info *txr;
8079
8080         if (bp->tx_ring) {
8081                 for (i = 0; i < bp->tx_nr_rings; i++) {
8082                         txr = &bp->tx_ring[i];
8083                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8084                 }
8085         }
8086         /* Stop all TX queues */
8087         netif_tx_disable(bp->dev);
8088         netif_carrier_off(bp->dev);
8089 }
8090
8091 void bnxt_tx_enable(struct bnxt *bp)
8092 {
8093         int i;
8094         struct bnxt_tx_ring_info *txr;
8095
8096         for (i = 0; i < bp->tx_nr_rings; i++) {
8097                 txr = &bp->tx_ring[i];
8098                 txr->dev_state = 0;
8099         }
8100         netif_tx_wake_all_queues(bp->dev);
8101         if (bp->link_info.link_up)
8102                 netif_carrier_on(bp->dev);
8103 }
8104
8105 static void bnxt_report_link(struct bnxt *bp)
8106 {
8107         if (bp->link_info.link_up) {
8108                 const char *duplex;
8109                 const char *flow_ctrl;
8110                 u32 speed;
8111                 u16 fec;
8112
8113                 netif_carrier_on(bp->dev);
8114                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8115                         duplex = "full";
8116                 else
8117                         duplex = "half";
8118                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8119                         flow_ctrl = "ON - receive & transmit";
8120                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8121                         flow_ctrl = "ON - transmit";
8122                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8123                         flow_ctrl = "ON - receive";
8124                 else
8125                         flow_ctrl = "none";
8126                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8127                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8128                             speed, duplex, flow_ctrl);
8129                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8130                         netdev_info(bp->dev, "EEE is %s\n",
8131                                     bp->eee.eee_active ? "active" :
8132                                                          "not active");
8133                 fec = bp->link_info.fec_cfg;
8134                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8135                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8136                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8137                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8138                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8139         } else {
8140                 netif_carrier_off(bp->dev);
8141                 netdev_err(bp->dev, "NIC Link is Down\n");
8142         }
8143 }
8144
8145 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8146 {
8147         int rc = 0;
8148         struct hwrm_port_phy_qcaps_input req = {0};
8149         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8150         struct bnxt_link_info *link_info = &bp->link_info;
8151
8152         if (bp->hwrm_spec_code < 0x10201)
8153                 return 0;
8154
8155         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8156
8157         mutex_lock(&bp->hwrm_cmd_lock);
8158         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8159         if (rc)
8160                 goto hwrm_phy_qcaps_exit;
8161
8162         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8163                 struct ethtool_eee *eee = &bp->eee;
8164                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8165
8166                 bp->flags |= BNXT_FLAG_EEE_CAP;
8167                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8168                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8169                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8170                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8171                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8172         }
8173         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8174                 if (bp->test_info)
8175                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8176         }
8177         if (resp->supported_speeds_auto_mode)
8178                 link_info->support_auto_speeds =
8179                         le16_to_cpu(resp->supported_speeds_auto_mode);
8180
8181         bp->port_count = resp->port_cnt;
8182
8183 hwrm_phy_qcaps_exit:
8184         mutex_unlock(&bp->hwrm_cmd_lock);
8185         return rc;
8186 }
8187
8188 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8189 {
8190         int rc = 0;
8191         struct bnxt_link_info *link_info = &bp->link_info;
8192         struct hwrm_port_phy_qcfg_input req = {0};
8193         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8194         u8 link_up = link_info->link_up;
8195         u16 diff;
8196
8197         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8198
8199         mutex_lock(&bp->hwrm_cmd_lock);
8200         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8201         if (rc) {
8202                 mutex_unlock(&bp->hwrm_cmd_lock);
8203                 return rc;
8204         }
8205
8206         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8207         link_info->phy_link_status = resp->link;
8208         link_info->duplex = resp->duplex_cfg;
8209         if (bp->hwrm_spec_code >= 0x10800)
8210                 link_info->duplex = resp->duplex_state;
8211         link_info->pause = resp->pause;
8212         link_info->auto_mode = resp->auto_mode;
8213         link_info->auto_pause_setting = resp->auto_pause;
8214         link_info->lp_pause = resp->link_partner_adv_pause;
8215         link_info->force_pause_setting = resp->force_pause;
8216         link_info->duplex_setting = resp->duplex_cfg;
8217         if (link_info->phy_link_status == BNXT_LINK_LINK)
8218                 link_info->link_speed = le16_to_cpu(resp->link_speed);
8219         else
8220                 link_info->link_speed = 0;
8221         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8222         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8223         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8224         link_info->lp_auto_link_speeds =
8225                 le16_to_cpu(resp->link_partner_adv_speeds);
8226         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8227         link_info->phy_ver[0] = resp->phy_maj;
8228         link_info->phy_ver[1] = resp->phy_min;
8229         link_info->phy_ver[2] = resp->phy_bld;
8230         link_info->media_type = resp->media_type;
8231         link_info->phy_type = resp->phy_type;
8232         link_info->transceiver = resp->xcvr_pkg_type;
8233         link_info->phy_addr = resp->eee_config_phy_addr &
8234                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8235         link_info->module_status = resp->module_status;
8236
8237         if (bp->flags & BNXT_FLAG_EEE_CAP) {
8238                 struct ethtool_eee *eee = &bp->eee;
8239                 u16 fw_speeds;
8240
8241                 eee->eee_active = 0;
8242                 if (resp->eee_config_phy_addr &
8243                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8244                         eee->eee_active = 1;
8245                         fw_speeds = le16_to_cpu(
8246                                 resp->link_partner_adv_eee_link_speed_mask);
8247                         eee->lp_advertised =
8248                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8249                 }
8250
8251                 /* Pull initial EEE config */
8252                 if (!chng_link_state) {
8253                         if (resp->eee_config_phy_addr &
8254                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8255                                 eee->eee_enabled = 1;
8256
8257                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8258                         eee->advertised =
8259                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8260
8261                         if (resp->eee_config_phy_addr &
8262                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8263                                 __le32 tmr;
8264
8265                                 eee->tx_lpi_enabled = 1;
8266                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8267                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8268                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8269                         }
8270                 }
8271         }
8272
8273         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8274         if (bp->hwrm_spec_code >= 0x10504)
8275                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8276
8277         /* TODO: need to add more logic to report VF link */
8278         if (chng_link_state) {
8279                 if (link_info->phy_link_status == BNXT_LINK_LINK)
8280                         link_info->link_up = 1;
8281                 else
8282                         link_info->link_up = 0;
8283                 if (link_up != link_info->link_up)
8284                         bnxt_report_link(bp);
8285         } else {
8286                 /* alwasy link down if not require to update link state */
8287                 link_info->link_up = 0;
8288         }
8289         mutex_unlock(&bp->hwrm_cmd_lock);
8290
8291         if (!BNXT_SINGLE_PF(bp))
8292                 return 0;
8293
8294         diff = link_info->support_auto_speeds ^ link_info->advertising;
8295         if ((link_info->support_auto_speeds | diff) !=
8296             link_info->support_auto_speeds) {
8297                 /* An advertised speed is no longer supported, so we need to
8298                  * update the advertisement settings.  Caller holds RTNL
8299                  * so we can modify link settings.
8300                  */
8301                 link_info->advertising = link_info->support_auto_speeds;
8302                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
8303                         bnxt_hwrm_set_link_setting(bp, true, false);
8304         }
8305         return 0;
8306 }
8307
8308 static void bnxt_get_port_module_status(struct bnxt *bp)
8309 {
8310         struct bnxt_link_info *link_info = &bp->link_info;
8311         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8312         u8 module_status;
8313
8314         if (bnxt_update_link(bp, true))
8315                 return;
8316
8317         module_status = link_info->module_status;
8318         switch (module_status) {
8319         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8320         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8321         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8322                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8323                             bp->pf.port_id);
8324                 if (bp->hwrm_spec_code >= 0x10201) {
8325                         netdev_warn(bp->dev, "Module part number %s\n",
8326                                     resp->phy_vendor_partnumber);
8327                 }
8328                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8329                         netdev_warn(bp->dev, "TX is disabled\n");
8330                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8331                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8332         }
8333 }
8334
8335 static void
8336 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8337 {
8338         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
8339                 if (bp->hwrm_spec_code >= 0x10201)
8340                         req->auto_pause =
8341                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
8342                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8343                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8344                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8345                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
8346                 req->enables |=
8347                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8348         } else {
8349                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8350                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8351                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8352                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
8353                 req->enables |=
8354                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
8355                 if (bp->hwrm_spec_code >= 0x10201) {
8356                         req->auto_pause = req->force_pause;
8357                         req->enables |= cpu_to_le32(
8358                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8359                 }
8360         }
8361 }
8362
8363 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
8364                                       struct hwrm_port_phy_cfg_input *req)
8365 {
8366         u8 autoneg = bp->link_info.autoneg;
8367         u16 fw_link_speed = bp->link_info.req_link_speed;
8368         u16 advertising = bp->link_info.advertising;
8369
8370         if (autoneg & BNXT_AUTONEG_SPEED) {
8371                 req->auto_mode |=
8372                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
8373
8374                 req->enables |= cpu_to_le32(
8375                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
8376                 req->auto_link_speed_mask = cpu_to_le16(advertising);
8377
8378                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
8379                 req->flags |=
8380                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
8381         } else {
8382                 req->force_link_speed = cpu_to_le16(fw_link_speed);
8383                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
8384         }
8385
8386         /* tell chimp that the setting takes effect immediately */
8387         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8388 }
8389
8390 int bnxt_hwrm_set_pause(struct bnxt *bp)
8391 {
8392         struct hwrm_port_phy_cfg_input req = {0};
8393         int rc;
8394
8395         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8396         bnxt_hwrm_set_pause_common(bp, &req);
8397
8398         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8399             bp->link_info.force_link_chng)
8400                 bnxt_hwrm_set_link_common(bp, &req);
8401
8402         mutex_lock(&bp->hwrm_cmd_lock);
8403         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8404         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8405                 /* since changing of pause setting doesn't trigger any link
8406                  * change event, the driver needs to update the current pause
8407                  * result upon successfully return of the phy_cfg command
8408                  */
8409                 bp->link_info.pause =
8410                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8411                 bp->link_info.auto_pause_setting = 0;
8412                 if (!bp->link_info.force_link_chng)
8413                         bnxt_report_link(bp);
8414         }
8415         bp->link_info.force_link_chng = false;
8416         mutex_unlock(&bp->hwrm_cmd_lock);
8417         return rc;
8418 }
8419
8420 static void bnxt_hwrm_set_eee(struct bnxt *bp,
8421                               struct hwrm_port_phy_cfg_input *req)
8422 {
8423         struct ethtool_eee *eee = &bp->eee;
8424
8425         if (eee->eee_enabled) {
8426                 u16 eee_speeds;
8427                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8428
8429                 if (eee->tx_lpi_enabled)
8430                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8431                 else
8432                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8433
8434                 req->flags |= cpu_to_le32(flags);
8435                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8436                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8437                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8438         } else {
8439                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8440         }
8441 }
8442
8443 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
8444 {
8445         struct hwrm_port_phy_cfg_input req = {0};
8446
8447         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8448         if (set_pause)
8449                 bnxt_hwrm_set_pause_common(bp, &req);
8450
8451         bnxt_hwrm_set_link_common(bp, &req);
8452
8453         if (set_eee)
8454                 bnxt_hwrm_set_eee(bp, &req);
8455         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8456 }
8457
8458 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8459 {
8460         struct hwrm_port_phy_cfg_input req = {0};
8461
8462         if (!BNXT_SINGLE_PF(bp))
8463                 return 0;
8464
8465         if (pci_num_vf(bp->pdev))
8466                 return 0;
8467
8468         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8469         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
8470         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8471 }
8472
8473 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8474 {
8475         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8476         struct hwrm_func_drv_if_change_input req = {0};
8477         bool resc_reinit = false;
8478         int rc;
8479
8480         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8481                 return 0;
8482
8483         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8484         if (up)
8485                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8486         mutex_lock(&bp->hwrm_cmd_lock);
8487         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8488         if (!rc && (resp->flags &
8489                     cpu_to_le32(FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)))
8490                 resc_reinit = true;
8491         mutex_unlock(&bp->hwrm_cmd_lock);
8492
8493         if (up && resc_reinit && BNXT_NEW_RM(bp)) {
8494                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8495
8496                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8497                 hw_resc->resv_cp_rings = 0;
8498                 hw_resc->resv_stat_ctxs = 0;
8499                 hw_resc->resv_irqs = 0;
8500                 hw_resc->resv_tx_rings = 0;
8501                 hw_resc->resv_rx_rings = 0;
8502                 hw_resc->resv_hw_ring_grps = 0;
8503                 hw_resc->resv_vnics = 0;
8504                 bp->tx_nr_rings = 0;
8505                 bp->rx_nr_rings = 0;
8506         }
8507         return rc;
8508 }
8509
8510 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8511 {
8512         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8513         struct hwrm_port_led_qcaps_input req = {0};
8514         struct bnxt_pf_info *pf = &bp->pf;
8515         int rc;
8516
8517         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8518                 return 0;
8519
8520         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8521         req.port_id = cpu_to_le16(pf->port_id);
8522         mutex_lock(&bp->hwrm_cmd_lock);
8523         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8524         if (rc) {
8525                 mutex_unlock(&bp->hwrm_cmd_lock);
8526                 return rc;
8527         }
8528         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8529                 int i;
8530
8531                 bp->num_leds = resp->num_leds;
8532                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8533                                                  bp->num_leds);
8534                 for (i = 0; i < bp->num_leds; i++) {
8535                         struct bnxt_led_info *led = &bp->leds[i];
8536                         __le16 caps = led->led_state_caps;
8537
8538                         if (!led->led_group_id ||
8539                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
8540                                 bp->num_leds = 0;
8541                                 break;
8542                         }
8543                 }
8544         }
8545         mutex_unlock(&bp->hwrm_cmd_lock);
8546         return 0;
8547 }
8548
8549 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8550 {
8551         struct hwrm_wol_filter_alloc_input req = {0};
8552         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8553         int rc;
8554
8555         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8556         req.port_id = cpu_to_le16(bp->pf.port_id);
8557         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8558         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8559         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8560         mutex_lock(&bp->hwrm_cmd_lock);
8561         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8562         if (!rc)
8563                 bp->wol_filter_id = resp->wol_filter_id;
8564         mutex_unlock(&bp->hwrm_cmd_lock);
8565         return rc;
8566 }
8567
8568 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8569 {
8570         struct hwrm_wol_filter_free_input req = {0};
8571         int rc;
8572
8573         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8574         req.port_id = cpu_to_le16(bp->pf.port_id);
8575         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8576         req.wol_filter_id = bp->wol_filter_id;
8577         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8578         return rc;
8579 }
8580
8581 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8582 {
8583         struct hwrm_wol_filter_qcfg_input req = {0};
8584         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8585         u16 next_handle = 0;
8586         int rc;
8587
8588         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8589         req.port_id = cpu_to_le16(bp->pf.port_id);
8590         req.handle = cpu_to_le16(handle);
8591         mutex_lock(&bp->hwrm_cmd_lock);
8592         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8593         if (!rc) {
8594                 next_handle = le16_to_cpu(resp->next_handle);
8595                 if (next_handle != 0) {
8596                         if (resp->wol_type ==
8597                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8598                                 bp->wol = 1;
8599                                 bp->wol_filter_id = resp->wol_filter_id;
8600                         }
8601                 }
8602         }
8603         mutex_unlock(&bp->hwrm_cmd_lock);
8604         return next_handle;
8605 }
8606
8607 static void bnxt_get_wol_settings(struct bnxt *bp)
8608 {
8609         u16 handle = 0;
8610
8611         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8612                 return;
8613
8614         do {
8615                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8616         } while (handle && handle != 0xffff);
8617 }
8618
8619 #ifdef CONFIG_BNXT_HWMON
8620 static ssize_t bnxt_show_temp(struct device *dev,
8621                               struct device_attribute *devattr, char *buf)
8622 {
8623         struct hwrm_temp_monitor_query_input req = {0};
8624         struct hwrm_temp_monitor_query_output *resp;
8625         struct bnxt *bp = dev_get_drvdata(dev);
8626         u32 temp = 0;
8627
8628         resp = bp->hwrm_cmd_resp_addr;
8629         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8630         mutex_lock(&bp->hwrm_cmd_lock);
8631         if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8632                 temp = resp->temp * 1000; /* display millidegree */
8633         mutex_unlock(&bp->hwrm_cmd_lock);
8634
8635         return sprintf(buf, "%u\n", temp);
8636 }
8637 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8638
8639 static struct attribute *bnxt_attrs[] = {
8640         &sensor_dev_attr_temp1_input.dev_attr.attr,
8641         NULL
8642 };
8643 ATTRIBUTE_GROUPS(bnxt);
8644
8645 static void bnxt_hwmon_close(struct bnxt *bp)
8646 {
8647         if (bp->hwmon_dev) {
8648                 hwmon_device_unregister(bp->hwmon_dev);
8649                 bp->hwmon_dev = NULL;
8650         }
8651 }
8652
8653 static void bnxt_hwmon_open(struct bnxt *bp)
8654 {
8655         struct pci_dev *pdev = bp->pdev;
8656
8657         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8658                                                           DRV_MODULE_NAME, bp,
8659                                                           bnxt_groups);
8660         if (IS_ERR(bp->hwmon_dev)) {
8661                 bp->hwmon_dev = NULL;
8662                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8663         }
8664 }
8665 #else
8666 static void bnxt_hwmon_close(struct bnxt *bp)
8667 {
8668 }
8669
8670 static void bnxt_hwmon_open(struct bnxt *bp)
8671 {
8672 }
8673 #endif
8674
8675 static bool bnxt_eee_config_ok(struct bnxt *bp)
8676 {
8677         struct ethtool_eee *eee = &bp->eee;
8678         struct bnxt_link_info *link_info = &bp->link_info;
8679
8680         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8681                 return true;
8682
8683         if (eee->eee_enabled) {
8684                 u32 advertising =
8685                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8686
8687                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8688                         eee->eee_enabled = 0;
8689                         return false;
8690                 }
8691                 if (eee->advertised & ~advertising) {
8692                         eee->advertised = advertising & eee->supported;
8693                         return false;
8694                 }
8695         }
8696         return true;
8697 }
8698
8699 static int bnxt_update_phy_setting(struct bnxt *bp)
8700 {
8701         int rc;
8702         bool update_link = false;
8703         bool update_pause = false;
8704         bool update_eee = false;
8705         struct bnxt_link_info *link_info = &bp->link_info;
8706
8707         rc = bnxt_update_link(bp, true);
8708         if (rc) {
8709                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
8710                            rc);
8711                 return rc;
8712         }
8713         if (!BNXT_SINGLE_PF(bp))
8714                 return 0;
8715
8716         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8717             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
8718             link_info->req_flow_ctrl)
8719                 update_pause = true;
8720         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8721             link_info->force_pause_setting != link_info->req_flow_ctrl)
8722                 update_pause = true;
8723         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8724                 if (BNXT_AUTO_MODE(link_info->auto_mode))
8725                         update_link = true;
8726                 if (link_info->req_link_speed != link_info->force_link_speed)
8727                         update_link = true;
8728                 if (link_info->req_duplex != link_info->duplex_setting)
8729                         update_link = true;
8730         } else {
8731                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
8732                         update_link = true;
8733                 if (link_info->advertising != link_info->auto_link_speeds)
8734                         update_link = true;
8735         }
8736
8737         /* The last close may have shutdown the link, so need to call
8738          * PHY_CFG to bring it back up.
8739          */
8740         if (!netif_carrier_ok(bp->dev))
8741                 update_link = true;
8742
8743         if (!bnxt_eee_config_ok(bp))
8744                 update_eee = true;
8745
8746         if (update_link)
8747                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
8748         else if (update_pause)
8749                 rc = bnxt_hwrm_set_pause(bp);
8750         if (rc) {
8751                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
8752                            rc);
8753                 return rc;
8754         }
8755
8756         return rc;
8757 }
8758
8759 /* Common routine to pre-map certain register block to different GRC window.
8760  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
8761  * in PF and 3 windows in VF that can be customized to map in different
8762  * register blocks.
8763  */
8764 static void bnxt_preset_reg_win(struct bnxt *bp)
8765 {
8766         if (BNXT_PF(bp)) {
8767                 /* CAG registers map to GRC window #4 */
8768                 writel(BNXT_CAG_REG_BASE,
8769                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
8770         }
8771 }
8772
8773 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
8774
8775 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8776 {
8777         int rc = 0;
8778
8779         bnxt_preset_reg_win(bp);
8780         netif_carrier_off(bp->dev);
8781         if (irq_re_init) {
8782                 /* Reserve rings now if none were reserved at driver probe. */
8783                 rc = bnxt_init_dflt_ring_mode(bp);
8784                 if (rc) {
8785                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
8786                         return rc;
8787                 }
8788         }
8789         rc = bnxt_reserve_rings(bp, irq_re_init);
8790         if (rc)
8791                 return rc;
8792         if ((bp->flags & BNXT_FLAG_RFS) &&
8793             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
8794                 /* disable RFS if falling back to INTA */
8795                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
8796                 bp->flags &= ~BNXT_FLAG_RFS;
8797         }
8798
8799         rc = bnxt_alloc_mem(bp, irq_re_init);
8800         if (rc) {
8801                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8802                 goto open_err_free_mem;
8803         }
8804
8805         if (irq_re_init) {
8806                 bnxt_init_napi(bp);
8807                 rc = bnxt_request_irq(bp);
8808                 if (rc) {
8809                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
8810                         goto open_err_irq;
8811                 }
8812         }
8813
8814         bnxt_enable_napi(bp);
8815         bnxt_debug_dev_init(bp);
8816
8817         rc = bnxt_init_nic(bp, irq_re_init);
8818         if (rc) {
8819                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8820                 goto open_err;
8821         }
8822
8823         if (link_re_init) {
8824                 mutex_lock(&bp->link_lock);
8825                 rc = bnxt_update_phy_setting(bp);
8826                 mutex_unlock(&bp->link_lock);
8827                 if (rc) {
8828                         netdev_warn(bp->dev, "failed to update phy settings\n");
8829                         if (BNXT_SINGLE_PF(bp)) {
8830                                 bp->link_info.phy_retry = true;
8831                                 bp->link_info.phy_retry_expires =
8832                                         jiffies + 5 * HZ;
8833                         }
8834                 }
8835         }
8836
8837         if (irq_re_init)
8838                 udp_tunnel_get_rx_info(bp->dev);
8839
8840         set_bit(BNXT_STATE_OPEN, &bp->state);
8841         bnxt_enable_int(bp);
8842         /* Enable TX queues */
8843         bnxt_tx_enable(bp);
8844         mod_timer(&bp->timer, jiffies + bp->current_interval);
8845         /* Poll link status and check for SFP+ module status */
8846         bnxt_get_port_module_status(bp);
8847
8848         /* VF-reps may need to be re-opened after the PF is re-opened */
8849         if (BNXT_PF(bp))
8850                 bnxt_vf_reps_open(bp);
8851         return 0;
8852
8853 open_err:
8854         bnxt_debug_dev_exit(bp);
8855         bnxt_disable_napi(bp);
8856
8857 open_err_irq:
8858         bnxt_del_napi(bp);
8859
8860 open_err_free_mem:
8861         bnxt_free_skbs(bp);
8862         bnxt_free_irq(bp);
8863         bnxt_free_mem(bp, true);
8864         return rc;
8865 }
8866
8867 /* rtnl_lock held */
8868 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8869 {
8870         int rc = 0;
8871
8872         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
8873         if (rc) {
8874                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
8875                 dev_close(bp->dev);
8876         }
8877         return rc;
8878 }
8879
8880 /* rtnl_lock held, open the NIC half way by allocating all resources, but
8881  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
8882  * self tests.
8883  */
8884 int bnxt_half_open_nic(struct bnxt *bp)
8885 {
8886         int rc = 0;
8887
8888         rc = bnxt_alloc_mem(bp, false);
8889         if (rc) {
8890                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8891                 goto half_open_err;
8892         }
8893         rc = bnxt_init_nic(bp, false);
8894         if (rc) {
8895                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8896                 goto half_open_err;
8897         }
8898         return 0;
8899
8900 half_open_err:
8901         bnxt_free_skbs(bp);
8902         bnxt_free_mem(bp, false);
8903         dev_close(bp->dev);
8904         return rc;
8905 }
8906
8907 /* rtnl_lock held, this call can only be made after a previous successful
8908  * call to bnxt_half_open_nic().
8909  */
8910 void bnxt_half_close_nic(struct bnxt *bp)
8911 {
8912         bnxt_hwrm_resource_free(bp, false, false);
8913         bnxt_free_skbs(bp);
8914         bnxt_free_mem(bp, false);
8915 }
8916
8917 static int bnxt_open(struct net_device *dev)
8918 {
8919         struct bnxt *bp = netdev_priv(dev);
8920         int rc;
8921
8922         bnxt_hwrm_if_change(bp, true);
8923         rc = __bnxt_open_nic(bp, true, true);
8924         if (rc)
8925                 bnxt_hwrm_if_change(bp, false);
8926
8927         bnxt_hwmon_open(bp);
8928
8929         return rc;
8930 }
8931
8932 static bool bnxt_drv_busy(struct bnxt *bp)
8933 {
8934         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
8935                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
8936 }
8937
8938 static void bnxt_get_ring_stats(struct bnxt *bp,
8939                                 struct rtnl_link_stats64 *stats);
8940
8941 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
8942                              bool link_re_init)
8943 {
8944         /* Close the VF-reps before closing PF */
8945         if (BNXT_PF(bp))
8946                 bnxt_vf_reps_close(bp);
8947
8948         /* Change device state to avoid TX queue wake up's */
8949         bnxt_tx_disable(bp);
8950
8951         clear_bit(BNXT_STATE_OPEN, &bp->state);
8952         smp_mb__after_atomic();
8953         while (bnxt_drv_busy(bp))
8954                 msleep(20);
8955
8956         /* Flush rings and and disable interrupts */
8957         bnxt_shutdown_nic(bp, irq_re_init);
8958
8959         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
8960
8961         bnxt_debug_dev_exit(bp);
8962         bnxt_disable_napi(bp);
8963         del_timer_sync(&bp->timer);
8964         bnxt_free_skbs(bp);
8965
8966         /* Save ring stats before shutdown */
8967         if (bp->bnapi)
8968                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
8969         if (irq_re_init) {
8970                 bnxt_free_irq(bp);
8971                 bnxt_del_napi(bp);
8972         }
8973         bnxt_free_mem(bp, irq_re_init);
8974 }
8975
8976 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8977 {
8978         int rc = 0;
8979
8980 #ifdef CONFIG_BNXT_SRIOV
8981         if (bp->sriov_cfg) {
8982                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
8983                                                       !bp->sriov_cfg,
8984                                                       BNXT_SRIOV_CFG_WAIT_TMO);
8985                 if (rc)
8986                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
8987         }
8988 #endif
8989         __bnxt_close_nic(bp, irq_re_init, link_re_init);
8990         return rc;
8991 }
8992
8993 static int bnxt_close(struct net_device *dev)
8994 {
8995         struct bnxt *bp = netdev_priv(dev);
8996
8997         bnxt_hwmon_close(bp);
8998         bnxt_close_nic(bp, true, true);
8999         bnxt_hwrm_shutdown_link(bp);
9000         bnxt_hwrm_if_change(bp, false);
9001         return 0;
9002 }
9003
9004 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9005                                    u16 *val)
9006 {
9007         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9008         struct hwrm_port_phy_mdio_read_input req = {0};
9009         int rc;
9010
9011         if (bp->hwrm_spec_code < 0x10a00)
9012                 return -EOPNOTSUPP;
9013
9014         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9015         req.port_id = cpu_to_le16(bp->pf.port_id);
9016         req.phy_addr = phy_addr;
9017         req.reg_addr = cpu_to_le16(reg & 0x1f);
9018         if (mdio_phy_id_is_c45(phy_addr)) {
9019                 req.cl45_mdio = 1;
9020                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9021                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9022                 req.reg_addr = cpu_to_le16(reg);
9023         }
9024
9025         mutex_lock(&bp->hwrm_cmd_lock);
9026         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9027         if (!rc)
9028                 *val = le16_to_cpu(resp->reg_data);
9029         mutex_unlock(&bp->hwrm_cmd_lock);
9030         return rc;
9031 }
9032
9033 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9034                                     u16 val)
9035 {
9036         struct hwrm_port_phy_mdio_write_input req = {0};
9037
9038         if (bp->hwrm_spec_code < 0x10a00)
9039                 return -EOPNOTSUPP;
9040
9041         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9042         req.port_id = cpu_to_le16(bp->pf.port_id);
9043         req.phy_addr = phy_addr;
9044         req.reg_addr = cpu_to_le16(reg & 0x1f);
9045         if (mdio_phy_id_is_c45(phy_addr)) {
9046                 req.cl45_mdio = 1;
9047                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9048                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9049                 req.reg_addr = cpu_to_le16(reg);
9050         }
9051         req.reg_data = cpu_to_le16(val);
9052
9053         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9054 }
9055
9056 /* rtnl_lock held */
9057 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9058 {
9059         struct mii_ioctl_data *mdio = if_mii(ifr);
9060         struct bnxt *bp = netdev_priv(dev);
9061         int rc;
9062
9063         switch (cmd) {
9064         case SIOCGMIIPHY:
9065                 mdio->phy_id = bp->link_info.phy_addr;
9066
9067                 /* fallthru */
9068         case SIOCGMIIREG: {
9069                 u16 mii_regval = 0;
9070
9071                 if (!netif_running(dev))
9072                         return -EAGAIN;
9073
9074                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9075                                              &mii_regval);
9076                 mdio->val_out = mii_regval;
9077                 return rc;
9078         }
9079
9080         case SIOCSMIIREG:
9081                 if (!netif_running(dev))
9082                         return -EAGAIN;
9083
9084                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9085                                                 mdio->val_in);
9086
9087         default:
9088                 /* do nothing */
9089                 break;
9090         }
9091         return -EOPNOTSUPP;
9092 }
9093
9094 static void bnxt_get_ring_stats(struct bnxt *bp,
9095                                 struct rtnl_link_stats64 *stats)
9096 {
9097         int i;
9098
9099
9100         for (i = 0; i < bp->cp_nr_rings; i++) {
9101                 struct bnxt_napi *bnapi = bp->bnapi[i];
9102                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9103                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
9104
9105                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
9106                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
9107                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
9108
9109                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
9110                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
9111                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
9112
9113                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
9114                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
9115                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
9116
9117                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
9118                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
9119                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
9120
9121                 stats->rx_missed_errors +=
9122                         le64_to_cpu(hw_stats->rx_discard_pkts);
9123
9124                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
9125
9126                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
9127         }
9128 }
9129
9130 static void bnxt_add_prev_stats(struct bnxt *bp,
9131                                 struct rtnl_link_stats64 *stats)
9132 {
9133         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9134
9135         stats->rx_packets += prev_stats->rx_packets;
9136         stats->tx_packets += prev_stats->tx_packets;
9137         stats->rx_bytes += prev_stats->rx_bytes;
9138         stats->tx_bytes += prev_stats->tx_bytes;
9139         stats->rx_missed_errors += prev_stats->rx_missed_errors;
9140         stats->multicast += prev_stats->multicast;
9141         stats->tx_dropped += prev_stats->tx_dropped;
9142 }
9143
9144 static void
9145 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9146 {
9147         struct bnxt *bp = netdev_priv(dev);
9148
9149         set_bit(BNXT_STATE_READ_STATS, &bp->state);
9150         /* Make sure bnxt_close_nic() sees that we are reading stats before
9151          * we check the BNXT_STATE_OPEN flag.
9152          */
9153         smp_mb__after_atomic();
9154         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9155                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9156                 *stats = bp->net_stats_prev;
9157                 return;
9158         }
9159
9160         bnxt_get_ring_stats(bp, stats);
9161         bnxt_add_prev_stats(bp, stats);
9162
9163         if (bp->flags & BNXT_FLAG_PORT_STATS) {
9164                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
9165                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
9166
9167                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
9168                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
9169                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
9170                                           le64_to_cpu(rx->rx_ovrsz_frames) +
9171                                           le64_to_cpu(rx->rx_runt_frames);
9172                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
9173                                    le64_to_cpu(rx->rx_jbr_frames);
9174                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
9175                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
9176                 stats->tx_errors = le64_to_cpu(tx->tx_err);
9177         }
9178         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9179 }
9180
9181 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9182 {
9183         struct net_device *dev = bp->dev;
9184         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9185         struct netdev_hw_addr *ha;
9186         u8 *haddr;
9187         int mc_count = 0;
9188         bool update = false;
9189         int off = 0;
9190
9191         netdev_for_each_mc_addr(ha, dev) {
9192                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9193                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9194                         vnic->mc_list_count = 0;
9195                         return false;
9196                 }
9197                 haddr = ha->addr;
9198                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9199                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9200                         update = true;
9201                 }
9202                 off += ETH_ALEN;
9203                 mc_count++;
9204         }
9205         if (mc_count)
9206                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9207
9208         if (mc_count != vnic->mc_list_count) {
9209                 vnic->mc_list_count = mc_count;
9210                 update = true;
9211         }
9212         return update;
9213 }
9214
9215 static bool bnxt_uc_list_updated(struct bnxt *bp)
9216 {
9217         struct net_device *dev = bp->dev;
9218         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9219         struct netdev_hw_addr *ha;
9220         int off = 0;
9221
9222         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9223                 return true;
9224
9225         netdev_for_each_uc_addr(ha, dev) {
9226                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9227                         return true;
9228
9229                 off += ETH_ALEN;
9230         }
9231         return false;
9232 }
9233
9234 static void bnxt_set_rx_mode(struct net_device *dev)
9235 {
9236         struct bnxt *bp = netdev_priv(dev);
9237         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9238         u32 mask = vnic->rx_mask;
9239         bool mc_update = false;
9240         bool uc_update;
9241
9242         if (!netif_running(dev))
9243                 return;
9244
9245         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9246                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
9247                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9248                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
9249
9250         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
9251                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9252
9253         uc_update = bnxt_uc_list_updated(bp);
9254
9255         if (dev->flags & IFF_BROADCAST)
9256                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9257         if (dev->flags & IFF_ALLMULTI) {
9258                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9259                 vnic->mc_list_count = 0;
9260         } else {
9261                 mc_update = bnxt_mc_list_updated(bp, &mask);
9262         }
9263
9264         if (mask != vnic->rx_mask || uc_update || mc_update) {
9265                 vnic->rx_mask = mask;
9266
9267                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
9268                 bnxt_queue_sp_work(bp);
9269         }
9270 }
9271
9272 static int bnxt_cfg_rx_mode(struct bnxt *bp)
9273 {
9274         struct net_device *dev = bp->dev;
9275         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9276         struct netdev_hw_addr *ha;
9277         int i, off = 0, rc;
9278         bool uc_update;
9279
9280         netif_addr_lock_bh(dev);
9281         uc_update = bnxt_uc_list_updated(bp);
9282         netif_addr_unlock_bh(dev);
9283
9284         if (!uc_update)
9285                 goto skip_uc;
9286
9287         mutex_lock(&bp->hwrm_cmd_lock);
9288         for (i = 1; i < vnic->uc_filter_count; i++) {
9289                 struct hwrm_cfa_l2_filter_free_input req = {0};
9290
9291                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
9292                                        -1);
9293
9294                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
9295
9296                 rc = _hwrm_send_message(bp, &req, sizeof(req),
9297                                         HWRM_CMD_TIMEOUT);
9298         }
9299         mutex_unlock(&bp->hwrm_cmd_lock);
9300
9301         vnic->uc_filter_count = 1;
9302
9303         netif_addr_lock_bh(dev);
9304         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
9305                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9306         } else {
9307                 netdev_for_each_uc_addr(ha, dev) {
9308                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
9309                         off += ETH_ALEN;
9310                         vnic->uc_filter_count++;
9311                 }
9312         }
9313         netif_addr_unlock_bh(dev);
9314
9315         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
9316                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
9317                 if (rc) {
9318                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
9319                                    rc);
9320                         vnic->uc_filter_count = i;
9321                         return rc;
9322                 }
9323         }
9324
9325 skip_uc:
9326         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9327         if (rc && vnic->mc_list_count) {
9328                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
9329                             rc);
9330                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9331                 vnic->mc_list_count = 0;
9332                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9333         }
9334         if (rc)
9335                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
9336                            rc);
9337
9338         return rc;
9339 }
9340
9341 static bool bnxt_can_reserve_rings(struct bnxt *bp)
9342 {
9343 #ifdef CONFIG_BNXT_SRIOV
9344         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
9345                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9346
9347                 /* No minimum rings were provisioned by the PF.  Don't
9348                  * reserve rings by default when device is down.
9349                  */
9350                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
9351                         return true;
9352
9353                 if (!netif_running(bp->dev))
9354                         return false;
9355         }
9356 #endif
9357         return true;
9358 }
9359
9360 /* If the chip and firmware supports RFS */
9361 static bool bnxt_rfs_supported(struct bnxt *bp)
9362 {
9363         if (bp->flags & BNXT_FLAG_CHIP_P5) {
9364                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX)
9365                         return true;
9366                 return false;
9367         }
9368         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
9369                 return true;
9370         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9371                 return true;
9372         return false;
9373 }
9374
9375 /* If runtime conditions support RFS */
9376 static bool bnxt_rfs_capable(struct bnxt *bp)
9377 {
9378 #ifdef CONFIG_RFS_ACCEL
9379         int vnics, max_vnics, max_rss_ctxs;
9380
9381         if (bp->flags & BNXT_FLAG_CHIP_P5)
9382                 return bnxt_rfs_supported(bp);
9383         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
9384                 return false;
9385
9386         vnics = 1 + bp->rx_nr_rings;
9387         max_vnics = bnxt_get_max_func_vnics(bp);
9388         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
9389
9390         /* RSS contexts not a limiting factor */
9391         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9392                 max_rss_ctxs = max_vnics;
9393         if (vnics > max_vnics || vnics > max_rss_ctxs) {
9394                 if (bp->rx_nr_rings > 1)
9395                         netdev_warn(bp->dev,
9396                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
9397                                     min(max_rss_ctxs - 1, max_vnics - 1));
9398                 return false;
9399         }
9400
9401         if (!BNXT_NEW_RM(bp))
9402                 return true;
9403
9404         if (vnics == bp->hw_resc.resv_vnics)
9405                 return true;
9406
9407         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
9408         if (vnics <= bp->hw_resc.resv_vnics)
9409                 return true;
9410
9411         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
9412         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
9413         return false;
9414 #else
9415         return false;
9416 #endif
9417 }
9418
9419 static netdev_features_t bnxt_fix_features(struct net_device *dev,
9420                                            netdev_features_t features)
9421 {
9422         struct bnxt *bp = netdev_priv(dev);
9423
9424         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
9425                 features &= ~NETIF_F_NTUPLE;
9426
9427         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9428                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9429
9430         if (!(features & NETIF_F_GRO))
9431                 features &= ~NETIF_F_GRO_HW;
9432
9433         if (features & NETIF_F_GRO_HW)
9434                 features &= ~NETIF_F_LRO;
9435
9436         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
9437          * turned on or off together.
9438          */
9439         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
9440             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
9441                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
9442                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9443                                       NETIF_F_HW_VLAN_STAG_RX);
9444                 else
9445                         features |= NETIF_F_HW_VLAN_CTAG_RX |
9446                                     NETIF_F_HW_VLAN_STAG_RX;
9447         }
9448 #ifdef CONFIG_BNXT_SRIOV
9449         if (BNXT_VF(bp)) {
9450                 if (bp->vf.vlan) {
9451                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9452                                       NETIF_F_HW_VLAN_STAG_RX);
9453                 }
9454         }
9455 #endif
9456         return features;
9457 }
9458
9459 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
9460 {
9461         struct bnxt *bp = netdev_priv(dev);
9462         u32 flags = bp->flags;
9463         u32 changes;
9464         int rc = 0;
9465         bool re_init = false;
9466         bool update_tpa = false;
9467
9468         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
9469         if (features & NETIF_F_GRO_HW)
9470                 flags |= BNXT_FLAG_GRO;
9471         else if (features & NETIF_F_LRO)
9472                 flags |= BNXT_FLAG_LRO;
9473
9474         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9475                 flags &= ~BNXT_FLAG_TPA;
9476
9477         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9478                 flags |= BNXT_FLAG_STRIP_VLAN;
9479
9480         if (features & NETIF_F_NTUPLE)
9481                 flags |= BNXT_FLAG_RFS;
9482
9483         changes = flags ^ bp->flags;
9484         if (changes & BNXT_FLAG_TPA) {
9485                 update_tpa = true;
9486                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
9487                     (flags & BNXT_FLAG_TPA) == 0 ||
9488                     (bp->flags & BNXT_FLAG_CHIP_P5))
9489                         re_init = true;
9490         }
9491
9492         if (changes & ~BNXT_FLAG_TPA)
9493                 re_init = true;
9494
9495         if (flags != bp->flags) {
9496                 u32 old_flags = bp->flags;
9497
9498                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9499                         bp->flags = flags;
9500                         if (update_tpa)
9501                                 bnxt_set_ring_params(bp);
9502                         return rc;
9503                 }
9504
9505                 if (re_init) {
9506                         bnxt_close_nic(bp, false, false);
9507                         bp->flags = flags;
9508                         if (update_tpa)
9509                                 bnxt_set_ring_params(bp);
9510
9511                         return bnxt_open_nic(bp, false, false);
9512                 }
9513                 if (update_tpa) {
9514                         bp->flags = flags;
9515                         rc = bnxt_set_tpa(bp,
9516                                           (flags & BNXT_FLAG_TPA) ?
9517                                           true : false);
9518                         if (rc)
9519                                 bp->flags = old_flags;
9520                 }
9521         }
9522         return rc;
9523 }
9524
9525 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9526                                        u32 ring_id, u32 *prod, u32 *cons)
9527 {
9528         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9529         struct hwrm_dbg_ring_info_get_input req = {0};
9530         int rc;
9531
9532         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9533         req.ring_type = ring_type;
9534         req.fw_ring_id = cpu_to_le32(ring_id);
9535         mutex_lock(&bp->hwrm_cmd_lock);
9536         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9537         if (!rc) {
9538                 *prod = le32_to_cpu(resp->producer_index);
9539                 *cons = le32_to_cpu(resp->consumer_index);
9540         }
9541         mutex_unlock(&bp->hwrm_cmd_lock);
9542         return rc;
9543 }
9544
9545 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9546 {
9547         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9548         int i = bnapi->index;
9549
9550         if (!txr)
9551                 return;
9552
9553         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9554                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9555                     txr->tx_cons);
9556 }
9557
9558 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9559 {
9560         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9561         int i = bnapi->index;
9562
9563         if (!rxr)
9564                 return;
9565
9566         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9567                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9568                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9569                     rxr->rx_sw_agg_prod);
9570 }
9571
9572 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9573 {
9574         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9575         int i = bnapi->index;
9576
9577         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9578                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9579 }
9580
9581 static void bnxt_dbg_dump_states(struct bnxt *bp)
9582 {
9583         int i;
9584         struct bnxt_napi *bnapi;
9585
9586         for (i = 0; i < bp->cp_nr_rings; i++) {
9587                 bnapi = bp->bnapi[i];
9588                 if (netif_msg_drv(bp)) {
9589                         bnxt_dump_tx_sw_state(bnapi);
9590                         bnxt_dump_rx_sw_state(bnapi);
9591                         bnxt_dump_cp_sw_state(bnapi);
9592                 }
9593         }
9594 }
9595
9596 static void bnxt_reset_task(struct bnxt *bp, bool silent)
9597 {
9598         if (!silent)
9599                 bnxt_dbg_dump_states(bp);
9600         if (netif_running(bp->dev)) {
9601                 int rc;
9602
9603                 if (!silent)
9604                         bnxt_ulp_stop(bp);
9605                 bnxt_close_nic(bp, false, false);
9606                 rc = bnxt_open_nic(bp, false, false);
9607                 if (!silent && !rc)
9608                         bnxt_ulp_start(bp);
9609         }
9610 }
9611
9612 static void bnxt_tx_timeout(struct net_device *dev)
9613 {
9614         struct bnxt *bp = netdev_priv(dev);
9615
9616         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
9617         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
9618         bnxt_queue_sp_work(bp);
9619 }
9620
9621 static void bnxt_timer(struct timer_list *t)
9622 {
9623         struct bnxt *bp = from_timer(bp, t, timer);
9624         struct net_device *dev = bp->dev;
9625
9626         if (!netif_running(dev))
9627                 return;
9628
9629         if (atomic_read(&bp->intr_sem) != 0)
9630                 goto bnxt_restart_timer;
9631
9632         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
9633             bp->stats_coal_ticks) {
9634                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
9635                 bnxt_queue_sp_work(bp);
9636         }
9637
9638         if (bnxt_tc_flower_enabled(bp)) {
9639                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
9640                 bnxt_queue_sp_work(bp);
9641         }
9642
9643         if (bp->link_info.phy_retry) {
9644                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
9645                         bp->link_info.phy_retry = 0;
9646                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
9647                 } else {
9648                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
9649                         bnxt_queue_sp_work(bp);
9650                 }
9651         }
9652
9653         if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
9654                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
9655                 bnxt_queue_sp_work(bp);
9656         }
9657 bnxt_restart_timer:
9658         mod_timer(&bp->timer, jiffies + bp->current_interval);
9659 }
9660
9661 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
9662 {
9663         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
9664          * set.  If the device is being closed, bnxt_close() may be holding
9665          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
9666          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
9667          */
9668         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9669         rtnl_lock();
9670 }
9671
9672 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
9673 {
9674         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9675         rtnl_unlock();
9676 }
9677
9678 /* Only called from bnxt_sp_task() */
9679 static void bnxt_reset(struct bnxt *bp, bool silent)
9680 {
9681         bnxt_rtnl_lock_sp(bp);
9682         if (test_bit(BNXT_STATE_OPEN, &bp->state))
9683                 bnxt_reset_task(bp, silent);
9684         bnxt_rtnl_unlock_sp(bp);
9685 }
9686
9687 static void bnxt_chk_missed_irq(struct bnxt *bp)
9688 {
9689         int i;
9690
9691         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9692                 return;
9693
9694         for (i = 0; i < bp->cp_nr_rings; i++) {
9695                 struct bnxt_napi *bnapi = bp->bnapi[i];
9696                 struct bnxt_cp_ring_info *cpr;
9697                 u32 fw_ring_id;
9698                 int j;
9699
9700                 if (!bnapi)
9701                         continue;
9702
9703                 cpr = &bnapi->cp_ring;
9704                 for (j = 0; j < 2; j++) {
9705                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
9706                         u32 val[2];
9707
9708                         if (!cpr2 || cpr2->has_more_work ||
9709                             !bnxt_has_work(bp, cpr2))
9710                                 continue;
9711
9712                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
9713                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
9714                                 continue;
9715                         }
9716                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
9717                         bnxt_dbg_hwrm_ring_info_get(bp,
9718                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
9719                                 fw_ring_id, &val[0], &val[1]);
9720                         cpr->missed_irqs++;
9721                 }
9722         }
9723 }
9724
9725 static void bnxt_cfg_ntp_filters(struct bnxt *);
9726
9727 static void bnxt_sp_task(struct work_struct *work)
9728 {
9729         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
9730
9731         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9732         smp_mb__after_atomic();
9733         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9734                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9735                 return;
9736         }
9737
9738         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
9739                 bnxt_cfg_rx_mode(bp);
9740
9741         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
9742                 bnxt_cfg_ntp_filters(bp);
9743         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
9744                 bnxt_hwrm_exec_fwd_req(bp);
9745         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9746                 bnxt_hwrm_tunnel_dst_port_alloc(
9747                         bp, bp->vxlan_port,
9748                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9749         }
9750         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9751                 bnxt_hwrm_tunnel_dst_port_free(
9752                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9753         }
9754         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
9755                 bnxt_hwrm_tunnel_dst_port_alloc(
9756                         bp, bp->nge_port,
9757                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9758         }
9759         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
9760                 bnxt_hwrm_tunnel_dst_port_free(
9761                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9762         }
9763         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
9764                 bnxt_hwrm_port_qstats(bp);
9765                 bnxt_hwrm_port_qstats_ext(bp);
9766                 bnxt_hwrm_pcie_qstats(bp);
9767         }
9768
9769         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
9770                 int rc;
9771
9772                 mutex_lock(&bp->link_lock);
9773                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
9774                                        &bp->sp_event))
9775                         bnxt_hwrm_phy_qcaps(bp);
9776
9777                 rc = bnxt_update_link(bp, true);
9778                 mutex_unlock(&bp->link_lock);
9779                 if (rc)
9780                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
9781                                    rc);
9782         }
9783         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
9784                 int rc;
9785
9786                 mutex_lock(&bp->link_lock);
9787                 rc = bnxt_update_phy_setting(bp);
9788                 mutex_unlock(&bp->link_lock);
9789                 if (rc) {
9790                         netdev_warn(bp->dev, "update phy settings retry failed\n");
9791                 } else {
9792                         bp->link_info.phy_retry = false;
9793                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
9794                 }
9795         }
9796         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
9797                 mutex_lock(&bp->link_lock);
9798                 bnxt_get_port_module_status(bp);
9799                 mutex_unlock(&bp->link_lock);
9800         }
9801
9802         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
9803                 bnxt_tc_flow_stats_work(bp);
9804
9805         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
9806                 bnxt_chk_missed_irq(bp);
9807
9808         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
9809          * must be the last functions to be called before exiting.
9810          */
9811         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
9812                 bnxt_reset(bp, false);
9813
9814         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
9815                 bnxt_reset(bp, true);
9816
9817         smp_mb__before_atomic();
9818         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9819 }
9820
9821 /* Under rtnl_lock */
9822 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
9823                      int tx_xdp)
9824 {
9825         int max_rx, max_tx, tx_sets = 1;
9826         int tx_rings_needed, stats;
9827         int rx_rings = rx;
9828         int cp, vnics, rc;
9829
9830         if (tcs)
9831                 tx_sets = tcs;
9832
9833         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
9834         if (rc)
9835                 return rc;
9836
9837         if (max_rx < rx)
9838                 return -ENOMEM;
9839
9840         tx_rings_needed = tx * tx_sets + tx_xdp;
9841         if (max_tx < tx_rings_needed)
9842                 return -ENOMEM;
9843
9844         vnics = 1;
9845         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
9846                 vnics += rx_rings;
9847
9848         if (bp->flags & BNXT_FLAG_AGG_RINGS)
9849                 rx_rings <<= 1;
9850         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
9851         stats = cp;
9852         if (BNXT_NEW_RM(bp)) {
9853                 cp += bnxt_get_ulp_msix_num(bp);
9854                 stats += bnxt_get_ulp_stat_ctxs(bp);
9855         }
9856         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
9857                                      stats, vnics);
9858 }
9859
9860 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
9861 {
9862         if (bp->bar2) {
9863                 pci_iounmap(pdev, bp->bar2);
9864                 bp->bar2 = NULL;
9865         }
9866
9867         if (bp->bar1) {
9868                 pci_iounmap(pdev, bp->bar1);
9869                 bp->bar1 = NULL;
9870         }
9871
9872         if (bp->bar0) {
9873                 pci_iounmap(pdev, bp->bar0);
9874                 bp->bar0 = NULL;
9875         }
9876 }
9877
9878 static void bnxt_cleanup_pci(struct bnxt *bp)
9879 {
9880         bnxt_unmap_bars(bp, bp->pdev);
9881         pci_release_regions(bp->pdev);
9882         pci_disable_device(bp->pdev);
9883 }
9884
9885 static void bnxt_init_dflt_coal(struct bnxt *bp)
9886 {
9887         struct bnxt_coal *coal;
9888
9889         /* Tick values in micro seconds.
9890          * 1 coal_buf x bufs_per_record = 1 completion record.
9891          */
9892         coal = &bp->rx_coal;
9893         coal->coal_ticks = 10;
9894         coal->coal_bufs = 30;
9895         coal->coal_ticks_irq = 1;
9896         coal->coal_bufs_irq = 2;
9897         coal->idle_thresh = 50;
9898         coal->bufs_per_record = 2;
9899         coal->budget = 64;              /* NAPI budget */
9900
9901         coal = &bp->tx_coal;
9902         coal->coal_ticks = 28;
9903         coal->coal_bufs = 30;
9904         coal->coal_ticks_irq = 2;
9905         coal->coal_bufs_irq = 2;
9906         coal->bufs_per_record = 1;
9907
9908         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
9909 }
9910
9911 static int bnxt_fw_init_one_p1(struct bnxt *bp)
9912 {
9913         int rc;
9914
9915         bp->fw_cap = 0;
9916         rc = bnxt_hwrm_ver_get(bp);
9917         if (rc)
9918                 return rc;
9919
9920         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
9921                 rc = bnxt_alloc_kong_hwrm_resources(bp);
9922                 if (rc)
9923                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
9924         }
9925
9926         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
9927             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
9928                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
9929                 if (rc)
9930                         return rc;
9931         }
9932         rc = bnxt_hwrm_func_reset(bp);
9933         if (rc)
9934                 return -ENODEV;
9935
9936         bnxt_hwrm_fw_set_time(bp);
9937         return 0;
9938 }
9939
9940 static int bnxt_fw_init_one_p2(struct bnxt *bp)
9941 {
9942         int rc;
9943
9944         /* Get the MAX capabilities for this function */
9945         rc = bnxt_hwrm_func_qcaps(bp);
9946         if (rc) {
9947                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
9948                            rc);
9949                 return -ENODEV;
9950         }
9951
9952         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
9953         if (rc)
9954                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
9955                             rc);
9956
9957         rc = bnxt_hwrm_func_drv_rgtr(bp);
9958         if (rc)
9959                 return -ENODEV;
9960
9961         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
9962         if (rc)
9963                 return -ENODEV;
9964
9965         bnxt_hwrm_func_qcfg(bp);
9966         bnxt_hwrm_vnic_qcaps(bp);
9967         bnxt_hwrm_port_led_qcaps(bp);
9968         bnxt_ethtool_init(bp);
9969         bnxt_dcb_init(bp);
9970         return 0;
9971 }
9972
9973 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
9974 {
9975         int rc;
9976         struct bnxt *bp = netdev_priv(dev);
9977
9978         SET_NETDEV_DEV(dev, &pdev->dev);
9979
9980         /* enable device (incl. PCI PM wakeup), and bus-mastering */
9981         rc = pci_enable_device(pdev);
9982         if (rc) {
9983                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
9984                 goto init_err;
9985         }
9986
9987         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
9988                 dev_err(&pdev->dev,
9989                         "Cannot find PCI device base address, aborting\n");
9990                 rc = -ENODEV;
9991                 goto init_err_disable;
9992         }
9993
9994         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
9995         if (rc) {
9996                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
9997                 goto init_err_disable;
9998         }
9999
10000         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
10001             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
10002                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
10003                 goto init_err_disable;
10004         }
10005
10006         pci_set_master(pdev);
10007
10008         bp->dev = dev;
10009         bp->pdev = pdev;
10010
10011         bp->bar0 = pci_ioremap_bar(pdev, 0);
10012         if (!bp->bar0) {
10013                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
10014                 rc = -ENOMEM;
10015                 goto init_err_release;
10016         }
10017
10018         bp->bar1 = pci_ioremap_bar(pdev, 2);
10019         if (!bp->bar1) {
10020                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
10021                 rc = -ENOMEM;
10022                 goto init_err_release;
10023         }
10024
10025         bp->bar2 = pci_ioremap_bar(pdev, 4);
10026         if (!bp->bar2) {
10027                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
10028                 rc = -ENOMEM;
10029                 goto init_err_release;
10030         }
10031
10032         pci_enable_pcie_error_reporting(pdev);
10033
10034         INIT_WORK(&bp->sp_task, bnxt_sp_task);
10035
10036         spin_lock_init(&bp->ntp_fltr_lock);
10037 #if BITS_PER_LONG == 32
10038         spin_lock_init(&bp->db_lock);
10039 #endif
10040
10041         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
10042         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
10043
10044         bnxt_init_dflt_coal(bp);
10045
10046         timer_setup(&bp->timer, bnxt_timer, 0);
10047         bp->current_interval = BNXT_TIMER_INTERVAL;
10048
10049         clear_bit(BNXT_STATE_OPEN, &bp->state);
10050         return 0;
10051
10052 init_err_release:
10053         bnxt_unmap_bars(bp, pdev);
10054         pci_release_regions(pdev);
10055
10056 init_err_disable:
10057         pci_disable_device(pdev);
10058
10059 init_err:
10060         return rc;
10061 }
10062
10063 /* rtnl_lock held */
10064 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
10065 {
10066         struct sockaddr *addr = p;
10067         struct bnxt *bp = netdev_priv(dev);
10068         int rc = 0;
10069
10070         if (!is_valid_ether_addr(addr->sa_data))
10071                 return -EADDRNOTAVAIL;
10072
10073         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
10074                 return 0;
10075
10076         rc = bnxt_approve_mac(bp, addr->sa_data, true);
10077         if (rc)
10078                 return rc;
10079
10080         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
10081         if (netif_running(dev)) {
10082                 bnxt_close_nic(bp, false, false);
10083                 rc = bnxt_open_nic(bp, false, false);
10084         }
10085
10086         return rc;
10087 }
10088
10089 /* rtnl_lock held */
10090 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
10091 {
10092         struct bnxt *bp = netdev_priv(dev);
10093
10094         if (netif_running(dev))
10095                 bnxt_close_nic(bp, false, false);
10096
10097         dev->mtu = new_mtu;
10098         bnxt_set_ring_params(bp);
10099
10100         if (netif_running(dev))
10101                 return bnxt_open_nic(bp, false, false);
10102
10103         return 0;
10104 }
10105
10106 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
10107 {
10108         struct bnxt *bp = netdev_priv(dev);
10109         bool sh = false;
10110         int rc;
10111
10112         if (tc > bp->max_tc) {
10113                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
10114                            tc, bp->max_tc);
10115                 return -EINVAL;
10116         }
10117
10118         if (netdev_get_num_tc(dev) == tc)
10119                 return 0;
10120
10121         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10122                 sh = true;
10123
10124         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
10125                               sh, tc, bp->tx_nr_rings_xdp);
10126         if (rc)
10127                 return rc;
10128
10129         /* Needs to close the device and do hw resource re-allocations */
10130         if (netif_running(bp->dev))
10131                 bnxt_close_nic(bp, true, false);
10132
10133         if (tc) {
10134                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
10135                 netdev_set_num_tc(dev, tc);
10136         } else {
10137                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10138                 netdev_reset_tc(dev);
10139         }
10140         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
10141         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
10142                                bp->tx_nr_rings + bp->rx_nr_rings;
10143
10144         if (netif_running(bp->dev))
10145                 return bnxt_open_nic(bp, true, false);
10146
10147         return 0;
10148 }
10149
10150 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10151                                   void *cb_priv)
10152 {
10153         struct bnxt *bp = cb_priv;
10154
10155         if (!bnxt_tc_flower_enabled(bp) ||
10156             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
10157                 return -EOPNOTSUPP;
10158
10159         switch (type) {
10160         case TC_SETUP_CLSFLOWER:
10161                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
10162         default:
10163                 return -EOPNOTSUPP;
10164         }
10165 }
10166
10167 static LIST_HEAD(bnxt_block_cb_list);
10168
10169 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
10170                          void *type_data)
10171 {
10172         struct bnxt *bp = netdev_priv(dev);
10173
10174         switch (type) {
10175         case TC_SETUP_BLOCK:
10176                 return flow_block_cb_setup_simple(type_data,
10177                                                   &bnxt_block_cb_list,
10178                                                   bnxt_setup_tc_block_cb,
10179                                                   bp, bp, true);
10180         case TC_SETUP_QDISC_MQPRIO: {
10181                 struct tc_mqprio_qopt *mqprio = type_data;
10182
10183                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
10184
10185                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
10186         }
10187         default:
10188                 return -EOPNOTSUPP;
10189         }
10190 }
10191
10192 #ifdef CONFIG_RFS_ACCEL
10193 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
10194                             struct bnxt_ntuple_filter *f2)
10195 {
10196         struct flow_keys *keys1 = &f1->fkeys;
10197         struct flow_keys *keys2 = &f2->fkeys;
10198
10199         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
10200             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
10201             keys1->ports.ports == keys2->ports.ports &&
10202             keys1->basic.ip_proto == keys2->basic.ip_proto &&
10203             keys1->basic.n_proto == keys2->basic.n_proto &&
10204             keys1->control.flags == keys2->control.flags &&
10205             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
10206             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
10207                 return true;
10208
10209         return false;
10210 }
10211
10212 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
10213                               u16 rxq_index, u32 flow_id)
10214 {
10215         struct bnxt *bp = netdev_priv(dev);
10216         struct bnxt_ntuple_filter *fltr, *new_fltr;
10217         struct flow_keys *fkeys;
10218         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
10219         int rc = 0, idx, bit_id, l2_idx = 0;
10220         struct hlist_head *head;
10221
10222         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
10223                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10224                 int off = 0, j;
10225
10226                 netif_addr_lock_bh(dev);
10227                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
10228                         if (ether_addr_equal(eth->h_dest,
10229                                              vnic->uc_list + off)) {
10230                                 l2_idx = j + 1;
10231                                 break;
10232                         }
10233                 }
10234                 netif_addr_unlock_bh(dev);
10235                 if (!l2_idx)
10236                         return -EINVAL;
10237         }
10238         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
10239         if (!new_fltr)
10240                 return -ENOMEM;
10241
10242         fkeys = &new_fltr->fkeys;
10243         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
10244                 rc = -EPROTONOSUPPORT;
10245                 goto err_free;
10246         }
10247
10248         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
10249              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
10250             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
10251              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
10252                 rc = -EPROTONOSUPPORT;
10253                 goto err_free;
10254         }
10255         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
10256             bp->hwrm_spec_code < 0x10601) {
10257                 rc = -EPROTONOSUPPORT;
10258                 goto err_free;
10259         }
10260         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
10261             bp->hwrm_spec_code < 0x10601) {
10262                 rc = -EPROTONOSUPPORT;
10263                 goto err_free;
10264         }
10265
10266         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
10267         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
10268
10269         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
10270         head = &bp->ntp_fltr_hash_tbl[idx];
10271         rcu_read_lock();
10272         hlist_for_each_entry_rcu(fltr, head, hash) {
10273                 if (bnxt_fltr_match(fltr, new_fltr)) {
10274                         rcu_read_unlock();
10275                         rc = 0;
10276                         goto err_free;
10277                 }
10278         }
10279         rcu_read_unlock();
10280
10281         spin_lock_bh(&bp->ntp_fltr_lock);
10282         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
10283                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
10284         if (bit_id < 0) {
10285                 spin_unlock_bh(&bp->ntp_fltr_lock);
10286                 rc = -ENOMEM;
10287                 goto err_free;
10288         }
10289
10290         new_fltr->sw_id = (u16)bit_id;
10291         new_fltr->flow_id = flow_id;
10292         new_fltr->l2_fltr_idx = l2_idx;
10293         new_fltr->rxq = rxq_index;
10294         hlist_add_head_rcu(&new_fltr->hash, head);
10295         bp->ntp_fltr_count++;
10296         spin_unlock_bh(&bp->ntp_fltr_lock);
10297
10298         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10299         bnxt_queue_sp_work(bp);
10300
10301         return new_fltr->sw_id;
10302
10303 err_free:
10304         kfree(new_fltr);
10305         return rc;
10306 }
10307
10308 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10309 {
10310         int i;
10311
10312         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
10313                 struct hlist_head *head;
10314                 struct hlist_node *tmp;
10315                 struct bnxt_ntuple_filter *fltr;
10316                 int rc;
10317
10318                 head = &bp->ntp_fltr_hash_tbl[i];
10319                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
10320                         bool del = false;
10321
10322                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
10323                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
10324                                                         fltr->flow_id,
10325                                                         fltr->sw_id)) {
10326                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
10327                                                                          fltr);
10328                                         del = true;
10329                                 }
10330                         } else {
10331                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
10332                                                                        fltr);
10333                                 if (rc)
10334                                         del = true;
10335                                 else
10336                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
10337                         }
10338
10339                         if (del) {
10340                                 spin_lock_bh(&bp->ntp_fltr_lock);
10341                                 hlist_del_rcu(&fltr->hash);
10342                                 bp->ntp_fltr_count--;
10343                                 spin_unlock_bh(&bp->ntp_fltr_lock);
10344                                 synchronize_rcu();
10345                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
10346                                 kfree(fltr);
10347                         }
10348                 }
10349         }
10350         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
10351                 netdev_info(bp->dev, "Receive PF driver unload event!");
10352 }
10353
10354 #else
10355
10356 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
10357 {
10358 }
10359
10360 #endif /* CONFIG_RFS_ACCEL */
10361
10362 static void bnxt_udp_tunnel_add(struct net_device *dev,
10363                                 struct udp_tunnel_info *ti)
10364 {
10365         struct bnxt *bp = netdev_priv(dev);
10366
10367         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
10368                 return;
10369
10370         if (!netif_running(dev))
10371                 return;
10372
10373         switch (ti->type) {
10374         case UDP_TUNNEL_TYPE_VXLAN:
10375                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
10376                         return;
10377
10378                 bp->vxlan_port_cnt++;
10379                 if (bp->vxlan_port_cnt == 1) {
10380                         bp->vxlan_port = ti->port;
10381                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
10382                         bnxt_queue_sp_work(bp);
10383                 }
10384                 break;
10385         case UDP_TUNNEL_TYPE_GENEVE:
10386                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
10387                         return;
10388
10389                 bp->nge_port_cnt++;
10390                 if (bp->nge_port_cnt == 1) {
10391                         bp->nge_port = ti->port;
10392                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
10393                 }
10394                 break;
10395         default:
10396                 return;
10397         }
10398
10399         bnxt_queue_sp_work(bp);
10400 }
10401
10402 static void bnxt_udp_tunnel_del(struct net_device *dev,
10403                                 struct udp_tunnel_info *ti)
10404 {
10405         struct bnxt *bp = netdev_priv(dev);
10406
10407         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
10408                 return;
10409
10410         if (!netif_running(dev))
10411                 return;
10412
10413         switch (ti->type) {
10414         case UDP_TUNNEL_TYPE_VXLAN:
10415                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
10416                         return;
10417                 bp->vxlan_port_cnt--;
10418
10419                 if (bp->vxlan_port_cnt != 0)
10420                         return;
10421
10422                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
10423                 break;
10424         case UDP_TUNNEL_TYPE_GENEVE:
10425                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
10426                         return;
10427                 bp->nge_port_cnt--;
10428
10429                 if (bp->nge_port_cnt != 0)
10430                         return;
10431
10432                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
10433                 break;
10434         default:
10435                 return;
10436         }
10437
10438         bnxt_queue_sp_work(bp);
10439 }
10440
10441 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
10442                                struct net_device *dev, u32 filter_mask,
10443                                int nlflags)
10444 {
10445         struct bnxt *bp = netdev_priv(dev);
10446
10447         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
10448                                        nlflags, filter_mask, NULL);
10449 }
10450
10451 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
10452                                u16 flags, struct netlink_ext_ack *extack)
10453 {
10454         struct bnxt *bp = netdev_priv(dev);
10455         struct nlattr *attr, *br_spec;
10456         int rem, rc = 0;
10457
10458         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
10459                 return -EOPNOTSUPP;
10460
10461         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
10462         if (!br_spec)
10463                 return -EINVAL;
10464
10465         nla_for_each_nested(attr, br_spec, rem) {
10466                 u16 mode;
10467
10468                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
10469                         continue;
10470
10471                 if (nla_len(attr) < sizeof(mode))
10472                         return -EINVAL;
10473
10474                 mode = nla_get_u16(attr);
10475                 if (mode == bp->br_mode)
10476                         break;
10477
10478                 rc = bnxt_hwrm_set_br_mode(bp, mode);
10479                 if (!rc)
10480                         bp->br_mode = mode;
10481                 break;
10482         }
10483         return rc;
10484 }
10485
10486 int bnxt_get_port_parent_id(struct net_device *dev,
10487                             struct netdev_phys_item_id *ppid)
10488 {
10489         struct bnxt *bp = netdev_priv(dev);
10490
10491         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
10492                 return -EOPNOTSUPP;
10493
10494         /* The PF and it's VF-reps only support the switchdev framework */
10495         if (!BNXT_PF(bp))
10496                 return -EOPNOTSUPP;
10497
10498         ppid->id_len = sizeof(bp->switch_id);
10499         memcpy(ppid->id, bp->switch_id, ppid->id_len);
10500
10501         return 0;
10502 }
10503
10504 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
10505 {
10506         struct bnxt *bp = netdev_priv(dev);
10507
10508         return &bp->dl_port;
10509 }
10510
10511 static const struct net_device_ops bnxt_netdev_ops = {
10512         .ndo_open               = bnxt_open,
10513         .ndo_start_xmit         = bnxt_start_xmit,
10514         .ndo_stop               = bnxt_close,
10515         .ndo_get_stats64        = bnxt_get_stats64,
10516         .ndo_set_rx_mode        = bnxt_set_rx_mode,
10517         .ndo_do_ioctl           = bnxt_ioctl,
10518         .ndo_validate_addr      = eth_validate_addr,
10519         .ndo_set_mac_address    = bnxt_change_mac_addr,
10520         .ndo_change_mtu         = bnxt_change_mtu,
10521         .ndo_fix_features       = bnxt_fix_features,
10522         .ndo_set_features       = bnxt_set_features,
10523         .ndo_tx_timeout         = bnxt_tx_timeout,
10524 #ifdef CONFIG_BNXT_SRIOV
10525         .ndo_get_vf_config      = bnxt_get_vf_config,
10526         .ndo_set_vf_mac         = bnxt_set_vf_mac,
10527         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
10528         .ndo_set_vf_rate        = bnxt_set_vf_bw,
10529         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
10530         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
10531         .ndo_set_vf_trust       = bnxt_set_vf_trust,
10532 #endif
10533         .ndo_setup_tc           = bnxt_setup_tc,
10534 #ifdef CONFIG_RFS_ACCEL
10535         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
10536 #endif
10537         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
10538         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
10539         .ndo_bpf                = bnxt_xdp,
10540         .ndo_xdp_xmit           = bnxt_xdp_xmit,
10541         .ndo_bridge_getlink     = bnxt_bridge_getlink,
10542         .ndo_bridge_setlink     = bnxt_bridge_setlink,
10543         .ndo_get_devlink_port   = bnxt_get_devlink_port,
10544 };
10545
10546 static void bnxt_remove_one(struct pci_dev *pdev)
10547 {
10548         struct net_device *dev = pci_get_drvdata(pdev);
10549         struct bnxt *bp = netdev_priv(dev);
10550
10551         if (BNXT_PF(bp)) {
10552                 bnxt_sriov_disable(bp);
10553                 bnxt_dl_unregister(bp);
10554         }
10555
10556         pci_disable_pcie_error_reporting(pdev);
10557         unregister_netdev(dev);
10558         bnxt_shutdown_tc(bp);
10559         bnxt_cancel_sp_work(bp);
10560         bp->sp_event = 0;
10561
10562         bnxt_clear_int_mode(bp);
10563         bnxt_hwrm_func_drv_unrgtr(bp);
10564         bnxt_free_hwrm_resources(bp);
10565         bnxt_free_hwrm_short_cmd_req(bp);
10566         bnxt_ethtool_free(bp);
10567         bnxt_dcb_free(bp);
10568         kfree(bp->edev);
10569         bp->edev = NULL;
10570         bnxt_cleanup_pci(bp);
10571         bnxt_free_ctx_mem(bp);
10572         kfree(bp->ctx);
10573         bp->ctx = NULL;
10574         bnxt_free_port_stats(bp);
10575         free_netdev(dev);
10576 }
10577
10578 static int bnxt_probe_phy(struct bnxt *bp)
10579 {
10580         int rc = 0;
10581         struct bnxt_link_info *link_info = &bp->link_info;
10582
10583         rc = bnxt_hwrm_phy_qcaps(bp);
10584         if (rc) {
10585                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
10586                            rc);
10587                 return rc;
10588         }
10589         mutex_init(&bp->link_lock);
10590
10591         rc = bnxt_update_link(bp, false);
10592         if (rc) {
10593                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
10594                            rc);
10595                 return rc;
10596         }
10597
10598         /* Older firmware does not have supported_auto_speeds, so assume
10599          * that all supported speeds can be autonegotiated.
10600          */
10601         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
10602                 link_info->support_auto_speeds = link_info->support_speeds;
10603
10604         /*initialize the ethool setting copy with NVM settings */
10605         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10606                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10607                 if (bp->hwrm_spec_code >= 0x10201) {
10608                         if (link_info->auto_pause_setting &
10609                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10610                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10611                 } else {
10612                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10613                 }
10614                 link_info->advertising = link_info->auto_link_speeds;
10615         } else {
10616                 link_info->req_link_speed = link_info->force_link_speed;
10617                 link_info->req_duplex = link_info->duplex_setting;
10618         }
10619         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10620                 link_info->req_flow_ctrl =
10621                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10622         else
10623                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10624         return rc;
10625 }
10626
10627 static int bnxt_get_max_irq(struct pci_dev *pdev)
10628 {
10629         u16 ctrl;
10630
10631         if (!pdev->msix_cap)
10632                 return 1;
10633
10634         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
10635         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
10636 }
10637
10638 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10639                                 int *max_cp)
10640 {
10641         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10642         int max_ring_grps = 0, max_irq;
10643
10644         *max_tx = hw_resc->max_tx_rings;
10645         *max_rx = hw_resc->max_rx_rings;
10646         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
10647         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
10648                         bnxt_get_ulp_msix_num(bp),
10649                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
10650         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10651                 *max_cp = min_t(int, *max_cp, max_irq);
10652         max_ring_grps = hw_resc->max_hw_ring_grps;
10653         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
10654                 *max_cp -= 1;
10655                 *max_rx -= 2;
10656         }
10657         if (bp->flags & BNXT_FLAG_AGG_RINGS)
10658                 *max_rx >>= 1;
10659         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10660                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
10661                 /* On P5 chips, max_cp output param should be available NQs */
10662                 *max_cp = max_irq;
10663         }
10664         *max_rx = min_t(int, *max_rx, max_ring_grps);
10665 }
10666
10667 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
10668 {
10669         int rx, tx, cp;
10670
10671         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
10672         *max_rx = rx;
10673         *max_tx = tx;
10674         if (!rx || !tx || !cp)
10675                 return -ENOMEM;
10676
10677         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
10678 }
10679
10680 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
10681                                bool shared)
10682 {
10683         int rc;
10684
10685         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
10686         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
10687                 /* Not enough rings, try disabling agg rings. */
10688                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
10689                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
10690                 if (rc) {
10691                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
10692                         bp->flags |= BNXT_FLAG_AGG_RINGS;
10693                         return rc;
10694                 }
10695                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
10696                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10697                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10698                 bnxt_set_ring_params(bp);
10699         }
10700
10701         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
10702                 int max_cp, max_stat, max_irq;
10703
10704                 /* Reserve minimum resources for RoCE */
10705                 max_cp = bnxt_get_max_func_cp_rings(bp);
10706                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
10707                 max_irq = bnxt_get_max_func_irqs(bp);
10708                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
10709                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
10710                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
10711                         return 0;
10712
10713                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
10714                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
10715                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
10716                 max_cp = min_t(int, max_cp, max_irq);
10717                 max_cp = min_t(int, max_cp, max_stat);
10718                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
10719                 if (rc)
10720                         rc = 0;
10721         }
10722         return rc;
10723 }
10724
10725 /* In initial default shared ring setting, each shared ring must have a
10726  * RX/TX ring pair.
10727  */
10728 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
10729 {
10730         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
10731         bp->rx_nr_rings = bp->cp_nr_rings;
10732         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
10733         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10734 }
10735
10736 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
10737 {
10738         int dflt_rings, max_rx_rings, max_tx_rings, rc;
10739
10740         if (!bnxt_can_reserve_rings(bp))
10741                 return 0;
10742
10743         if (sh)
10744                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
10745         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
10746         /* Reduce default rings on multi-port cards so that total default
10747          * rings do not exceed CPU count.
10748          */
10749         if (bp->port_count > 1) {
10750                 int max_rings =
10751                         max_t(int, num_online_cpus() / bp->port_count, 1);
10752
10753                 dflt_rings = min_t(int, dflt_rings, max_rings);
10754         }
10755         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
10756         if (rc)
10757                 return rc;
10758         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
10759         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
10760         if (sh)
10761                 bnxt_trim_dflt_sh_rings(bp);
10762         else
10763                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
10764         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10765
10766         rc = __bnxt_reserve_rings(bp);
10767         if (rc)
10768                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
10769         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10770         if (sh)
10771                 bnxt_trim_dflt_sh_rings(bp);
10772
10773         /* Rings may have been trimmed, re-reserve the trimmed rings. */
10774         if (bnxt_need_reserve_rings(bp)) {
10775                 rc = __bnxt_reserve_rings(bp);
10776                 if (rc)
10777                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
10778                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10779         }
10780         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10781                 bp->rx_nr_rings++;
10782                 bp->cp_nr_rings++;
10783         }
10784         return rc;
10785 }
10786
10787 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
10788 {
10789         int rc;
10790
10791         if (bp->tx_nr_rings)
10792                 return 0;
10793
10794         bnxt_ulp_irq_stop(bp);
10795         bnxt_clear_int_mode(bp);
10796         rc = bnxt_set_dflt_rings(bp, true);
10797         if (rc) {
10798                 netdev_err(bp->dev, "Not enough rings available.\n");
10799                 goto init_dflt_ring_err;
10800         }
10801         rc = bnxt_init_int_mode(bp);
10802         if (rc)
10803                 goto init_dflt_ring_err;
10804
10805         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10806         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
10807                 bp->flags |= BNXT_FLAG_RFS;
10808                 bp->dev->features |= NETIF_F_NTUPLE;
10809         }
10810 init_dflt_ring_err:
10811         bnxt_ulp_irq_restart(bp, rc);
10812         return rc;
10813 }
10814
10815 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
10816 {
10817         int rc;
10818
10819         ASSERT_RTNL();
10820         bnxt_hwrm_func_qcaps(bp);
10821
10822         if (netif_running(bp->dev))
10823                 __bnxt_close_nic(bp, true, false);
10824
10825         bnxt_ulp_irq_stop(bp);
10826         bnxt_clear_int_mode(bp);
10827         rc = bnxt_init_int_mode(bp);
10828         bnxt_ulp_irq_restart(bp, rc);
10829
10830         if (netif_running(bp->dev)) {
10831                 if (rc)
10832                         dev_close(bp->dev);
10833                 else
10834                         rc = bnxt_open_nic(bp, true, false);
10835         }
10836
10837         return rc;
10838 }
10839
10840 static int bnxt_init_mac_addr(struct bnxt *bp)
10841 {
10842         int rc = 0;
10843
10844         if (BNXT_PF(bp)) {
10845                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
10846         } else {
10847 #ifdef CONFIG_BNXT_SRIOV
10848                 struct bnxt_vf_info *vf = &bp->vf;
10849                 bool strict_approval = true;
10850
10851                 if (is_valid_ether_addr(vf->mac_addr)) {
10852                         /* overwrite netdev dev_addr with admin VF MAC */
10853                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
10854                         /* Older PF driver or firmware may not approve this
10855                          * correctly.
10856                          */
10857                         strict_approval = false;
10858                 } else {
10859                         eth_hw_addr_random(bp->dev);
10860                 }
10861                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
10862 #endif
10863         }
10864         return rc;
10865 }
10866
10867 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
10868 {
10869         struct pci_dev *pdev = bp->pdev;
10870         int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
10871         u32 dw;
10872
10873         if (!pos) {
10874                 netdev_info(bp->dev, "Unable do read adapter's DSN");
10875                 return -EOPNOTSUPP;
10876         }
10877
10878         /* DSN (two dw) is at an offset of 4 from the cap pos */
10879         pos += 4;
10880         pci_read_config_dword(pdev, pos, &dw);
10881         put_unaligned_le32(dw, &dsn[0]);
10882         pci_read_config_dword(pdev, pos + 4, &dw);
10883         put_unaligned_le32(dw, &dsn[4]);
10884         return 0;
10885 }
10886
10887 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
10888 {
10889         static int version_printed;
10890         struct net_device *dev;
10891         struct bnxt *bp;
10892         int rc, max_irqs;
10893
10894         if (pci_is_bridge(pdev))
10895                 return -ENODEV;
10896
10897         if (version_printed++ == 0)
10898                 pr_info("%s", version);
10899
10900         max_irqs = bnxt_get_max_irq(pdev);
10901         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
10902         if (!dev)
10903                 return -ENOMEM;
10904
10905         bp = netdev_priv(dev);
10906         bnxt_set_max_func_irqs(bp, max_irqs);
10907
10908         if (bnxt_vf_pciid(ent->driver_data))
10909                 bp->flags |= BNXT_FLAG_VF;
10910
10911         if (pdev->msix_cap)
10912                 bp->flags |= BNXT_FLAG_MSIX_CAP;
10913
10914         rc = bnxt_init_board(pdev, dev);
10915         if (rc < 0)
10916                 goto init_err_free;
10917
10918         dev->netdev_ops = &bnxt_netdev_ops;
10919         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
10920         dev->ethtool_ops = &bnxt_ethtool_ops;
10921         pci_set_drvdata(pdev, dev);
10922
10923         rc = bnxt_alloc_hwrm_resources(bp);
10924         if (rc)
10925                 goto init_err_pci_clean;
10926
10927         mutex_init(&bp->hwrm_cmd_lock);
10928
10929         rc = bnxt_fw_init_one_p1(bp);
10930         if (rc)
10931                 goto init_err_pci_clean;
10932
10933         if (BNXT_CHIP_P5(bp))
10934                 bp->flags |= BNXT_FLAG_CHIP_P5;
10935
10936         rc = bnxt_fw_init_one_p2(bp);
10937         if (rc)
10938                 goto init_err_pci_clean;
10939
10940         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10941                            NETIF_F_TSO | NETIF_F_TSO6 |
10942                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10943                            NETIF_F_GSO_IPXIP4 |
10944                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10945                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
10946                            NETIF_F_RXCSUM | NETIF_F_GRO;
10947
10948         if (BNXT_SUPPORTS_TPA(bp))
10949                 dev->hw_features |= NETIF_F_LRO;
10950
10951         dev->hw_enc_features =
10952                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10953                         NETIF_F_TSO | NETIF_F_TSO6 |
10954                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10955                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10956                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
10957         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
10958                                     NETIF_F_GSO_GRE_CSUM;
10959         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
10960         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
10961                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
10962         if (BNXT_SUPPORTS_TPA(bp))
10963                 dev->hw_features |= NETIF_F_GRO_HW;
10964         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
10965         if (dev->features & NETIF_F_GRO_HW)
10966                 dev->features &= ~NETIF_F_LRO;
10967         dev->priv_flags |= IFF_UNICAST_FLT;
10968
10969 #ifdef CONFIG_BNXT_SRIOV
10970         init_waitqueue_head(&bp->sriov_cfg_wait);
10971         mutex_init(&bp->sriov_lock);
10972 #endif
10973         if (BNXT_SUPPORTS_TPA(bp)) {
10974                 bp->gro_func = bnxt_gro_func_5730x;
10975                 if (BNXT_CHIP_P4(bp))
10976                         bp->gro_func = bnxt_gro_func_5731x;
10977                 else if (BNXT_CHIP_P5(bp))
10978                         bp->gro_func = bnxt_gro_func_5750x;
10979         }
10980         if (!BNXT_CHIP_P4_PLUS(bp))
10981                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
10982
10983         bp->ulp_probe = bnxt_ulp_probe;
10984
10985         rc = bnxt_init_mac_addr(bp);
10986         if (rc) {
10987                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
10988                 rc = -EADDRNOTAVAIL;
10989                 goto init_err_pci_clean;
10990         }
10991
10992         if (BNXT_PF(bp)) {
10993                 /* Read the adapter's DSN to use as the eswitch switch_id */
10994                 rc = bnxt_pcie_dsn_get(bp, bp->switch_id);
10995                 if (rc)
10996                         goto init_err_pci_clean;
10997         }
10998
10999         /* MTU range: 60 - FW defined max */
11000         dev->min_mtu = ETH_ZLEN;
11001         dev->max_mtu = bp->max_mtu;
11002
11003         rc = bnxt_probe_phy(bp);
11004         if (rc)
11005                 goto init_err_pci_clean;
11006
11007         bnxt_set_rx_skb_mode(bp, false);
11008         bnxt_set_tpa_flags(bp);
11009         bnxt_set_ring_params(bp);
11010         rc = bnxt_set_dflt_rings(bp, true);
11011         if (rc) {
11012                 netdev_err(bp->dev, "Not enough rings available.\n");
11013                 rc = -ENOMEM;
11014                 goto init_err_pci_clean;
11015         }
11016
11017         /* Default RSS hash cfg. */
11018         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11019                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11020                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11021                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11022         if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
11023                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11024                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11025                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11026         }
11027
11028         if (bnxt_rfs_supported(bp)) {
11029                 dev->hw_features |= NETIF_F_NTUPLE;
11030                 if (bnxt_rfs_capable(bp)) {
11031                         bp->flags |= BNXT_FLAG_RFS;
11032                         dev->features |= NETIF_F_NTUPLE;
11033                 }
11034         }
11035
11036         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
11037                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
11038
11039         rc = bnxt_init_int_mode(bp);
11040         if (rc)
11041                 goto init_err_pci_clean;
11042
11043         /* No TC has been set yet and rings may have been trimmed due to
11044          * limited MSIX, so we re-initialize the TX rings per TC.
11045          */
11046         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11047
11048         bnxt_get_wol_settings(bp);
11049         if (bp->flags & BNXT_FLAG_WOL_CAP)
11050                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11051         else
11052                 device_set_wakeup_capable(&pdev->dev, false);
11053
11054         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11055
11056         bnxt_hwrm_coal_params_qcaps(bp);
11057
11058         if (BNXT_PF(bp)) {
11059                 if (!bnxt_pf_wq) {
11060                         bnxt_pf_wq =
11061                                 create_singlethread_workqueue("bnxt_pf_wq");
11062                         if (!bnxt_pf_wq) {
11063                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
11064                                 goto init_err_pci_clean;
11065                         }
11066                 }
11067                 bnxt_init_tc(bp);
11068         }
11069
11070         rc = register_netdev(dev);
11071         if (rc)
11072                 goto init_err_cleanup_tc;
11073
11074         if (BNXT_PF(bp))
11075                 bnxt_dl_register(bp);
11076
11077         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
11078                     board_info[ent->driver_data].name,
11079                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
11080         pcie_print_link_status(pdev);
11081
11082         return 0;
11083
11084 init_err_cleanup_tc:
11085         bnxt_shutdown_tc(bp);
11086         bnxt_clear_int_mode(bp);
11087
11088 init_err_pci_clean:
11089         bnxt_free_hwrm_short_cmd_req(bp);
11090         bnxt_free_hwrm_resources(bp);
11091         bnxt_free_ctx_mem(bp);
11092         kfree(bp->ctx);
11093         bp->ctx = NULL;
11094         bnxt_cleanup_pci(bp);
11095
11096 init_err_free:
11097         free_netdev(dev);
11098         return rc;
11099 }
11100
11101 static void bnxt_shutdown(struct pci_dev *pdev)
11102 {
11103         struct net_device *dev = pci_get_drvdata(pdev);
11104         struct bnxt *bp;
11105
11106         if (!dev)
11107                 return;
11108
11109         rtnl_lock();
11110         bp = netdev_priv(dev);
11111         if (!bp)
11112                 goto shutdown_exit;
11113
11114         if (netif_running(dev))
11115                 dev_close(dev);
11116
11117         bnxt_ulp_shutdown(bp);
11118
11119         if (system_state == SYSTEM_POWER_OFF) {
11120                 bnxt_clear_int_mode(bp);
11121                 pci_disable_device(pdev);
11122                 pci_wake_from_d3(pdev, bp->wol);
11123                 pci_set_power_state(pdev, PCI_D3hot);
11124         }
11125
11126 shutdown_exit:
11127         rtnl_unlock();
11128 }
11129
11130 #ifdef CONFIG_PM_SLEEP
11131 static int bnxt_suspend(struct device *device)
11132 {
11133         struct net_device *dev = dev_get_drvdata(device);
11134         struct bnxt *bp = netdev_priv(dev);
11135         int rc = 0;
11136
11137         rtnl_lock();
11138         if (netif_running(dev)) {
11139                 netif_device_detach(dev);
11140                 rc = bnxt_close(dev);
11141         }
11142         bnxt_hwrm_func_drv_unrgtr(bp);
11143         rtnl_unlock();
11144         return rc;
11145 }
11146
11147 static int bnxt_resume(struct device *device)
11148 {
11149         struct net_device *dev = dev_get_drvdata(device);
11150         struct bnxt *bp = netdev_priv(dev);
11151         int rc = 0;
11152
11153         rtnl_lock();
11154         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
11155                 rc = -ENODEV;
11156                 goto resume_exit;
11157         }
11158         rc = bnxt_hwrm_func_reset(bp);
11159         if (rc) {
11160                 rc = -EBUSY;
11161                 goto resume_exit;
11162         }
11163         bnxt_get_wol_settings(bp);
11164         if (netif_running(dev)) {
11165                 rc = bnxt_open(dev);
11166                 if (!rc)
11167                         netif_device_attach(dev);
11168         }
11169
11170 resume_exit:
11171         rtnl_unlock();
11172         return rc;
11173 }
11174
11175 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
11176 #define BNXT_PM_OPS (&bnxt_pm_ops)
11177
11178 #else
11179
11180 #define BNXT_PM_OPS NULL
11181
11182 #endif /* CONFIG_PM_SLEEP */
11183
11184 /**
11185  * bnxt_io_error_detected - called when PCI error is detected
11186  * @pdev: Pointer to PCI device
11187  * @state: The current pci connection state
11188  *
11189  * This function is called after a PCI bus error affecting
11190  * this device has been detected.
11191  */
11192 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
11193                                                pci_channel_state_t state)
11194 {
11195         struct net_device *netdev = pci_get_drvdata(pdev);
11196         struct bnxt *bp = netdev_priv(netdev);
11197
11198         netdev_info(netdev, "PCI I/O error detected\n");
11199
11200         rtnl_lock();
11201         netif_device_detach(netdev);
11202
11203         bnxt_ulp_stop(bp);
11204
11205         if (state == pci_channel_io_perm_failure) {
11206                 rtnl_unlock();
11207                 return PCI_ERS_RESULT_DISCONNECT;
11208         }
11209
11210         if (netif_running(netdev))
11211                 bnxt_close(netdev);
11212
11213         pci_disable_device(pdev);
11214         rtnl_unlock();
11215
11216         /* Request a slot slot reset. */
11217         return PCI_ERS_RESULT_NEED_RESET;
11218 }
11219
11220 /**
11221  * bnxt_io_slot_reset - called after the pci bus has been reset.
11222  * @pdev: Pointer to PCI device
11223  *
11224  * Restart the card from scratch, as if from a cold-boot.
11225  * At this point, the card has exprienced a hard reset,
11226  * followed by fixups by BIOS, and has its config space
11227  * set up identically to what it was at cold boot.
11228  */
11229 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
11230 {
11231         struct net_device *netdev = pci_get_drvdata(pdev);
11232         struct bnxt *bp = netdev_priv(netdev);
11233         int err = 0;
11234         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
11235
11236         netdev_info(bp->dev, "PCI Slot Reset\n");
11237
11238         rtnl_lock();
11239
11240         if (pci_enable_device(pdev)) {
11241                 dev_err(&pdev->dev,
11242                         "Cannot re-enable PCI device after reset.\n");
11243         } else {
11244                 pci_set_master(pdev);
11245
11246                 err = bnxt_hwrm_func_reset(bp);
11247                 if (!err && netif_running(netdev))
11248                         err = bnxt_open(netdev);
11249
11250                 if (!err) {
11251                         result = PCI_ERS_RESULT_RECOVERED;
11252                         bnxt_ulp_start(bp);
11253                 }
11254         }
11255
11256         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
11257                 dev_close(netdev);
11258
11259         rtnl_unlock();
11260
11261         return PCI_ERS_RESULT_RECOVERED;
11262 }
11263
11264 /**
11265  * bnxt_io_resume - called when traffic can start flowing again.
11266  * @pdev: Pointer to PCI device
11267  *
11268  * This callback is called when the error recovery driver tells
11269  * us that its OK to resume normal operation.
11270  */
11271 static void bnxt_io_resume(struct pci_dev *pdev)
11272 {
11273         struct net_device *netdev = pci_get_drvdata(pdev);
11274
11275         rtnl_lock();
11276
11277         netif_device_attach(netdev);
11278
11279         rtnl_unlock();
11280 }
11281
11282 static const struct pci_error_handlers bnxt_err_handler = {
11283         .error_detected = bnxt_io_error_detected,
11284         .slot_reset     = bnxt_io_slot_reset,
11285         .resume         = bnxt_io_resume
11286 };
11287
11288 static struct pci_driver bnxt_pci_driver = {
11289         .name           = DRV_MODULE_NAME,
11290         .id_table       = bnxt_pci_tbl,
11291         .probe          = bnxt_init_one,
11292         .remove         = bnxt_remove_one,
11293         .shutdown       = bnxt_shutdown,
11294         .driver.pm      = BNXT_PM_OPS,
11295         .err_handler    = &bnxt_err_handler,
11296 #if defined(CONFIG_BNXT_SRIOV)
11297         .sriov_configure = bnxt_sriov_configure,
11298 #endif
11299 };
11300
11301 static int __init bnxt_init(void)
11302 {
11303         bnxt_debug_init();
11304         return pci_register_driver(&bnxt_pci_driver);
11305 }
11306
11307 static void __exit bnxt_exit(void)
11308 {
11309         pci_unregister_driver(&bnxt_pci_driver);
11310         if (bnxt_pf_wq)
11311                 destroy_workqueue(bnxt_pf_wq);
11312         bnxt_debug_exit();
11313 }
11314
11315 module_init(bnxt_init);
11316 module_exit(bnxt_exit);