drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72
  73 static const char version[] =
  74         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  75
  76 MODULE_LICENSE("GPL");
  77 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  78 MODULE_VERSION(DRV_MODULE_VERSION);
  79
  80 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  81 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  82 #define BNXT_RX_COPY_THRESH 256
  83
  84 #define BNXT_TX_PUSH_THRESH 164
  85
  86 enum board_idx {
  87         BCM57301,
  88         BCM57302,
  89         BCM57304,
  90         BCM57417_NPAR,
  91         BCM58700,
  92         BCM57311,
  93         BCM57312,
  94         BCM57402,
  95         BCM57404,
  96         BCM57406,
  97         BCM57402_NPAR,
  98         BCM57407,
  99         BCM57412,
 100         BCM57414,
 101         BCM57416,
 102         BCM57417,
 103         BCM57412_NPAR,
 104         BCM57314,
 105         BCM57417_SFP,
 106         BCM57416_SFP,
 107         BCM57404_NPAR,
 108         BCM57406_NPAR,
 109         BCM57407_SFP,
 110         BCM57407_NPAR,
 111         BCM57414_NPAR,
 112         BCM57416_NPAR,
 113         BCM57452,
 114         BCM57454,
 115         BCM5745x_NPAR,
 116         BCM57508,
 117         BCM57504,
 118         BCM57502,
 119         BCM57508_NPAR,
 120         BCM57504_NPAR,
 121         BCM57502_NPAR,
 122         BCM58802,
 123         BCM58804,
 124         BCM58808,
 125         NETXTREME_E_VF,
 126         NETXTREME_C_VF,
 127         NETXTREME_S_VF,
 128         NETXTREME_E_P5_VF,
 129 };
 130
 131 /* indexed by enum above */
 132 static const struct {
 133         char *name;
 134 } board_info[] = {
 135         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 136         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 137         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 138         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 139         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 140         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 141         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 142         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 143         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 144         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 145         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 146         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 148         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 149         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 150         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 151         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 152         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 153         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 154         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 155         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 156         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 157         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 158         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 159         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 160         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 161         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 162         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 163         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 164         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 165         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 166         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 167         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 168         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 169         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 170         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 171         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 172         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 173         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 174         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 175         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 176         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 177 };
 178
 179 static const struct pci_device_id bnxt_pci_tbl[] = {
 180         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 182         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 183         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 185         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 188         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 189         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 190         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 192         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 195         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 197         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 198         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 199         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 200         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 201         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 202         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 203         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 207         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 211         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 212         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 213         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 214         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 215         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 216         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 217         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 218         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 222         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 223         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 224         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 225         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 226 #ifdef CONFIG_BNXT_SRIOV
 227         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 234         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 235         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 236         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 237         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 238 #endif
 239         { 0 }
 240 };
 241
 242 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 243
 244 static const u16 bnxt_vf_req_snif[] = {
 245         HWRM_FUNC_CFG,
 246         HWRM_FUNC_VF_CFG,
 247         HWRM_PORT_PHY_QCFG,
 248         HWRM_CFA_L2_FILTER_ALLOC,
 249 };
 250
 251 static const u16 bnxt_async_events_arr[] = {
 252         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 253         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 254         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 255         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 257         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 258         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 259 };
 260
 261 static struct workqueue_struct *bnxt_pf_wq;
 262
 263 static bool bnxt_vf_pciid(enum board_idx idx)
 264 {
 265         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 266                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 267 }
 268
 269 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 270 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 271 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 272
 273 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 274                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 275
 276 #define BNXT_DB_CQ(db, idx)                                             \
 277         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 278
 279 #define BNXT_DB_NQ_P5(db, idx)                                          \
 280         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 281
 282 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 283         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 284
 285 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 286         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 287
 288 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 289 {
 290         if (bp->flags & BNXT_FLAG_CHIP_P5)
 291                 BNXT_DB_NQ_P5(db, idx);
 292         else
 293                 BNXT_DB_CQ(db, idx);
 294 }
 295
 296 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 297 {
 298         if (bp->flags & BNXT_FLAG_CHIP_P5)
 299                 BNXT_DB_NQ_ARM_P5(db, idx);
 300         else
 301                 BNXT_DB_CQ_ARM(db, idx);
 302 }
 303
 304 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 305 {
 306         if (bp->flags & BNXT_FLAG_CHIP_P5)
 307                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 308                        db->doorbell);
 309         else
 310                 BNXT_DB_CQ(db, idx);
 311 }
 312
 313 const u16 bnxt_lhint_arr[] = {
 314         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 315         TX_BD_FLAGS_LHINT_512_TO_1023,
 316         TX_BD_FLAGS_LHINT_1024_TO_2047,
 317         TX_BD_FLAGS_LHINT_1024_TO_2047,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 332         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 333 };
 334
 335 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 336 {
 337         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 338
 339         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 340                 return 0;
 341
 342         return md_dst->u.port_info.port_id;
 343 }
 344
 345 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 346 {
 347         struct bnxt *bp = netdev_priv(dev);
 348         struct tx_bd *txbd;
 349         struct tx_bd_ext *txbd1;
 350         struct netdev_queue *txq;
 351         int i;
 352         dma_addr_t mapping;
 353         unsigned int length, pad = 0;
 354         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 355         u16 prod, last_frag;
 356         struct pci_dev *pdev = bp->pdev;
 357         struct bnxt_tx_ring_info *txr;
 358         struct bnxt_sw_tx_bd *tx_buf;
 359
 360         i = skb_get_queue_mapping(skb);
 361         if (unlikely(i >= bp->tx_nr_rings)) {
 362                 dev_kfree_skb_any(skb);
 363                 return NETDEV_TX_OK;
 364         }
 365
 366         txq = netdev_get_tx_queue(dev, i);
 367         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 368         prod = txr->tx_prod;
 369
 370         free_size = bnxt_tx_avail(bp, txr);
 371         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 372                 netif_tx_stop_queue(txq);
 373                 return NETDEV_TX_BUSY;
 374         }
 375
 376         length = skb->len;
 377         len = skb_headlen(skb);
 378         last_frag = skb_shinfo(skb)->nr_frags;
 379
 380         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 381
 382         txbd->tx_bd_opaque = prod;
 383
 384         tx_buf = &txr->tx_buf_ring[prod];
 385         tx_buf->skb = skb;
 386         tx_buf->nr_frags = last_frag;
 387
 388         vlan_tag_flags = 0;
 389         cfa_action = bnxt_xmit_get_cfa_action(skb);
 390         if (skb_vlan_tag_present(skb)) {
 391                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 392                                  skb_vlan_tag_get(skb);
 393                 /* Currently supports 8021Q, 8021AD vlan offloads
 394                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 395                  */
 396                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 397                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 398         }
 399
 400         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 401                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 402                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 403                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 404                 void __iomem *db = txr->tx_db.doorbell;
 405                 void *pdata = tx_push_buf->data;
 406                 u64 *end;
 407                 int j, push_len;
 408
 409                 /* Set COAL_NOW to be ready quickly for the next push */
 410                 tx_push->tx_bd_len_flags_type =
 411                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 412                                         TX_BD_TYPE_LONG_TX_BD |
 413                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 414                                         TX_BD_FLAGS_COAL_NOW |
 415                                         TX_BD_FLAGS_PACKET_END |
 416                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 417
 418                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 419                         tx_push1->tx_bd_hsize_lflags =
 420                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 421                 else
 422                         tx_push1->tx_bd_hsize_lflags = 0;
 423
 424                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 425                 tx_push1->tx_bd_cfa_action =
 426                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 427
 428                 end = pdata + length;
 429                 end = PTR_ALIGN(end, 8) - 1;
 430                 *end = 0;
 431
 432                 skb_copy_from_linear_data(skb, pdata, len);
 433                 pdata += len;
 434                 for (j = 0; j < last_frag; j++) {
 435                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 436                         void *fptr;
 437
 438                         fptr = skb_frag_address_safe(frag);
 439                         if (!fptr)
 440                                 goto normal_tx;
 441
 442                         memcpy(pdata, fptr, skb_frag_size(frag));
 443                         pdata += skb_frag_size(frag);
 444                 }
 445
 446                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 447                 txbd->tx_bd_haddr = txr->data_mapping;
 448                 prod = NEXT_TX(prod);
 449                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 450                 memcpy(txbd, tx_push1, sizeof(*txbd));
 451                 prod = NEXT_TX(prod);
 452                 tx_push->doorbell =
 453                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 454                 txr->tx_prod = prod;
 455
 456                 tx_buf->is_push = 1;
 457                 netdev_tx_sent_queue(txq, skb->len);
 458                 wmb();  /* Sync is_push and byte queue before pushing data */
 459
 460                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 461                 if (push_len > 16) {
 462                         __iowrite64_copy(db, tx_push_buf, 16);
 463                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 464                                          (push_len - 16) << 1);
 465                 } else {
 466                         __iowrite64_copy(db, tx_push_buf, push_len);
 467                 }
 468
 469                 goto tx_done;
 470         }
 471
 472 normal_tx:
 473         if (length < BNXT_MIN_PKT_SIZE) {
 474                 pad = BNXT_MIN_PKT_SIZE - length;
 475                 if (skb_pad(skb, pad)) {
 476                         /* SKB already freed. */
 477                         tx_buf->skb = NULL;
 478                         return NETDEV_TX_OK;
 479                 }
 480                 length = BNXT_MIN_PKT_SIZE;
 481         }
 482
 483         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 484
 485         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 486                 dev_kfree_skb_any(skb);
 487                 tx_buf->skb = NULL;
 488                 return NETDEV_TX_OK;
 489         }
 490
 491         dma_unmap_addr_set(tx_buf, mapping, mapping);
 492         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 493                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 494
 495         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 496
 497         prod = NEXT_TX(prod);
 498         txbd1 = (struct tx_bd_ext *)
 499                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 500
 501         txbd1->tx_bd_hsize_lflags = 0;
 502         if (skb_is_gso(skb)) {
 503                 u32 hdr_len;
 504
 505                 if (skb->encapsulation)
 506                         hdr_len = skb_inner_network_offset(skb) +
 507                                 skb_inner_network_header_len(skb) +
 508                                 inner_tcp_hdrlen(skb);
 509                 else
 510                         hdr_len = skb_transport_offset(skb) +
 511                                 tcp_hdrlen(skb);
 512
 513                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 514                                         TX_BD_FLAGS_T_IPID |
 515                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 516                 length = skb_shinfo(skb)->gso_size;
 517                 txbd1->tx_bd_mss = cpu_to_le32(length);
 518                 length += hdr_len;
 519         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 520                 txbd1->tx_bd_hsize_lflags =
 521                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 522                 txbd1->tx_bd_mss = 0;
 523         }
 524
 525         length >>= 9;
 526         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 527                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 528                                      skb->len);
 529                 i = 0;
 530                 goto tx_dma_error;
 531         }
 532         flags |= bnxt_lhint_arr[length];
 533         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 534
 535         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 536         txbd1->tx_bd_cfa_action =
 537                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 538         for (i = 0; i < last_frag; i++) {
 539                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 540
 541                 prod = NEXT_TX(prod);
 542                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 543
 544                 len = skb_frag_size(frag);
 545                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 546                                            DMA_TO_DEVICE);
 547
 548                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 549                         goto tx_dma_error;
 550
 551                 tx_buf = &txr->tx_buf_ring[prod];
 552                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 553
 554                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 555
 556                 flags = len << TX_BD_LEN_SHIFT;
 557                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 558         }
 559
 560         flags &= ~TX_BD_LEN;
 561         txbd->tx_bd_len_flags_type =
 562                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 563                             TX_BD_FLAGS_PACKET_END);
 564
 565         netdev_tx_sent_queue(txq, skb->len);
 566
 567         /* Sync BD data before updating doorbell */
 568         wmb();
 569
 570         prod = NEXT_TX(prod);
 571         txr->tx_prod = prod;
 572
 573         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 574                 bnxt_db_write(bp, &txr->tx_db, prod);
 575
 576 tx_done:
 577
 578         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 579                 if (netdev_xmit_more() && !tx_buf->is_push)
 580                         bnxt_db_write(bp, &txr->tx_db, prod);
 581
 582                 netif_tx_stop_queue(txq);
 583
 584                 /* netif_tx_stop_queue() must be done before checking
 585                  * tx index in bnxt_tx_avail() below, because in
 586                  * bnxt_tx_int(), we update tx index before checking for
 587                  * netif_tx_queue_stopped().
 588                  */
 589                 smp_mb();
 590                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 591                         netif_tx_wake_queue(txq);
 592         }
 593         return NETDEV_TX_OK;
 594
 595 tx_dma_error:
 596         last_frag = i;
 597
 598         /* start back at beginning and unmap skb */
 599         prod = txr->tx_prod;
 600         tx_buf = &txr->tx_buf_ring[prod];
 601         tx_buf->skb = NULL;
 602         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 603                          skb_headlen(skb), PCI_DMA_TODEVICE);
 604         prod = NEXT_TX(prod);
 605
 606         /* unmap remaining mapped pages */
 607         for (i = 0; i < last_frag; i++) {
 608                 prod = NEXT_TX(prod);
 609                 tx_buf = &txr->tx_buf_ring[prod];
 610                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 611                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 612                                PCI_DMA_TODEVICE);
 613         }
 614
 615         dev_kfree_skb_any(skb);
 616         return NETDEV_TX_OK;
 617 }
 618
 619 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 620 {
 621         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 622         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 623         u16 cons = txr->tx_cons;
 624         struct pci_dev *pdev = bp->pdev;
 625         int i;
 626         unsigned int tx_bytes = 0;
 627
 628         for (i = 0; i < nr_pkts; i++) {
 629                 struct bnxt_sw_tx_bd *tx_buf;
 630                 struct sk_buff *skb;
 631                 int j, last;
 632
 633                 tx_buf = &txr->tx_buf_ring[cons];
 634                 cons = NEXT_TX(cons);
 635                 skb = tx_buf->skb;
 636                 tx_buf->skb = NULL;
 637
 638                 if (tx_buf->is_push) {
 639                         tx_buf->is_push = 0;
 640                         goto next_tx_int;
 641                 }
 642
 643                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 644                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 645                 last = tx_buf->nr_frags;
 646
 647                 for (j = 0; j < last; j++) {
 648                         cons = NEXT_TX(cons);
 649                         tx_buf = &txr->tx_buf_ring[cons];
 650                         dma_unmap_page(
 651                                 &pdev->dev,
 652                                 dma_unmap_addr(tx_buf, mapping),
 653                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 654                                 PCI_DMA_TODEVICE);
 655                 }
 656
 657 next_tx_int:
 658                 cons = NEXT_TX(cons);
 659
 660                 tx_bytes += skb->len;
 661                 dev_kfree_skb_any(skb);
 662         }
 663
 664         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 665         txr->tx_cons = cons;
 666
 667         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 668          * before checking for netif_tx_queue_stopped().  Without the
 669          * memory barrier, there is a small possibility that bnxt_start_xmit()
 670          * will miss it and cause the queue to be stopped forever.
 671          */
 672         smp_mb();
 673
 674         if (unlikely(netif_tx_queue_stopped(txq)) &&
 675             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 676                 __netif_tx_lock(txq, smp_processor_id());
 677                 if (netif_tx_queue_stopped(txq) &&
 678                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 679                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 680                         netif_tx_wake_queue(txq);
 681                 __netif_tx_unlock(txq);
 682         }
 683 }
 684
 685 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 686                                          struct bnxt_rx_ring_info *rxr,
 687                                          gfp_t gfp)
 688 {
 689         struct device *dev = &bp->pdev->dev;
 690         struct page *page;
 691
 692         page = page_pool_dev_alloc_pages(rxr->page_pool);
 693         if (!page)
 694                 return NULL;
 695
 696         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 697                                       DMA_ATTR_WEAK_ORDERING);
 698         if (dma_mapping_error(dev, *mapping)) {
 699                 page_pool_recycle_direct(rxr->page_pool, page);
 700                 return NULL;
 701         }
 702         *mapping += bp->rx_dma_offset;
 703         return page;
 704 }
 705
 706 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 707                                        gfp_t gfp)
 708 {
 709         u8 *data;
 710         struct pci_dev *pdev = bp->pdev;
 711
 712         data = kmalloc(bp->rx_buf_size, gfp);
 713         if (!data)
 714                 return NULL;
 715
 716         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 717                                         bp->rx_buf_use_size, bp->rx_dir,
 718                                         DMA_ATTR_WEAK_ORDERING);
 719
 720         if (dma_mapping_error(&pdev->dev, *mapping)) {
 721                 kfree(data);
 722                 data = NULL;
 723         }
 724         return data;
 725 }
 726
 727 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 728                        u16 prod, gfp_t gfp)
 729 {
 730         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 731         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 732         dma_addr_t mapping;
 733
 734         if (BNXT_RX_PAGE_MODE(bp)) {
 735                 struct page *page =
 736                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 737
 738                 if (!page)
 739                         return -ENOMEM;
 740
 741                 rx_buf->data = page;
 742                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 743         } else {
 744                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 745
 746                 if (!data)
 747                         return -ENOMEM;
 748
 749                 rx_buf->data = data;
 750                 rx_buf->data_ptr = data + bp->rx_offset;
 751         }
 752         rx_buf->mapping = mapping;
 753
 754         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 755         return 0;
 756 }
 757
 758 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 759 {
 760         u16 prod = rxr->rx_prod;
 761         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 762         struct rx_bd *cons_bd, *prod_bd;
 763
 764         prod_rx_buf = &rxr->rx_buf_ring[prod];
 765         cons_rx_buf = &rxr->rx_buf_ring[cons];
 766
 767         prod_rx_buf->data = data;
 768         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 769
 770         prod_rx_buf->mapping = cons_rx_buf->mapping;
 771
 772         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 773         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 774
 775         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 776 }
 777
 778 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 779 {
 780         u16 next, max = rxr->rx_agg_bmap_size;
 781
 782         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 783         if (next >= max)
 784                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 785         return next;
 786 }
 787
 788 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 789                                      struct bnxt_rx_ring_info *rxr,
 790                                      u16 prod, gfp_t gfp)
 791 {
 792         struct rx_bd *rxbd =
 793                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 794         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 795         struct pci_dev *pdev = bp->pdev;
 796         struct page *page;
 797         dma_addr_t mapping;
 798         u16 sw_prod = rxr->rx_sw_agg_prod;
 799         unsigned int offset = 0;
 800
 801         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 802                 page = rxr->rx_page;
 803                 if (!page) {
 804                         page = alloc_page(gfp);
 805                         if (!page)
 806                                 return -ENOMEM;
 807                         rxr->rx_page = page;
 808                         rxr->rx_page_offset = 0;
 809                 }
 810                 offset = rxr->rx_page_offset;
 811                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 812                 if (rxr->rx_page_offset == PAGE_SIZE)
 813                         rxr->rx_page = NULL;
 814                 else
 815                         get_page(page);
 816         } else {
 817                 page = alloc_page(gfp);
 818                 if (!page)
 819                         return -ENOMEM;
 820         }
 821
 822         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 823                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 824                                      DMA_ATTR_WEAK_ORDERING);
 825         if (dma_mapping_error(&pdev->dev, mapping)) {
 826                 __free_page(page);
 827                 return -EIO;
 828         }
 829
 830         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 831                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 832
 833         __set_bit(sw_prod, rxr->rx_agg_bmap);
 834         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 835         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 836
 837         rx_agg_buf->page = page;
 838         rx_agg_buf->offset = offset;
 839         rx_agg_buf->mapping = mapping;
 840         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 841         rxbd->rx_bd_opaque = sw_prod;
 842         return 0;
 843 }
 844
 845 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 846                                        struct bnxt_cp_ring_info *cpr,
 847                                        u16 cp_cons, u16 curr)
 848 {
 849         struct rx_agg_cmp *agg;
 850
 851         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 852         agg = (struct rx_agg_cmp *)
 853                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 854         return agg;
 855 }
 856
 857 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 858                                               struct bnxt_rx_ring_info *rxr,
 859                                               u16 agg_id, u16 curr)
 860 {
 861         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 862
 863         return &tpa_info->agg_arr[curr];
 864 }
 865
 866 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 867                                    u16 start, u32 agg_bufs, bool tpa)
 868 {
 869         struct bnxt_napi *bnapi = cpr->bnapi;
 870         struct bnxt *bp = bnapi->bp;
 871         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 872         u16 prod = rxr->rx_agg_prod;
 873         u16 sw_prod = rxr->rx_sw_agg_prod;
 874         bool p5_tpa = false;
 875         u32 i;
 876
 877         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 878                 p5_tpa = true;
 879
 880         for (i = 0; i < agg_bufs; i++) {
 881                 u16 cons;
 882                 struct rx_agg_cmp *agg;
 883                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 884                 struct rx_bd *prod_bd;
 885                 struct page *page;
 886
 887                 if (p5_tpa)
 888                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 889                 else
 890                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 891                 cons = agg->rx_agg_cmp_opaque;
 892                 __clear_bit(cons, rxr->rx_agg_bmap);
 893
 894                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 895                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 896
 897                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 898                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 899                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 900
 901                 /* It is possible for sw_prod to be equal to cons, so
 902                  * set cons_rx_buf->page to NULL first.
 903                  */
 904                 page = cons_rx_buf->page;
 905                 cons_rx_buf->page = NULL;
 906                 prod_rx_buf->page = page;
 907                 prod_rx_buf->offset = cons_rx_buf->offset;
 908
 909                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 910
 911                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 912
 913                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 914                 prod_bd->rx_bd_opaque = sw_prod;
 915
 916                 prod = NEXT_RX_AGG(prod);
 917                 sw_prod = NEXT_RX_AGG(sw_prod);
 918         }
 919         rxr->rx_agg_prod = prod;
 920         rxr->rx_sw_agg_prod = sw_prod;
 921 }
 922
 923 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 924                                         struct bnxt_rx_ring_info *rxr,
 925                                         u16 cons, void *data, u8 *data_ptr,
 926                                         dma_addr_t dma_addr,
 927                                         unsigned int offset_and_len)
 928 {
 929         unsigned int payload = offset_and_len >> 16;
 930         unsigned int len = offset_and_len & 0xffff;
 931         skb_frag_t *frag;
 932         struct page *page = data;
 933         u16 prod = rxr->rx_prod;
 934         struct sk_buff *skb;
 935         int off, err;
 936
 937         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 938         if (unlikely(err)) {
 939                 bnxt_reuse_rx_data(rxr, cons, data);
 940                 return NULL;
 941         }
 942         dma_addr -= bp->rx_dma_offset;
 943         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 944                              DMA_ATTR_WEAK_ORDERING);
 945
 946         if (unlikely(!payload))
 947                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 948
 949         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 950         if (!skb) {
 951                 __free_page(page);
 952                 return NULL;
 953         }
 954
 955         off = (void *)data_ptr - page_address(page);
 956         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 957         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 958                payload + NET_IP_ALIGN);
 959
 960         frag = &skb_shinfo(skb)->frags[0];
 961         skb_frag_size_sub(frag, payload);
 962         skb_frag_off_add(frag, payload);
 963         skb->data_len -= payload;
 964         skb->tail += payload;
 965
 966         return skb;
 967 }
 968
 969 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 970                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 971                                    void *data, u8 *data_ptr,
 972                                    dma_addr_t dma_addr,
 973                                    unsigned int offset_and_len)
 974 {
 975         u16 prod = rxr->rx_prod;
 976         struct sk_buff *skb;
 977         int err;
 978
 979         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 980         if (unlikely(err)) {
 981                 bnxt_reuse_rx_data(rxr, cons, data);
 982                 return NULL;
 983         }
 984
 985         skb = build_skb(data, 0);
 986         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 987                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 988         if (!skb) {
 989                 kfree(data);
 990                 return NULL;
 991         }
 992
 993         skb_reserve(skb, bp->rx_offset);
 994         skb_put(skb, offset_and_len & 0xffff);
 995         return skb;
 996 }
 997
 998 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
 999                                      struct bnxt_cp_ring_info *cpr,
1000                                      struct sk_buff *skb, u16 idx,
1001                                      u32 agg_bufs, bool tpa)
1002 {
1003         struct bnxt_napi *bnapi = cpr->bnapi;
1004         struct pci_dev *pdev = bp->pdev;
1005         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1006         u16 prod = rxr->rx_agg_prod;
1007         bool p5_tpa = false;
1008         u32 i;
1009
1010         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1011                 p5_tpa = true;
1012
1013         for (i = 0; i < agg_bufs; i++) {
1014                 u16 cons, frag_len;
1015                 struct rx_agg_cmp *agg;
1016                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1017                 struct page *page;
1018                 dma_addr_t mapping;
1019
1020                 if (p5_tpa)
1021                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1022                 else
1023                         agg = bnxt_get_agg(bp, cpr, idx, i);
1024                 cons = agg->rx_agg_cmp_opaque;
1025                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1026                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1027
1028                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1029                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1030                                    cons_rx_buf->offset, frag_len);
1031                 __clear_bit(cons, rxr->rx_agg_bmap);
1032
1033                 /* It is possible for bnxt_alloc_rx_page() to allocate
1034                  * a sw_prod index that equals the cons index, so we
1035                  * need to clear the cons entry now.
1036                  */
1037                 mapping = cons_rx_buf->mapping;
1038                 page = cons_rx_buf->page;
1039                 cons_rx_buf->page = NULL;
1040
1041                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1042                         struct skb_shared_info *shinfo;
1043                         unsigned int nr_frags;
1044
1045                         shinfo = skb_shinfo(skb);
1046                         nr_frags = --shinfo->nr_frags;
1047                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1048
1049                         dev_kfree_skb(skb);
1050
1051                         cons_rx_buf->page = page;
1052
1053                         /* Update prod since possibly some pages have been
1054                          * allocated already.
1055                          */
1056                         rxr->rx_agg_prod = prod;
1057                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1058                         return NULL;
1059                 }
1060
1061                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1062                                      PCI_DMA_FROMDEVICE,
1063                                      DMA_ATTR_WEAK_ORDERING);
1064
1065                 skb->data_len += frag_len;
1066                 skb->len += frag_len;
1067                 skb->truesize += PAGE_SIZE;
1068
1069                 prod = NEXT_RX_AGG(prod);
1070         }
1071         rxr->rx_agg_prod = prod;
1072         return skb;
1073 }
1074
1075 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1076                                u8 agg_bufs, u32 *raw_cons)
1077 {
1078         u16 last;
1079         struct rx_agg_cmp *agg;
1080
1081         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1082         last = RING_CMP(*raw_cons);
1083         agg = (struct rx_agg_cmp *)
1084                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1085         return RX_AGG_CMP_VALID(agg, *raw_cons);
1086 }
1087
1088 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1089                                             unsigned int len,
1090                                             dma_addr_t mapping)
1091 {
1092         struct bnxt *bp = bnapi->bp;
1093         struct pci_dev *pdev = bp->pdev;
1094         struct sk_buff *skb;
1095
1096         skb = napi_alloc_skb(&bnapi->napi, len);
1097         if (!skb)
1098                 return NULL;
1099
1100         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1101                                 bp->rx_dir);
1102
1103         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1104                len + NET_IP_ALIGN);
1105
1106         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1107                                    bp->rx_dir);
1108
1109         skb_put(skb, len);
1110         return skb;
1111 }
1112
1113 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1114                            u32 *raw_cons, void *cmp)
1115 {
1116         struct rx_cmp *rxcmp = cmp;
1117         u32 tmp_raw_cons = *raw_cons;
1118         u8 cmp_type, agg_bufs = 0;
1119
1120         cmp_type = RX_CMP_TYPE(rxcmp);
1121
1122         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1123                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1124                             RX_CMP_AGG_BUFS) >>
1125                            RX_CMP_AGG_BUFS_SHIFT;
1126         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1127                 struct rx_tpa_end_cmp *tpa_end = cmp;
1128
1129                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1130                         return 0;
1131
1132                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1133         }
1134
1135         if (agg_bufs) {
1136                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1137                         return -EBUSY;
1138         }
1139         *raw_cons = tmp_raw_cons;
1140         return 0;
1141 }
1142
1143 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1144 {
1145         if (BNXT_PF(bp))
1146                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1147         else
1148                 schedule_delayed_work(&bp->fw_reset_task, delay);
1149 }
1150
1151 static void bnxt_queue_sp_work(struct bnxt *bp)
1152 {
1153         if (BNXT_PF(bp))
1154                 queue_work(bnxt_pf_wq, &bp->sp_task);
1155         else
1156                 schedule_work(&bp->sp_task);
1157 }
1158
1159 static void bnxt_cancel_sp_work(struct bnxt *bp)
1160 {
1161         if (BNXT_PF(bp))
1162                 flush_workqueue(bnxt_pf_wq);
1163         else
1164                 cancel_work_sync(&bp->sp_task);
1165 }
1166
1167 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1168 {
1169         if (!rxr->bnapi->in_reset) {
1170                 rxr->bnapi->in_reset = true;
1171                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1172                 bnxt_queue_sp_work(bp);
1173         }
1174         rxr->rx_next_cons = 0xffff;
1175 }
1176
1177 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1178 {
1179         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1180         u16 idx = agg_id & MAX_TPA_P5_MASK;
1181
1182         if (test_bit(idx, map->agg_idx_bmap))
1183                 idx = find_first_zero_bit(map->agg_idx_bmap,
1184                                           BNXT_AGG_IDX_BMAP_SIZE);
1185         __set_bit(idx, map->agg_idx_bmap);
1186         map->agg_id_tbl[agg_id] = idx;
1187         return idx;
1188 }
1189
1190 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1191 {
1192         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1193
1194         __clear_bit(idx, map->agg_idx_bmap);
1195 }
1196
1197 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1198 {
1199         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1200
1201         return map->agg_id_tbl[agg_id];
1202 }
1203
1204 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1205                            struct rx_tpa_start_cmp *tpa_start,
1206                            struct rx_tpa_start_cmp_ext *tpa_start1)
1207 {
1208         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1209         struct bnxt_tpa_info *tpa_info;
1210         u16 cons, prod, agg_id;
1211         struct rx_bd *prod_bd;
1212         dma_addr_t mapping;
1213
1214         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1215                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1216                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1217         } else {
1218                 agg_id = TPA_START_AGG_ID(tpa_start);
1219         }
1220         cons = tpa_start->rx_tpa_start_cmp_opaque;
1221         prod = rxr->rx_prod;
1222         cons_rx_buf = &rxr->rx_buf_ring[cons];
1223         prod_rx_buf = &rxr->rx_buf_ring[prod];
1224         tpa_info = &rxr->rx_tpa[agg_id];
1225
1226         if (unlikely(cons != rxr->rx_next_cons ||
1227                      TPA_START_ERROR(tpa_start))) {
1228                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1229                             cons, rxr->rx_next_cons,
1230                             TPA_START_ERROR_CODE(tpa_start1));
1231                 bnxt_sched_reset(bp, rxr);
1232                 return;
1233         }
1234         /* Store cfa_code in tpa_info to use in tpa_end
1235          * completion processing.
1236          */
1237         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1238         prod_rx_buf->data = tpa_info->data;
1239         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1240
1241         mapping = tpa_info->mapping;
1242         prod_rx_buf->mapping = mapping;
1243
1244         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1245
1246         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1247
1248         tpa_info->data = cons_rx_buf->data;
1249         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1250         cons_rx_buf->data = NULL;
1251         tpa_info->mapping = cons_rx_buf->mapping;
1252
1253         tpa_info->len =
1254                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1255                                 RX_TPA_START_CMP_LEN_SHIFT;
1256         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1257                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1258
1259                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1260                 tpa_info->gso_type = SKB_GSO_TCPV4;
1261                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1262                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1263                         tpa_info->gso_type = SKB_GSO_TCPV6;
1264                 tpa_info->rss_hash =
1265                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1266         } else {
1267                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1268                 tpa_info->gso_type = 0;
1269                 if (netif_msg_rx_err(bp))
1270                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1271         }
1272         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1273         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1274         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1275         tpa_info->agg_count = 0;
1276
1277         rxr->rx_prod = NEXT_RX(prod);
1278         cons = NEXT_RX(cons);
1279         rxr->rx_next_cons = NEXT_RX(cons);
1280         cons_rx_buf = &rxr->rx_buf_ring[cons];
1281
1282         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1283         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1284         cons_rx_buf->data = NULL;
1285 }
1286
1287 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1288 {
1289         if (agg_bufs)
1290                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1291 }
1292
1293 #ifdef CONFIG_INET
1294 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1295 {
1296         struct udphdr *uh = NULL;
1297
1298         if (ip_proto == htons(ETH_P_IP)) {
1299                 struct iphdr *iph = (struct iphdr *)skb->data;
1300
1301                 if (iph->protocol == IPPROTO_UDP)
1302                         uh = (struct udphdr *)(iph + 1);
1303         } else {
1304                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1305
1306                 if (iph->nexthdr == IPPROTO_UDP)
1307                         uh = (struct udphdr *)(iph + 1);
1308         }
1309         if (uh) {
1310                 if (uh->check)
1311                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1312                 else
1313                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1314         }
1315 }
1316 #endif
1317
1318 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1319                                            int payload_off, int tcp_ts,
1320                                            struct sk_buff *skb)
1321 {
1322 #ifdef CONFIG_INET
1323         struct tcphdr *th;
1324         int len, nw_off;
1325         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1326         u32 hdr_info = tpa_info->hdr_info;
1327         bool loopback = false;
1328
1329         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1330         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1331         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1332
1333         /* If the packet is an internal loopback packet, the offsets will
1334          * have an extra 4 bytes.
1335          */
1336         if (inner_mac_off == 4) {
1337                 loopback = true;
1338         } else if (inner_mac_off > 4) {
1339                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1340                                             ETH_HLEN - 2));
1341
1342                 /* We only support inner iPv4/ipv6.  If we don't see the
1343                  * correct protocol ID, it must be a loopback packet where
1344                  * the offsets are off by 4.
1345                  */
1346                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1347                         loopback = true;
1348         }
1349         if (loopback) {
1350                 /* internal loopback packet, subtract all offsets by 4 */
1351                 inner_ip_off -= 4;
1352                 inner_mac_off -= 4;
1353                 outer_ip_off -= 4;
1354         }
1355
1356         nw_off = inner_ip_off - ETH_HLEN;
1357         skb_set_network_header(skb, nw_off);
1358         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1359                 struct ipv6hdr *iph = ipv6_hdr(skb);
1360
1361                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1362                 len = skb->len - skb_transport_offset(skb);
1363                 th = tcp_hdr(skb);
1364                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1365         } else {
1366                 struct iphdr *iph = ip_hdr(skb);
1367
1368                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1369                 len = skb->len - skb_transport_offset(skb);
1370                 th = tcp_hdr(skb);
1371                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1372         }
1373
1374         if (inner_mac_off) { /* tunnel */
1375                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1376                                             ETH_HLEN - 2));
1377
1378                 bnxt_gro_tunnel(skb, proto);
1379         }
1380 #endif
1381         return skb;
1382 }
1383
1384 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1385                                            int payload_off, int tcp_ts,
1386                                            struct sk_buff *skb)
1387 {
1388 #ifdef CONFIG_INET
1389         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1390         u32 hdr_info = tpa_info->hdr_info;
1391         int iphdr_len, nw_off;
1392
1393         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1394         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1395         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1396
1397         nw_off = inner_ip_off - ETH_HLEN;
1398         skb_set_network_header(skb, nw_off);
1399         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1400                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1401         skb_set_transport_header(skb, nw_off + iphdr_len);
1402
1403         if (inner_mac_off) { /* tunnel */
1404                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1405                                             ETH_HLEN - 2));
1406
1407                 bnxt_gro_tunnel(skb, proto);
1408         }
1409 #endif
1410         return skb;
1411 }
1412
1413 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1414 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1415
1416 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1417                                            int payload_off, int tcp_ts,
1418                                            struct sk_buff *skb)
1419 {
1420 #ifdef CONFIG_INET
1421         struct tcphdr *th;
1422         int len, nw_off, tcp_opt_len = 0;
1423
1424         if (tcp_ts)
1425                 tcp_opt_len = 12;
1426
1427         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1428                 struct iphdr *iph;
1429
1430                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1431                          ETH_HLEN;
1432                 skb_set_network_header(skb, nw_off);
1433                 iph = ip_hdr(skb);
1434                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1435                 len = skb->len - skb_transport_offset(skb);
1436                 th = tcp_hdr(skb);
1437                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1438         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1439                 struct ipv6hdr *iph;
1440
1441                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1442                          ETH_HLEN;
1443                 skb_set_network_header(skb, nw_off);
1444                 iph = ipv6_hdr(skb);
1445                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1446                 len = skb->len - skb_transport_offset(skb);
1447                 th = tcp_hdr(skb);
1448                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1449         } else {
1450                 dev_kfree_skb_any(skb);
1451                 return NULL;
1452         }
1453
1454         if (nw_off) /* tunnel */
1455                 bnxt_gro_tunnel(skb, skb->protocol);
1456 #endif
1457         return skb;
1458 }
1459
1460 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1461                                            struct bnxt_tpa_info *tpa_info,
1462                                            struct rx_tpa_end_cmp *tpa_end,
1463                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1464                                            struct sk_buff *skb)
1465 {
1466 #ifdef CONFIG_INET
1467         int payload_off;
1468         u16 segs;
1469
1470         segs = TPA_END_TPA_SEGS(tpa_end);
1471         if (segs == 1)
1472                 return skb;
1473
1474         NAPI_GRO_CB(skb)->count = segs;
1475         skb_shinfo(skb)->gso_size =
1476                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1477         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1478         if (bp->flags & BNXT_FLAG_CHIP_P5)
1479                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1480         else
1481                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1482         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1483         if (likely(skb))
1484                 tcp_gro_complete(skb);
1485 #endif
1486         return skb;
1487 }
1488
1489 /* Given the cfa_code of a received packet determine which
1490  * netdev (vf-rep or PF) the packet is destined to.
1491  */
1492 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1493 {
1494         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1495
1496         /* if vf-rep dev is NULL, the must belongs to the PF */
1497         return dev ? dev : bp->dev;
1498 }
1499
1500 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1501                                            struct bnxt_cp_ring_info *cpr,
1502                                            u32 *raw_cons,
1503                                            struct rx_tpa_end_cmp *tpa_end,
1504                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1505                                            u8 *event)
1506 {
1507         struct bnxt_napi *bnapi = cpr->bnapi;
1508         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1509         u8 *data_ptr, agg_bufs;
1510         unsigned int len;
1511         struct bnxt_tpa_info *tpa_info;
1512         dma_addr_t mapping;
1513         struct sk_buff *skb;
1514         u16 idx = 0, agg_id;
1515         void *data;
1516         bool gro;
1517
1518         if (unlikely(bnapi->in_reset)) {
1519                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1520
1521                 if (rc < 0)
1522                         return ERR_PTR(-EBUSY);
1523                 return NULL;
1524         }
1525
1526         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1527                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1528                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1529                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1530                 tpa_info = &rxr->rx_tpa[agg_id];
1531                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1532                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1533                                     agg_bufs, tpa_info->agg_count);
1534                         agg_bufs = tpa_info->agg_count;
1535                 }
1536                 tpa_info->agg_count = 0;
1537                 *event |= BNXT_AGG_EVENT;
1538                 bnxt_free_agg_idx(rxr, agg_id);
1539                 idx = agg_id;
1540                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1541         } else {
1542                 agg_id = TPA_END_AGG_ID(tpa_end);
1543                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1544                 tpa_info = &rxr->rx_tpa[agg_id];
1545                 idx = RING_CMP(*raw_cons);
1546                 if (agg_bufs) {
1547                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1548                                 return ERR_PTR(-EBUSY);
1549
1550                         *event |= BNXT_AGG_EVENT;
1551                         idx = NEXT_CMP(idx);
1552                 }
1553                 gro = !!TPA_END_GRO(tpa_end);
1554         }
1555         data = tpa_info->data;
1556         data_ptr = tpa_info->data_ptr;
1557         prefetch(data_ptr);
1558         len = tpa_info->len;
1559         mapping = tpa_info->mapping;
1560
1561         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1562                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1563                 if (agg_bufs > MAX_SKB_FRAGS)
1564                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1565                                     agg_bufs, (int)MAX_SKB_FRAGS);
1566                 return NULL;
1567         }
1568
1569         if (len <= bp->rx_copy_thresh) {
1570                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1571                 if (!skb) {
1572                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1573                         return NULL;
1574                 }
1575         } else {
1576                 u8 *new_data;
1577                 dma_addr_t new_mapping;
1578
1579                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1580                 if (!new_data) {
1581                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1582                         return NULL;
1583                 }
1584
1585                 tpa_info->data = new_data;
1586                 tpa_info->data_ptr = new_data + bp->rx_offset;
1587                 tpa_info->mapping = new_mapping;
1588
1589                 skb = build_skb(data, 0);
1590                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1591                                        bp->rx_buf_use_size, bp->rx_dir,
1592                                        DMA_ATTR_WEAK_ORDERING);
1593
1594                 if (!skb) {
1595                         kfree(data);
1596                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1597                         return NULL;
1598                 }
1599                 skb_reserve(skb, bp->rx_offset);
1600                 skb_put(skb, len);
1601         }
1602
1603         if (agg_bufs) {
1604                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1605                 if (!skb) {
1606                         /* Page reuse already handled by bnxt_rx_pages(). */
1607                         return NULL;
1608                 }
1609         }
1610
1611         skb->protocol =
1612                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1613
1614         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1615                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1616
1617         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1618             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1619                 u16 vlan_proto = tpa_info->metadata >>
1620                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1621                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1622
1623                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1624         }
1625
1626         skb_checksum_none_assert(skb);
1627         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1628                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1629                 skb->csum_level =
1630                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1631         }
1632
1633         if (gro)
1634                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1635
1636         return skb;
1637 }
1638
1639 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1640                          struct rx_agg_cmp *rx_agg)
1641 {
1642         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1643         struct bnxt_tpa_info *tpa_info;
1644
1645         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1646         tpa_info = &rxr->rx_tpa[agg_id];
1647         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1648         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1649 }
1650
1651 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1652                              struct sk_buff *skb)
1653 {
1654         if (skb->dev != bp->dev) {
1655                 /* this packet belongs to a vf-rep */
1656                 bnxt_vf_rep_rx(bp, skb);
1657                 return;
1658         }
1659         skb_record_rx_queue(skb, bnapi->index);
1660         napi_gro_receive(&bnapi->napi, skb);
1661 }
1662
1663 /* returns the following:
1664  * 1       - 1 packet successfully received
1665  * 0       - successful TPA_START, packet not completed yet
1666  * -EBUSY  - completion ring does not have all the agg buffers yet
1667  * -ENOMEM - packet aborted due to out of memory
1668  * -EIO    - packet aborted due to hw error indicated in BD
1669  */
1670 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1671                        u32 *raw_cons, u8 *event)
1672 {
1673         struct bnxt_napi *bnapi = cpr->bnapi;
1674         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1675         struct net_device *dev = bp->dev;
1676         struct rx_cmp *rxcmp;
1677         struct rx_cmp_ext *rxcmp1;
1678         u32 tmp_raw_cons = *raw_cons;
1679         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1680         struct bnxt_sw_rx_bd *rx_buf;
1681         unsigned int len;
1682         u8 *data_ptr, agg_bufs, cmp_type;
1683         dma_addr_t dma_addr;
1684         struct sk_buff *skb;
1685         void *data;
1686         int rc = 0;
1687         u32 misc;
1688
1689         rxcmp = (struct rx_cmp *)
1690                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1691
1692         cmp_type = RX_CMP_TYPE(rxcmp);
1693
1694         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1695                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1696                 goto next_rx_no_prod_no_len;
1697         }
1698
1699         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1700         cp_cons = RING_CMP(tmp_raw_cons);
1701         rxcmp1 = (struct rx_cmp_ext *)
1702                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1703
1704         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1705                 return -EBUSY;
1706
1707         prod = rxr->rx_prod;
1708
1709         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1710                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1711                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1712
1713                 *event |= BNXT_RX_EVENT;
1714                 goto next_rx_no_prod_no_len;
1715
1716         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1717                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1718                                    (struct rx_tpa_end_cmp *)rxcmp,
1719                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1720
1721                 if (IS_ERR(skb))
1722                         return -EBUSY;
1723
1724                 rc = -ENOMEM;
1725                 if (likely(skb)) {
1726                         bnxt_deliver_skb(bp, bnapi, skb);
1727                         rc = 1;
1728                 }
1729                 *event |= BNXT_RX_EVENT;
1730                 goto next_rx_no_prod_no_len;
1731         }
1732
1733         cons = rxcmp->rx_cmp_opaque;
1734         if (unlikely(cons != rxr->rx_next_cons)) {
1735                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1736
1737                 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1738                             cons, rxr->rx_next_cons);
1739                 bnxt_sched_reset(bp, rxr);
1740                 return rc1;
1741         }
1742         rx_buf = &rxr->rx_buf_ring[cons];
1743         data = rx_buf->data;
1744         data_ptr = rx_buf->data_ptr;
1745         prefetch(data_ptr);
1746
1747         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1748         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1749
1750         if (agg_bufs) {
1751                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1752                         return -EBUSY;
1753
1754                 cp_cons = NEXT_CMP(cp_cons);
1755                 *event |= BNXT_AGG_EVENT;
1756         }
1757         *event |= BNXT_RX_EVENT;
1758
1759         rx_buf->data = NULL;
1760         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1761                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1762
1763                 bnxt_reuse_rx_data(rxr, cons, data);
1764                 if (agg_bufs)
1765                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1766                                                false);
1767
1768                 rc = -EIO;
1769                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1770                         bnapi->cp_ring.rx_buf_errors++;
1771                         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
1772                                 netdev_warn(bp->dev, "RX buffer error %x\n",
1773                                             rx_err);
1774                                 bnxt_sched_reset(bp, rxr);
1775                         }
1776                 }
1777                 goto next_rx_no_len;
1778         }
1779
1780         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1781         dma_addr = rx_buf->mapping;
1782
1783         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1784                 rc = 1;
1785                 goto next_rx;
1786         }
1787
1788         if (len <= bp->rx_copy_thresh) {
1789                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1790                 bnxt_reuse_rx_data(rxr, cons, data);
1791                 if (!skb) {
1792                         if (agg_bufs)
1793                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1794                                                        agg_bufs, false);
1795                         rc = -ENOMEM;
1796                         goto next_rx;
1797                 }
1798         } else {
1799                 u32 payload;
1800
1801                 if (rx_buf->data_ptr == data_ptr)
1802                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1803                 else
1804                         payload = 0;
1805                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1806                                       payload | len);
1807                 if (!skb) {
1808                         rc = -ENOMEM;
1809                         goto next_rx;
1810                 }
1811         }
1812
1813         if (agg_bufs) {
1814                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1815                 if (!skb) {
1816                         rc = -ENOMEM;
1817                         goto next_rx;
1818                 }
1819         }
1820
1821         if (RX_CMP_HASH_VALID(rxcmp)) {
1822                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1823                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1824
1825                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1826                 if (hash_type != 1 && hash_type != 3)
1827                         type = PKT_HASH_TYPE_L3;
1828                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1829         }
1830
1831         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1832         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1833
1834         if ((rxcmp1->rx_cmp_flags2 &
1835              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1836             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1837                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1838                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1839                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1840
1841                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1842         }
1843
1844         skb_checksum_none_assert(skb);
1845         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1846                 if (dev->features & NETIF_F_RXCSUM) {
1847                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1848                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1849                 }
1850         } else {
1851                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1852                         if (dev->features & NETIF_F_RXCSUM)
1853                                 bnapi->cp_ring.rx_l4_csum_errors++;
1854                 }
1855         }
1856
1857         bnxt_deliver_skb(bp, bnapi, skb);
1858         rc = 1;
1859
1860 next_rx:
1861         cpr->rx_packets += 1;
1862         cpr->rx_bytes += len;
1863
1864 next_rx_no_len:
1865         rxr->rx_prod = NEXT_RX(prod);
1866         rxr->rx_next_cons = NEXT_RX(cons);
1867
1868 next_rx_no_prod_no_len:
1869         *raw_cons = tmp_raw_cons;
1870
1871         return rc;
1872 }
1873
1874 /* In netpoll mode, if we are using a combined completion ring, we need to
1875  * discard the rx packets and recycle the buffers.
1876  */
1877 static int bnxt_force_rx_discard(struct bnxt *bp,
1878                                  struct bnxt_cp_ring_info *cpr,
1879                                  u32 *raw_cons, u8 *event)
1880 {
1881         u32 tmp_raw_cons = *raw_cons;
1882         struct rx_cmp_ext *rxcmp1;
1883         struct rx_cmp *rxcmp;
1884         u16 cp_cons;
1885         u8 cmp_type;
1886
1887         cp_cons = RING_CMP(tmp_raw_cons);
1888         rxcmp = (struct rx_cmp *)
1889                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1890
1891         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1892         cp_cons = RING_CMP(tmp_raw_cons);
1893         rxcmp1 = (struct rx_cmp_ext *)
1894                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1895
1896         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1897                 return -EBUSY;
1898
1899         cmp_type = RX_CMP_TYPE(rxcmp);
1900         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1901                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1902                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1903         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1904                 struct rx_tpa_end_cmp_ext *tpa_end1;
1905
1906                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1907                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1908                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1909         }
1910         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1911 }
1912
1913 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1914 {
1915         struct bnxt_fw_health *fw_health = bp->fw_health;
1916         u32 reg = fw_health->regs[reg_idx];
1917         u32 reg_type, reg_off, val = 0;
1918
1919         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1920         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1921         switch (reg_type) {
1922         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1923                 pci_read_config_dword(bp->pdev, reg_off, &val);
1924                 break;
1925         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1926                 reg_off = fw_health->mapped_regs[reg_idx];
1927                 /* fall through */
1928         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1929                 val = readl(bp->bar0 + reg_off);
1930                 break;
1931         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1932                 val = readl(bp->bar1 + reg_off);
1933                 break;
1934         }
1935         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1936                 val &= fw_health->fw_reset_inprog_reg_mask;
1937         return val;
1938 }
1939
1940 #define BNXT_GET_EVENT_PORT(data)       \
1941         ((data) &                       \
1942          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1943
1944 static int bnxt_async_event_process(struct bnxt *bp,
1945                                     struct hwrm_async_event_cmpl *cmpl)
1946 {
1947         u16 event_id = le16_to_cpu(cmpl->event_id);
1948
1949         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1950         switch (event_id) {
1951         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1952                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1953                 struct bnxt_link_info *link_info = &bp->link_info;
1954
1955                 if (BNXT_VF(bp))
1956                         goto async_event_process_exit;
1957
1958                 /* print unsupported speed warning in forced speed mode only */
1959                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1960                     (data1 & 0x20000)) {
1961                         u16 fw_speed = link_info->force_link_speed;
1962                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1963
1964                         if (speed != SPEED_UNKNOWN)
1965                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1966                                             speed);
1967                 }
1968                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1969         }
1970         /* fall through */
1971         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1972                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1973                 break;
1974         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1975                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1976                 break;
1977         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1978                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1979                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1980
1981                 if (BNXT_VF(bp))
1982                         break;
1983
1984                 if (bp->pf.port_id != port_id)
1985                         break;
1986
1987                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1988                 break;
1989         }
1990         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1991                 if (BNXT_PF(bp))
1992                         goto async_event_process_exit;
1993                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1994                 break;
1995         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
1996                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1997
1998                 bp->fw_reset_timestamp = jiffies;
1999                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2000                 if (!bp->fw_reset_min_dsecs)
2001                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2002                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2003                 if (!bp->fw_reset_max_dsecs)
2004                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2005                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2006                         netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2007                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2008                 } else {
2009                         netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2010                                     bp->fw_reset_max_dsecs * 100);
2011                 }
2012                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2013                 break;
2014         }
2015         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2016                 struct bnxt_fw_health *fw_health = bp->fw_health;
2017                 u32 data1 = le32_to_cpu(cmpl->event_data1);
2018
2019                 if (!fw_health)
2020                         goto async_event_process_exit;
2021
2022                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2023                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2024                 if (!fw_health->enabled)
2025                         break;
2026
2027                 if (netif_msg_drv(bp))
2028                         netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2029                                     fw_health->enabled, fw_health->master,
2030                                     bnxt_fw_health_readl(bp,
2031                                                          BNXT_FW_RESET_CNT_REG),
2032                                     bnxt_fw_health_readl(bp,
2033                                                          BNXT_FW_HEALTH_REG));
2034                 fw_health->tmr_multiplier =
2035                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2036                                      bp->current_interval * 10);
2037                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2038                 fw_health->last_fw_heartbeat =
2039                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2040                 fw_health->last_fw_reset_cnt =
2041                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2042                 goto async_event_process_exit;
2043         }
2044         default:
2045                 goto async_event_process_exit;
2046         }
2047         bnxt_queue_sp_work(bp);
2048 async_event_process_exit:
2049         bnxt_ulp_async_events(bp, cmpl);
2050         return 0;
2051 }
2052
2053 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2054 {
2055         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2056         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2057         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2058                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2059
2060         switch (cmpl_type) {
2061         case CMPL_BASE_TYPE_HWRM_DONE:
2062                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2063                 if (seq_id == bp->hwrm_intr_seq_id)
2064                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2065                 else
2066                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2067                 break;
2068
2069         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2070                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2071
2072                 if ((vf_id < bp->pf.first_vf_id) ||
2073                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2074                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2075                                    vf_id);
2076                         return -EINVAL;
2077                 }
2078
2079                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2080                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2081                 bnxt_queue_sp_work(bp);
2082                 break;
2083
2084         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2085                 bnxt_async_event_process(bp,
2086                                          (struct hwrm_async_event_cmpl *)txcmp);
2087
2088         default:
2089                 break;
2090         }
2091
2092         return 0;
2093 }
2094
2095 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2096 {
2097         struct bnxt_napi *bnapi = dev_instance;
2098         struct bnxt *bp = bnapi->bp;
2099         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2100         u32 cons = RING_CMP(cpr->cp_raw_cons);
2101
2102         cpr->event_ctr++;
2103         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2104         napi_schedule(&bnapi->napi);
2105         return IRQ_HANDLED;
2106 }
2107
2108 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2109 {
2110         u32 raw_cons = cpr->cp_raw_cons;
2111         u16 cons = RING_CMP(raw_cons);
2112         struct tx_cmp *txcmp;
2113
2114         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2115
2116         return TX_CMP_VALID(txcmp, raw_cons);
2117 }
2118
2119 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2120 {
2121         struct bnxt_napi *bnapi = dev_instance;
2122         struct bnxt *bp = bnapi->bp;
2123         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2124         u32 cons = RING_CMP(cpr->cp_raw_cons);
2125         u32 int_status;
2126
2127         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2128
2129         if (!bnxt_has_work(bp, cpr)) {
2130                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2131                 /* return if erroneous interrupt */
2132                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2133                         return IRQ_NONE;
2134         }
2135
2136         /* disable ring IRQ */
2137         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2138
2139         /* Return here if interrupt is shared and is disabled. */
2140         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2141                 return IRQ_HANDLED;
2142
2143         napi_schedule(&bnapi->napi);
2144         return IRQ_HANDLED;
2145 }
2146
2147 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2148                             int budget)
2149 {
2150         struct bnxt_napi *bnapi = cpr->bnapi;
2151         u32 raw_cons = cpr->cp_raw_cons;
2152         u32 cons;
2153         int tx_pkts = 0;
2154         int rx_pkts = 0;
2155         u8 event = 0;
2156         struct tx_cmp *txcmp;
2157
2158         cpr->has_more_work = 0;
2159         while (1) {
2160                 int rc;
2161
2162                 cons = RING_CMP(raw_cons);
2163                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2164
2165                 if (!TX_CMP_VALID(txcmp, raw_cons))
2166                         break;
2167
2168                 /* The valid test of the entry must be done first before
2169                  * reading any further.
2170                  */
2171                 dma_rmb();
2172                 cpr->had_work_done = 1;
2173                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2174                         tx_pkts++;
2175                         /* return full budget so NAPI will complete. */
2176                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2177                                 rx_pkts = budget;
2178                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2179                                 if (budget)
2180                                         cpr->has_more_work = 1;
2181                                 break;
2182                         }
2183                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2184                         if (likely(budget))
2185                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2186                         else
2187                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2188                                                            &event);
2189                         if (likely(rc >= 0))
2190                                 rx_pkts += rc;
2191                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2192                          * the NAPI budget.  Otherwise, we may potentially loop
2193                          * here forever if we consistently cannot allocate
2194                          * buffers.
2195                          */
2196                         else if (rc == -ENOMEM && budget)
2197                                 rx_pkts++;
2198                         else if (rc == -EBUSY)  /* partial completion */
2199                                 break;
2200                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2201                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2202                                     (TX_CMP_TYPE(txcmp) ==
2203                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2204                                     (TX_CMP_TYPE(txcmp) ==
2205                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2206                         bnxt_hwrm_handler(bp, txcmp);
2207                 }
2208                 raw_cons = NEXT_RAW_CMP(raw_cons);
2209
2210                 if (rx_pkts && rx_pkts == budget) {
2211                         cpr->has_more_work = 1;
2212                         break;
2213                 }
2214         }
2215
2216         if (event & BNXT_REDIRECT_EVENT)
2217                 xdp_do_flush_map();
2218
2219         if (event & BNXT_TX_EVENT) {
2220                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2221                 u16 prod = txr->tx_prod;
2222
2223                 /* Sync BD data before updating doorbell */
2224                 wmb();
2225
2226                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2227         }
2228
2229         cpr->cp_raw_cons = raw_cons;
2230         bnapi->tx_pkts += tx_pkts;
2231         bnapi->events |= event;
2232         return rx_pkts;
2233 }
2234
2235 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2236 {
2237         if (bnapi->tx_pkts) {
2238                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2239                 bnapi->tx_pkts = 0;
2240         }
2241
2242         if (bnapi->events & BNXT_RX_EVENT) {
2243                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2244
2245                 if (bnapi->events & BNXT_AGG_EVENT)
2246                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2247                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2248         }
2249         bnapi->events = 0;
2250 }
2251
2252 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2253                           int budget)
2254 {
2255         struct bnxt_napi *bnapi = cpr->bnapi;
2256         int rx_pkts;
2257
2258         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2259
2260         /* ACK completion ring before freeing tx ring and producing new
2261          * buffers in rx/agg rings to prevent overflowing the completion
2262          * ring.
2263          */
2264         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2265
2266         __bnxt_poll_work_done(bp, bnapi);
2267         return rx_pkts;
2268 }
2269
2270 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2271 {
2272         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2273         struct bnxt *bp = bnapi->bp;
2274         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2275         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2276         struct tx_cmp *txcmp;
2277         struct rx_cmp_ext *rxcmp1;
2278         u32 cp_cons, tmp_raw_cons;
2279         u32 raw_cons = cpr->cp_raw_cons;
2280         u32 rx_pkts = 0;
2281         u8 event = 0;
2282
2283         while (1) {
2284                 int rc;
2285
2286                 cp_cons = RING_CMP(raw_cons);
2287                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2288
2289                 if (!TX_CMP_VALID(txcmp, raw_cons))
2290                         break;
2291
2292                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2293                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2294                         cp_cons = RING_CMP(tmp_raw_cons);
2295                         rxcmp1 = (struct rx_cmp_ext *)
2296                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2297
2298                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2299                                 break;
2300
2301                         /* force an error to recycle the buffer */
2302                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2303                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2304
2305                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2306                         if (likely(rc == -EIO) && budget)
2307                                 rx_pkts++;
2308                         else if (rc == -EBUSY)  /* partial completion */
2309                                 break;
2310                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2311                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2312                         bnxt_hwrm_handler(bp, txcmp);
2313                 } else {
2314                         netdev_err(bp->dev,
2315                                    "Invalid completion received on special ring\n");
2316                 }
2317                 raw_cons = NEXT_RAW_CMP(raw_cons);
2318
2319                 if (rx_pkts == budget)
2320                         break;
2321         }
2322
2323         cpr->cp_raw_cons = raw_cons;
2324         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2325         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2326
2327         if (event & BNXT_AGG_EVENT)
2328                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2329
2330         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2331                 napi_complete_done(napi, rx_pkts);
2332                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2333         }
2334         return rx_pkts;
2335 }
2336
2337 static int bnxt_poll(struct napi_struct *napi, int budget)
2338 {
2339         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2340         struct bnxt *bp = bnapi->bp;
2341         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2342         int work_done = 0;
2343
2344         while (1) {
2345                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2346
2347                 if (work_done >= budget) {
2348                         if (!budget)
2349                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2350                         break;
2351                 }
2352
2353                 if (!bnxt_has_work(bp, cpr)) {
2354                         if (napi_complete_done(napi, work_done))
2355                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2356                         break;
2357                 }
2358         }
2359         if (bp->flags & BNXT_FLAG_DIM) {
2360                 struct dim_sample dim_sample = {};
2361
2362                 dim_update_sample(cpr->event_ctr,
2363                                   cpr->rx_packets,
2364                                   cpr->rx_bytes,
2365                                   &dim_sample);
2366                 net_dim(&cpr->dim, dim_sample);
2367         }
2368         return work_done;
2369 }
2370
2371 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2372 {
2373         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2374         int i, work_done = 0;
2375
2376         for (i = 0; i < 2; i++) {
2377                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2378
2379                 if (cpr2) {
2380                         work_done += __bnxt_poll_work(bp, cpr2,
2381                                                       budget - work_done);
2382                         cpr->has_more_work |= cpr2->has_more_work;
2383                 }
2384         }
2385         return work_done;
2386 }
2387
2388 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2389                                  u64 dbr_type, bool all)
2390 {
2391         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2392         int i;
2393
2394         for (i = 0; i < 2; i++) {
2395                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2396                 struct bnxt_db_info *db;
2397
2398                 if (cpr2 && (all || cpr2->had_work_done)) {
2399                         db = &cpr2->cp_db;
2400                         writeq(db->db_key64 | dbr_type |
2401                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2402                         cpr2->had_work_done = 0;
2403                 }
2404         }
2405         __bnxt_poll_work_done(bp, bnapi);
2406 }
2407
2408 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2409 {
2410         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2411         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2412         u32 raw_cons = cpr->cp_raw_cons;
2413         struct bnxt *bp = bnapi->bp;
2414         struct nqe_cn *nqcmp;
2415         int work_done = 0;
2416         u32 cons;
2417
2418         if (cpr->has_more_work) {
2419                 cpr->has_more_work = 0;
2420                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2421                 if (cpr->has_more_work) {
2422                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2423                         return work_done;
2424                 }
2425                 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2426                 if (napi_complete_done(napi, work_done))
2427                         BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2428                 return work_done;
2429         }
2430         while (1) {
2431                 cons = RING_CMP(raw_cons);
2432                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2433
2434                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2435                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2436                                              false);
2437                         cpr->cp_raw_cons = raw_cons;
2438                         if (napi_complete_done(napi, work_done))
2439                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2440                                                   cpr->cp_raw_cons);
2441                         return work_done;
2442                 }
2443
2444                 /* The valid test of the entry must be done first before
2445                  * reading any further.
2446                  */
2447                 dma_rmb();
2448
2449                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2450                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2451                         struct bnxt_cp_ring_info *cpr2;
2452
2453                         cpr2 = cpr->cp_ring_arr[idx];
2454                         work_done += __bnxt_poll_work(bp, cpr2,
2455                                                       budget - work_done);
2456                         cpr->has_more_work = cpr2->has_more_work;
2457                 } else {
2458                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2459                 }
2460                 raw_cons = NEXT_RAW_CMP(raw_cons);
2461                 if (cpr->has_more_work)
2462                         break;
2463         }
2464         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2465         cpr->cp_raw_cons = raw_cons;
2466         return work_done;
2467 }
2468
2469 static void bnxt_free_tx_skbs(struct bnxt *bp)
2470 {
2471         int i, max_idx;
2472         struct pci_dev *pdev = bp->pdev;
2473
2474         if (!bp->tx_ring)
2475                 return;
2476
2477         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2478         for (i = 0; i < bp->tx_nr_rings; i++) {
2479                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2480                 int j;
2481
2482                 for (j = 0; j < max_idx;) {
2483                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2484                         struct sk_buff *skb;
2485                         int k, last;
2486
2487                         if (i < bp->tx_nr_rings_xdp &&
2488                             tx_buf->action == XDP_REDIRECT) {
2489                                 dma_unmap_single(&pdev->dev,
2490                                         dma_unmap_addr(tx_buf, mapping),
2491                                         dma_unmap_len(tx_buf, len),
2492                                         PCI_DMA_TODEVICE);
2493                                 xdp_return_frame(tx_buf->xdpf);
2494                                 tx_buf->action = 0;
2495                                 tx_buf->xdpf = NULL;
2496                                 j++;
2497                                 continue;
2498                         }
2499
2500                         skb = tx_buf->skb;
2501                         if (!skb) {
2502                                 j++;
2503                                 continue;
2504                         }
2505
2506                         tx_buf->skb = NULL;
2507
2508                         if (tx_buf->is_push) {
2509                                 dev_kfree_skb(skb);
2510                                 j += 2;
2511                                 continue;
2512                         }
2513
2514                         dma_unmap_single(&pdev->dev,
2515                                          dma_unmap_addr(tx_buf, mapping),
2516                                          skb_headlen(skb),
2517                                          PCI_DMA_TODEVICE);
2518
2519                         last = tx_buf->nr_frags;
2520                         j += 2;
2521                         for (k = 0; k < last; k++, j++) {
2522                                 int ring_idx = j & bp->tx_ring_mask;
2523                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2524
2525                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2526                                 dma_unmap_page(
2527                                         &pdev->dev,
2528                                         dma_unmap_addr(tx_buf, mapping),
2529                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2530                         }
2531                         dev_kfree_skb(skb);
2532                 }
2533                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2534         }
2535 }
2536
2537 static void bnxt_free_rx_skbs(struct bnxt *bp)
2538 {
2539         int i, max_idx, max_agg_idx;
2540         struct pci_dev *pdev = bp->pdev;
2541
2542         if (!bp->rx_ring)
2543                 return;
2544
2545         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2546         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2547         for (i = 0; i < bp->rx_nr_rings; i++) {
2548                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2549                 struct bnxt_tpa_idx_map *map;
2550                 int j;
2551
2552                 if (rxr->rx_tpa) {
2553                         for (j = 0; j < bp->max_tpa; j++) {
2554                                 struct bnxt_tpa_info *tpa_info =
2555                                                         &rxr->rx_tpa[j];
2556                                 u8 *data = tpa_info->data;
2557
2558                                 if (!data)
2559                                         continue;
2560
2561                                 dma_unmap_single_attrs(&pdev->dev,
2562                                                        tpa_info->mapping,
2563                                                        bp->rx_buf_use_size,
2564                                                        bp->rx_dir,
2565                                                        DMA_ATTR_WEAK_ORDERING);
2566
2567                                 tpa_info->data = NULL;
2568
2569                                 kfree(data);
2570                         }
2571                 }
2572
2573                 for (j = 0; j < max_idx; j++) {
2574                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2575                         dma_addr_t mapping = rx_buf->mapping;
2576                         void *data = rx_buf->data;
2577
2578                         if (!data)
2579                                 continue;
2580
2581                         rx_buf->data = NULL;
2582
2583                         if (BNXT_RX_PAGE_MODE(bp)) {
2584                                 mapping -= bp->rx_dma_offset;
2585                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2586                                                      PAGE_SIZE, bp->rx_dir,
2587                                                      DMA_ATTR_WEAK_ORDERING);
2588                                 page_pool_recycle_direct(rxr->page_pool, data);
2589                         } else {
2590                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2591                                                        bp->rx_buf_use_size,
2592                                                        bp->rx_dir,
2593                                                        DMA_ATTR_WEAK_ORDERING);
2594                                 kfree(data);
2595                         }
2596                 }
2597
2598                 for (j = 0; j < max_agg_idx; j++) {
2599                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2600                                 &rxr->rx_agg_ring[j];
2601                         struct page *page = rx_agg_buf->page;
2602
2603                         if (!page)
2604                                 continue;
2605
2606                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2607                                              BNXT_RX_PAGE_SIZE,
2608                                              PCI_DMA_FROMDEVICE,
2609                                              DMA_ATTR_WEAK_ORDERING);
2610
2611                         rx_agg_buf->page = NULL;
2612                         __clear_bit(j, rxr->rx_agg_bmap);
2613
2614                         __free_page(page);
2615                 }
2616                 if (rxr->rx_page) {
2617                         __free_page(rxr->rx_page);
2618                         rxr->rx_page = NULL;
2619                 }
2620                 map = rxr->rx_tpa_idx_map;
2621                 if (map)
2622                         memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2623         }
2624 }
2625
2626 static void bnxt_free_skbs(struct bnxt *bp)
2627 {
2628         bnxt_free_tx_skbs(bp);
2629         bnxt_free_rx_skbs(bp);
2630 }
2631
2632 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2633 {
2634         struct pci_dev *pdev = bp->pdev;
2635         int i;
2636
2637         for (i = 0; i < rmem->nr_pages; i++) {
2638                 if (!rmem->pg_arr[i])
2639                         continue;
2640
2641                 dma_free_coherent(&pdev->dev, rmem->page_size,
2642                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2643
2644                 rmem->pg_arr[i] = NULL;
2645         }
2646         if (rmem->pg_tbl) {
2647                 size_t pg_tbl_size = rmem->nr_pages * 8;
2648
2649                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2650                         pg_tbl_size = rmem->page_size;
2651                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2652                                   rmem->pg_tbl, rmem->pg_tbl_map);
2653                 rmem->pg_tbl = NULL;
2654         }
2655         if (rmem->vmem_size && *rmem->vmem) {
2656                 vfree(*rmem->vmem);
2657                 *rmem->vmem = NULL;
2658         }
2659 }
2660
2661 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2662 {
2663         struct pci_dev *pdev = bp->pdev;
2664         u64 valid_bit = 0;
2665         int i;
2666
2667         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2668                 valid_bit = PTU_PTE_VALID;
2669         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2670                 size_t pg_tbl_size = rmem->nr_pages * 8;
2671
2672                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2673                         pg_tbl_size = rmem->page_size;
2674                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2675                                                   &rmem->pg_tbl_map,
2676                                                   GFP_KERNEL);
2677                 if (!rmem->pg_tbl)
2678                         return -ENOMEM;
2679         }
2680
2681         for (i = 0; i < rmem->nr_pages; i++) {
2682                 u64 extra_bits = valid_bit;
2683
2684                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2685                                                      rmem->page_size,
2686                                                      &rmem->dma_arr[i],
2687                                                      GFP_KERNEL);
2688                 if (!rmem->pg_arr[i])
2689                         return -ENOMEM;
2690
2691                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2692                         if (i == rmem->nr_pages - 2 &&
2693                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2694                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2695                         else if (i == rmem->nr_pages - 1 &&
2696                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2697                                 extra_bits |= PTU_PTE_LAST;
2698                         rmem->pg_tbl[i] =
2699                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2700                 }
2701         }
2702
2703         if (rmem->vmem_size) {
2704                 *rmem->vmem = vzalloc(rmem->vmem_size);
2705                 if (!(*rmem->vmem))
2706                         return -ENOMEM;
2707         }
2708         return 0;
2709 }
2710
2711 static void bnxt_free_tpa_info(struct bnxt *bp)
2712 {
2713         int i;
2714
2715         for (i = 0; i < bp->rx_nr_rings; i++) {
2716                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2717
2718                 kfree(rxr->rx_tpa_idx_map);
2719                 rxr->rx_tpa_idx_map = NULL;
2720                 if (rxr->rx_tpa) {
2721                         kfree(rxr->rx_tpa[0].agg_arr);
2722                         rxr->rx_tpa[0].agg_arr = NULL;
2723                 }
2724                 kfree(rxr->rx_tpa);
2725                 rxr->rx_tpa = NULL;
2726         }
2727 }
2728
2729 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2730 {
2731         int i, j, total_aggs = 0;
2732
2733         bp->max_tpa = MAX_TPA;
2734         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2735                 if (!bp->max_tpa_v2)
2736                         return 0;
2737                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2738                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2739         }
2740
2741         for (i = 0; i < bp->rx_nr_rings; i++) {
2742                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2743                 struct rx_agg_cmp *agg;
2744
2745                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2746                                       GFP_KERNEL);
2747                 if (!rxr->rx_tpa)
2748                         return -ENOMEM;
2749
2750                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2751                         continue;
2752                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2753                 rxr->rx_tpa[0].agg_arr = agg;
2754                 if (!agg)
2755                         return -ENOMEM;
2756                 for (j = 1; j < bp->max_tpa; j++)
2757                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2758                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2759                                               GFP_KERNEL);
2760                 if (!rxr->rx_tpa_idx_map)
2761                         return -ENOMEM;
2762         }
2763         return 0;
2764 }
2765
2766 static void bnxt_free_rx_rings(struct bnxt *bp)
2767 {
2768         int i;
2769
2770         if (!bp->rx_ring)
2771                 return;
2772
2773         bnxt_free_tpa_info(bp);
2774         for (i = 0; i < bp->rx_nr_rings; i++) {
2775                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2776                 struct bnxt_ring_struct *ring;
2777
2778                 if (rxr->xdp_prog)
2779                         bpf_prog_put(rxr->xdp_prog);
2780
2781                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2782                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2783
2784                 page_pool_destroy(rxr->page_pool);
2785                 rxr->page_pool = NULL;
2786
2787                 kfree(rxr->rx_agg_bmap);
2788                 rxr->rx_agg_bmap = NULL;
2789
2790                 ring = &rxr->rx_ring_struct;
2791                 bnxt_free_ring(bp, &ring->ring_mem);
2792
2793                 ring = &rxr->rx_agg_ring_struct;
2794                 bnxt_free_ring(bp, &ring->ring_mem);
2795         }
2796 }
2797
2798 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2799                                    struct bnxt_rx_ring_info *rxr)
2800 {
2801         struct page_pool_params pp = { 0 };
2802
2803         pp.pool_size = bp->rx_ring_size;
2804         pp.nid = dev_to_node(&bp->pdev->dev);
2805         pp.dev = &bp->pdev->dev;
2806         pp.dma_dir = DMA_BIDIRECTIONAL;
2807
2808         rxr->page_pool = page_pool_create(&pp);
2809         if (IS_ERR(rxr->page_pool)) {
2810                 int err = PTR_ERR(rxr->page_pool);
2811
2812                 rxr->page_pool = NULL;
2813                 return err;
2814         }
2815         return 0;
2816 }
2817
2818 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2819 {
2820         int i, rc = 0, agg_rings = 0;
2821
2822         if (!bp->rx_ring)
2823                 return -ENOMEM;
2824
2825         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2826                 agg_rings = 1;
2827
2828         for (i = 0; i < bp->rx_nr_rings; i++) {
2829                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2830                 struct bnxt_ring_struct *ring;
2831
2832                 ring = &rxr->rx_ring_struct;
2833
2834                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2835                 if (rc)
2836                         return rc;
2837
2838                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2839                 if (rc < 0)
2840                         return rc;
2841
2842                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2843                                                 MEM_TYPE_PAGE_POOL,
2844                                                 rxr->page_pool);
2845                 if (rc) {
2846                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2847                         return rc;
2848                 }
2849
2850                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2851                 if (rc)
2852                         return rc;
2853
2854                 ring->grp_idx = i;
2855                 if (agg_rings) {
2856                         u16 mem_size;
2857
2858                         ring = &rxr->rx_agg_ring_struct;
2859                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2860                         if (rc)
2861                                 return rc;
2862
2863                         ring->grp_idx = i;
2864                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2865                         mem_size = rxr->rx_agg_bmap_size / 8;
2866                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2867                         if (!rxr->rx_agg_bmap)
2868                                 return -ENOMEM;
2869                 }
2870         }
2871         if (bp->flags & BNXT_FLAG_TPA)
2872                 rc = bnxt_alloc_tpa_info(bp);
2873         return rc;
2874 }
2875
2876 static void bnxt_free_tx_rings(struct bnxt *bp)
2877 {
2878         int i;
2879         struct pci_dev *pdev = bp->pdev;
2880
2881         if (!bp->tx_ring)
2882                 return;
2883
2884         for (i = 0; i < bp->tx_nr_rings; i++) {
2885                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2886                 struct bnxt_ring_struct *ring;
2887
2888                 if (txr->tx_push) {
2889                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2890                                           txr->tx_push, txr->tx_push_mapping);
2891                         txr->tx_push = NULL;
2892                 }
2893
2894                 ring = &txr->tx_ring_struct;
2895
2896                 bnxt_free_ring(bp, &ring->ring_mem);
2897         }
2898 }
2899
2900 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2901 {
2902         int i, j, rc;
2903         struct pci_dev *pdev = bp->pdev;
2904
2905         bp->tx_push_size = 0;
2906         if (bp->tx_push_thresh) {
2907                 int push_size;
2908
2909                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2910                                         bp->tx_push_thresh);
2911
2912                 if (push_size > 256) {
2913                         push_size = 0;
2914                         bp->tx_push_thresh = 0;
2915                 }
2916
2917                 bp->tx_push_size = push_size;
2918         }
2919
2920         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2921                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2922                 struct bnxt_ring_struct *ring;
2923                 u8 qidx;
2924
2925                 ring = &txr->tx_ring_struct;
2926
2927                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2928                 if (rc)
2929                         return rc;
2930
2931                 ring->grp_idx = txr->bnapi->index;
2932                 if (bp->tx_push_size) {
2933                         dma_addr_t mapping;
2934
2935                         /* One pre-allocated DMA buffer to backup
2936                          * TX push operation
2937                          */
2938                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2939                                                 bp->tx_push_size,
2940                                                 &txr->tx_push_mapping,
2941                                                 GFP_KERNEL);
2942
2943                         if (!txr->tx_push)
2944                                 return -ENOMEM;
2945
2946                         mapping = txr->tx_push_mapping +
2947                                 sizeof(struct tx_push_bd);
2948                         txr->data_mapping = cpu_to_le64(mapping);
2949                 }
2950                 qidx = bp->tc_to_qidx[j];
2951                 ring->queue_id = bp->q_info[qidx].queue_id;
2952                 if (i < bp->tx_nr_rings_xdp)
2953                         continue;
2954                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2955                         j++;
2956         }
2957         return 0;
2958 }
2959
2960 static void bnxt_free_cp_rings(struct bnxt *bp)
2961 {
2962         int i;
2963
2964         if (!bp->bnapi)
2965                 return;
2966
2967         for (i = 0; i < bp->cp_nr_rings; i++) {
2968                 struct bnxt_napi *bnapi = bp->bnapi[i];
2969                 struct bnxt_cp_ring_info *cpr;
2970                 struct bnxt_ring_struct *ring;
2971                 int j;
2972
2973                 if (!bnapi)
2974                         continue;
2975
2976                 cpr = &bnapi->cp_ring;
2977                 ring = &cpr->cp_ring_struct;
2978
2979                 bnxt_free_ring(bp, &ring->ring_mem);
2980
2981                 for (j = 0; j < 2; j++) {
2982                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2983
2984                         if (cpr2) {
2985                                 ring = &cpr2->cp_ring_struct;
2986                                 bnxt_free_ring(bp, &ring->ring_mem);
2987                                 kfree(cpr2);
2988                                 cpr->cp_ring_arr[j] = NULL;
2989                         }
2990                 }
2991         }
2992 }
2993
2994 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2995 {
2996         struct bnxt_ring_mem_info *rmem;
2997         struct bnxt_ring_struct *ring;
2998         struct bnxt_cp_ring_info *cpr;
2999         int rc;
3000
3001         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3002         if (!cpr)
3003                 return NULL;
3004
3005         ring = &cpr->cp_ring_struct;
3006         rmem = &ring->ring_mem;
3007         rmem->nr_pages = bp->cp_nr_pages;
3008         rmem->page_size = HW_CMPD_RING_SIZE;
3009         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3010         rmem->dma_arr = cpr->cp_desc_mapping;
3011         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3012         rc = bnxt_alloc_ring(bp, rmem);
3013         if (rc) {
3014                 bnxt_free_ring(bp, rmem);
3015                 kfree(cpr);
3016                 cpr = NULL;
3017         }
3018         return cpr;
3019 }
3020
3021 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3022 {
3023         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3024         int i, rc, ulp_base_vec, ulp_msix;
3025
3026         ulp_msix = bnxt_get_ulp_msix_num(bp);
3027         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3028         for (i = 0; i < bp->cp_nr_rings; i++) {
3029                 struct bnxt_napi *bnapi = bp->bnapi[i];
3030                 struct bnxt_cp_ring_info *cpr;
3031                 struct bnxt_ring_struct *ring;
3032
3033                 if (!bnapi)
3034                         continue;
3035
3036                 cpr = &bnapi->cp_ring;
3037                 cpr->bnapi = bnapi;
3038                 ring = &cpr->cp_ring_struct;
3039
3040                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3041                 if (rc)
3042                         return rc;
3043
3044                 if (ulp_msix && i >= ulp_base_vec)
3045                         ring->map_idx = i + ulp_msix;
3046                 else
3047                         ring->map_idx = i;
3048
3049                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3050                         continue;
3051
3052                 if (i < bp->rx_nr_rings) {
3053                         struct bnxt_cp_ring_info *cpr2 =
3054                                 bnxt_alloc_cp_sub_ring(bp);
3055
3056                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3057                         if (!cpr2)
3058                                 return -ENOMEM;
3059                         cpr2->bnapi = bnapi;
3060                 }
3061                 if ((sh && i < bp->tx_nr_rings) ||
3062                     (!sh && i >= bp->rx_nr_rings)) {
3063                         struct bnxt_cp_ring_info *cpr2 =
3064                                 bnxt_alloc_cp_sub_ring(bp);
3065
3066                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3067                         if (!cpr2)
3068                                 return -ENOMEM;
3069                         cpr2->bnapi = bnapi;
3070                 }
3071         }
3072         return 0;
3073 }
3074
3075 static void bnxt_init_ring_struct(struct bnxt *bp)
3076 {
3077         int i;
3078
3079         for (i = 0; i < bp->cp_nr_rings; i++) {
3080                 struct bnxt_napi *bnapi = bp->bnapi[i];
3081                 struct bnxt_ring_mem_info *rmem;
3082                 struct bnxt_cp_ring_info *cpr;
3083                 struct bnxt_rx_ring_info *rxr;
3084                 struct bnxt_tx_ring_info *txr;
3085                 struct bnxt_ring_struct *ring;
3086
3087                 if (!bnapi)
3088                         continue;
3089
3090                 cpr = &bnapi->cp_ring;
3091                 ring = &cpr->cp_ring_struct;
3092                 rmem = &ring->ring_mem;
3093                 rmem->nr_pages = bp->cp_nr_pages;
3094                 rmem->page_size = HW_CMPD_RING_SIZE;
3095                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3096                 rmem->dma_arr = cpr->cp_desc_mapping;
3097                 rmem->vmem_size = 0;
3098
3099                 rxr = bnapi->rx_ring;
3100                 if (!rxr)
3101                         goto skip_rx;
3102
3103                 ring = &rxr->rx_ring_struct;
3104                 rmem = &ring->ring_mem;
3105                 rmem->nr_pages = bp->rx_nr_pages;
3106                 rmem->page_size = HW_RXBD_RING_SIZE;
3107                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3108                 rmem->dma_arr = rxr->rx_desc_mapping;
3109                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3110                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3111
3112                 ring = &rxr->rx_agg_ring_struct;
3113                 rmem = &ring->ring_mem;
3114                 rmem->nr_pages = bp->rx_agg_nr_pages;
3115                 rmem->page_size = HW_RXBD_RING_SIZE;
3116                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3117                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3118                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3119                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3120
3121 skip_rx:
3122                 txr = bnapi->tx_ring;
3123                 if (!txr)
3124                         continue;
3125
3126                 ring = &txr->tx_ring_struct;
3127                 rmem = &ring->ring_mem;
3128                 rmem->nr_pages = bp->tx_nr_pages;
3129                 rmem->page_size = HW_RXBD_RING_SIZE;
3130                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3131                 rmem->dma_arr = txr->tx_desc_mapping;
3132                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3133                 rmem->vmem = (void **)&txr->tx_buf_ring;
3134         }
3135 }
3136
3137 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3138 {
3139         int i;
3140         u32 prod;
3141         struct rx_bd **rx_buf_ring;
3142
3143         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3144         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3145                 int j;
3146                 struct rx_bd *rxbd;
3147
3148                 rxbd = rx_buf_ring[i];
3149                 if (!rxbd)
3150                         continue;
3151
3152                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3153                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3154                         rxbd->rx_bd_opaque = prod;
3155                 }
3156         }
3157 }
3158
3159 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3160 {
3161         struct net_device *dev = bp->dev;
3162         struct bnxt_rx_ring_info *rxr;
3163         struct bnxt_ring_struct *ring;
3164         u32 prod, type;
3165         int i;
3166
3167         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3168                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3169
3170         if (NET_IP_ALIGN == 2)
3171                 type |= RX_BD_FLAGS_SOP;
3172
3173         rxr = &bp->rx_ring[ring_nr];
3174         ring = &rxr->rx_ring_struct;
3175         bnxt_init_rxbd_pages(ring, type);
3176
3177         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3178                 bpf_prog_add(bp->xdp_prog, 1);
3179                 rxr->xdp_prog = bp->xdp_prog;
3180         }
3181         prod = rxr->rx_prod;
3182         for (i = 0; i < bp->rx_ring_size; i++) {
3183                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3184                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3185                                     ring_nr, i, bp->rx_ring_size);
3186                         break;
3187                 }
3188                 prod = NEXT_RX(prod);
3189         }
3190         rxr->rx_prod = prod;
3191         ring->fw_ring_id = INVALID_HW_RING_ID;
3192
3193         ring = &rxr->rx_agg_ring_struct;
3194         ring->fw_ring_id = INVALID_HW_RING_ID;
3195
3196         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3197                 return 0;
3198
3199         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3200                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3201
3202         bnxt_init_rxbd_pages(ring, type);
3203
3204         prod = rxr->rx_agg_prod;
3205         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3206                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3207                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3208                                     ring_nr, i, bp->rx_ring_size);
3209                         break;
3210                 }
3211                 prod = NEXT_RX_AGG(prod);
3212         }
3213         rxr->rx_agg_prod = prod;
3214
3215         if (bp->flags & BNXT_FLAG_TPA) {
3216                 if (rxr->rx_tpa) {
3217                         u8 *data;
3218                         dma_addr_t mapping;
3219
3220                         for (i = 0; i < bp->max_tpa; i++) {
3221                                 data = __bnxt_alloc_rx_data(bp, &mapping,
3222                                                             GFP_KERNEL);
3223                                 if (!data)
3224                                         return -ENOMEM;
3225
3226                                 rxr->rx_tpa[i].data = data;
3227                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3228                                 rxr->rx_tpa[i].mapping = mapping;
3229                         }
3230                 } else {
3231                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3232                         return -ENOMEM;
3233                 }
3234         }
3235
3236         return 0;
3237 }
3238
3239 static void bnxt_init_cp_rings(struct bnxt *bp)
3240 {
3241         int i, j;
3242
3243         for (i = 0; i < bp->cp_nr_rings; i++) {
3244                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3245                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3246
3247                 ring->fw_ring_id = INVALID_HW_RING_ID;
3248                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3249                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3250                 for (j = 0; j < 2; j++) {
3251                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3252
3253                         if (!cpr2)
3254                                 continue;
3255
3256                         ring = &cpr2->cp_ring_struct;
3257                         ring->fw_ring_id = INVALID_HW_RING_ID;
3258                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3259                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3260                 }
3261         }
3262 }
3263
3264 static int bnxt_init_rx_rings(struct bnxt *bp)
3265 {
3266         int i, rc = 0;
3267
3268         if (BNXT_RX_PAGE_MODE(bp)) {
3269                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3270                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3271         } else {
3272                 bp->rx_offset = BNXT_RX_OFFSET;
3273                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3274         }
3275
3276         for (i = 0; i < bp->rx_nr_rings; i++) {
3277                 rc = bnxt_init_one_rx_ring(bp, i);
3278                 if (rc)
3279                         break;
3280         }
3281
3282         return rc;
3283 }
3284
3285 static int bnxt_init_tx_rings(struct bnxt *bp)
3286 {
3287         u16 i;
3288
3289         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3290                                    MAX_SKB_FRAGS + 1);
3291
3292         for (i = 0; i < bp->tx_nr_rings; i++) {
3293                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3294                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3295
3296                 ring->fw_ring_id = INVALID_HW_RING_ID;
3297         }
3298
3299         return 0;
3300 }
3301
3302 static void bnxt_free_ring_grps(struct bnxt *bp)
3303 {
3304         kfree(bp->grp_info);
3305         bp->grp_info = NULL;
3306 }
3307
3308 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3309 {
3310         int i;
3311
3312         if (irq_re_init) {
3313                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3314                                        sizeof(struct bnxt_ring_grp_info),
3315                                        GFP_KERNEL);
3316                 if (!bp->grp_info)
3317                         return -ENOMEM;
3318         }
3319         for (i = 0; i < bp->cp_nr_rings; i++) {
3320                 if (irq_re_init)
3321                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3322                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3323                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3324                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3325                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3326         }
3327         return 0;
3328 }
3329
3330 static void bnxt_free_vnics(struct bnxt *bp)
3331 {
3332         kfree(bp->vnic_info);
3333         bp->vnic_info = NULL;
3334         bp->nr_vnics = 0;
3335 }
3336
3337 static int bnxt_alloc_vnics(struct bnxt *bp)
3338 {
3339         int num_vnics = 1;
3340
3341 #ifdef CONFIG_RFS_ACCEL
3342         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3343                 num_vnics += bp->rx_nr_rings;
3344 #endif
3345
3346         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3347                 num_vnics++;
3348
3349         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3350                                 GFP_KERNEL);
3351         if (!bp->vnic_info)
3352                 return -ENOMEM;
3353
3354         bp->nr_vnics = num_vnics;
3355         return 0;
3356 }
3357
3358 static void bnxt_init_vnics(struct bnxt *bp)
3359 {
3360         int i;
3361
3362         for (i = 0; i < bp->nr_vnics; i++) {
3363                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3364                 int j;
3365
3366                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3367                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3368                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3369
3370                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3371
3372                 if (bp->vnic_info[i].rss_hash_key) {
3373                         if (i == 0)
3374                                 prandom_bytes(vnic->rss_hash_key,
3375                                               HW_HASH_KEY_SIZE);
3376                         else
3377                                 memcpy(vnic->rss_hash_key,
3378                                        bp->vnic_info[0].rss_hash_key,
3379                                        HW_HASH_KEY_SIZE);
3380                 }
3381         }
3382 }
3383
3384 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3385 {
3386         int pages;
3387
3388         pages = ring_size / desc_per_pg;
3389
3390         if (!pages)
3391                 return 1;
3392
3393         pages++;
3394
3395         while (pages & (pages - 1))
3396                 pages++;
3397
3398         return pages;
3399 }
3400
3401 void bnxt_set_tpa_flags(struct bnxt *bp)
3402 {
3403         bp->flags &= ~BNXT_FLAG_TPA;
3404         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3405                 return;
3406         if (bp->dev->features & NETIF_F_LRO)
3407                 bp->flags |= BNXT_FLAG_LRO;
3408         else if (bp->dev->features & NETIF_F_GRO_HW)
3409                 bp->flags |= BNXT_FLAG_GRO;
3410 }
3411
3412 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3413  * be set on entry.
3414  */
3415 void bnxt_set_ring_params(struct bnxt *bp)
3416 {
3417         u32 ring_size, rx_size, rx_space;
3418         u32 agg_factor = 0, agg_ring_size = 0;
3419
3420         /* 8 for CRC and VLAN */
3421         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3422
3423         rx_space = rx_size + NET_SKB_PAD +
3424                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3425
3426         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3427         ring_size = bp->rx_ring_size;
3428         bp->rx_agg_ring_size = 0;
3429         bp->rx_agg_nr_pages = 0;
3430
3431         if (bp->flags & BNXT_FLAG_TPA)
3432                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3433
3434         bp->flags &= ~BNXT_FLAG_JUMBO;
3435         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3436                 u32 jumbo_factor;
3437
3438                 bp->flags |= BNXT_FLAG_JUMBO;
3439                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3440                 if (jumbo_factor > agg_factor)
3441                         agg_factor = jumbo_factor;
3442         }
3443         agg_ring_size = ring_size * agg_factor;
3444
3445         if (agg_ring_size) {
3446                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3447                                                         RX_DESC_CNT);
3448                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3449                         u32 tmp = agg_ring_size;
3450
3451                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3452                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3453                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3454                                     tmp, agg_ring_size);
3455                 }
3456                 bp->rx_agg_ring_size = agg_ring_size;
3457                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3458                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3459                 rx_space = rx_size + NET_SKB_PAD +
3460                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3461         }
3462
3463         bp->rx_buf_use_size = rx_size;
3464         bp->rx_buf_size = rx_space;
3465
3466         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3467         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3468
3469         ring_size = bp->tx_ring_size;
3470         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3471         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3472
3473         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3474         bp->cp_ring_size = ring_size;
3475
3476         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3477         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3478                 bp->cp_nr_pages = MAX_CP_PAGES;
3479                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3480                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3481                             ring_size, bp->cp_ring_size);
3482         }
3483         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3484         bp->cp_ring_mask = bp->cp_bit - 1;
3485 }
3486
3487 /* Changing allocation mode of RX rings.
3488  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3489  */
3490 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3491 {
3492         if (page_mode) {
3493                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3494                         return -EOPNOTSUPP;
3495                 bp->dev->max_mtu =
3496                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3497                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3498                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3499                 bp->rx_dir = DMA_BIDIRECTIONAL;
3500                 bp->rx_skb_func = bnxt_rx_page_skb;
3501                 /* Disable LRO or GRO_HW */
3502                 netdev_update_features(bp->dev);
3503         } else {
3504                 bp->dev->max_mtu = bp->max_mtu;
3505                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3506                 bp->rx_dir = DMA_FROM_DEVICE;
3507                 bp->rx_skb_func = bnxt_rx_skb;
3508         }
3509         return 0;
3510 }
3511
3512 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3513 {
3514         int i;
3515         struct bnxt_vnic_info *vnic;
3516         struct pci_dev *pdev = bp->pdev;
3517
3518         if (!bp->vnic_info)
3519                 return;
3520
3521         for (i = 0; i < bp->nr_vnics; i++) {
3522                 vnic = &bp->vnic_info[i];
3523
3524                 kfree(vnic->fw_grp_ids);
3525                 vnic->fw_grp_ids = NULL;
3526
3527                 kfree(vnic->uc_list);
3528                 vnic->uc_list = NULL;
3529
3530                 if (vnic->mc_list) {
3531                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3532                                           vnic->mc_list, vnic->mc_list_mapping);
3533                         vnic->mc_list = NULL;
3534                 }
3535
3536                 if (vnic->rss_table) {
3537                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
3538                                           vnic->rss_table,
3539                                           vnic->rss_table_dma_addr);
3540                         vnic->rss_table = NULL;
3541                 }
3542
3543                 vnic->rss_hash_key = NULL;
3544                 vnic->flags = 0;
3545         }
3546 }
3547
3548 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3549 {
3550         int i, rc = 0, size;
3551         struct bnxt_vnic_info *vnic;
3552         struct pci_dev *pdev = bp->pdev;
3553         int max_rings;
3554
3555         for (i = 0; i < bp->nr_vnics; i++) {
3556                 vnic = &bp->vnic_info[i];
3557
3558                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3559                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3560
3561                         if (mem_size > 0) {
3562                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3563                                 if (!vnic->uc_list) {
3564                                         rc = -ENOMEM;
3565                                         goto out;
3566                                 }
3567                         }
3568                 }
3569
3570                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3571                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3572                         vnic->mc_list =
3573                                 dma_alloc_coherent(&pdev->dev,
3574                                                    vnic->mc_list_size,
3575                                                    &vnic->mc_list_mapping,
3576                                                    GFP_KERNEL);
3577                         if (!vnic->mc_list) {
3578                                 rc = -ENOMEM;
3579                                 goto out;
3580                         }
3581                 }
3582
3583                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3584                         goto vnic_skip_grps;
3585
3586                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3587                         max_rings = bp->rx_nr_rings;
3588                 else
3589                         max_rings = 1;
3590
3591                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3592                 if (!vnic->fw_grp_ids) {
3593                         rc = -ENOMEM;
3594                         goto out;
3595                 }
3596 vnic_skip_grps:
3597                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3598                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3599                         continue;
3600
3601                 /* Allocate rss table and hash key */
3602                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3603                                                      &vnic->rss_table_dma_addr,
3604                                                      GFP_KERNEL);
3605                 if (!vnic->rss_table) {
3606                         rc = -ENOMEM;
3607                         goto out;
3608                 }
3609
3610                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3611
3612                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3613                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3614         }
3615         return 0;
3616
3617 out:
3618         return rc;
3619 }
3620
3621 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3622 {
3623         struct pci_dev *pdev = bp->pdev;
3624
3625         if (bp->hwrm_cmd_resp_addr) {
3626                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3627                                   bp->hwrm_cmd_resp_dma_addr);
3628                 bp->hwrm_cmd_resp_addr = NULL;
3629         }
3630
3631         if (bp->hwrm_cmd_kong_resp_addr) {
3632                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3633                                   bp->hwrm_cmd_kong_resp_addr,
3634                                   bp->hwrm_cmd_kong_resp_dma_addr);
3635                 bp->hwrm_cmd_kong_resp_addr = NULL;
3636         }
3637 }
3638
3639 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3640 {
3641         struct pci_dev *pdev = bp->pdev;
3642
3643         if (bp->hwrm_cmd_kong_resp_addr)
3644                 return 0;
3645
3646         bp->hwrm_cmd_kong_resp_addr =
3647                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3648                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3649                                    GFP_KERNEL);
3650         if (!bp->hwrm_cmd_kong_resp_addr)
3651                 return -ENOMEM;
3652
3653         return 0;
3654 }
3655
3656 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3657 {
3658         struct pci_dev *pdev = bp->pdev;
3659
3660         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3661                                                    &bp->hwrm_cmd_resp_dma_addr,
3662                                                    GFP_KERNEL);
3663         if (!bp->hwrm_cmd_resp_addr)
3664                 return -ENOMEM;
3665
3666         return 0;
3667 }
3668
3669 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3670 {
3671         if (bp->hwrm_short_cmd_req_addr) {
3672                 struct pci_dev *pdev = bp->pdev;
3673
3674                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3675                                   bp->hwrm_short_cmd_req_addr,
3676                                   bp->hwrm_short_cmd_req_dma_addr);
3677                 bp->hwrm_short_cmd_req_addr = NULL;
3678         }
3679 }
3680
3681 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3682 {
3683         struct pci_dev *pdev = bp->pdev;
3684
3685         if (bp->hwrm_short_cmd_req_addr)
3686                 return 0;
3687
3688         bp->hwrm_short_cmd_req_addr =
3689                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3690                                    &bp->hwrm_short_cmd_req_dma_addr,
3691                                    GFP_KERNEL);
3692         if (!bp->hwrm_short_cmd_req_addr)
3693                 return -ENOMEM;
3694
3695         return 0;
3696 }
3697
3698 static void bnxt_free_port_stats(struct bnxt *bp)
3699 {
3700         struct pci_dev *pdev = bp->pdev;
3701
3702         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3703         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3704
3705         if (bp->hw_rx_port_stats) {
3706                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3707                                   bp->hw_rx_port_stats,
3708                                   bp->hw_rx_port_stats_map);
3709                 bp->hw_rx_port_stats = NULL;
3710         }
3711
3712         if (bp->hw_tx_port_stats_ext) {
3713                 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3714                                   bp->hw_tx_port_stats_ext,
3715                                   bp->hw_tx_port_stats_ext_map);
3716                 bp->hw_tx_port_stats_ext = NULL;
3717         }
3718
3719         if (bp->hw_rx_port_stats_ext) {
3720                 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3721                                   bp->hw_rx_port_stats_ext,
3722                                   bp->hw_rx_port_stats_ext_map);
3723                 bp->hw_rx_port_stats_ext = NULL;
3724         }
3725
3726         if (bp->hw_pcie_stats) {
3727                 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3728                                   bp->hw_pcie_stats, bp->hw_pcie_stats_map);
3729                 bp->hw_pcie_stats = NULL;
3730         }
3731 }
3732
3733 static void bnxt_free_ring_stats(struct bnxt *bp)
3734 {
3735         struct pci_dev *pdev = bp->pdev;
3736         int size, i;
3737
3738         if (!bp->bnapi)
3739                 return;
3740
3741         size = bp->hw_ring_stats_size;
3742
3743         for (i = 0; i < bp->cp_nr_rings; i++) {
3744                 struct bnxt_napi *bnapi = bp->bnapi[i];
3745                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3746
3747                 if (cpr->hw_stats) {
3748                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3749                                           cpr->hw_stats_map);
3750                         cpr->hw_stats = NULL;
3751                 }
3752         }
3753 }
3754
3755 static int bnxt_alloc_stats(struct bnxt *bp)
3756 {
3757         u32 size, i;
3758         struct pci_dev *pdev = bp->pdev;
3759
3760         size = bp->hw_ring_stats_size;
3761
3762         for (i = 0; i < bp->cp_nr_rings; i++) {
3763                 struct bnxt_napi *bnapi = bp->bnapi[i];
3764                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3765
3766                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3767                                                    &cpr->hw_stats_map,
3768                                                    GFP_KERNEL);
3769                 if (!cpr->hw_stats)
3770                         return -ENOMEM;
3771
3772                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3773         }
3774
3775         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3776                 return 0;
3777
3778         if (bp->hw_rx_port_stats)
3779                 goto alloc_ext_stats;
3780
3781         bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3782                                  sizeof(struct tx_port_stats) + 1024;
3783
3784         bp->hw_rx_port_stats =
3785                 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3786                                    &bp->hw_rx_port_stats_map,
3787                                    GFP_KERNEL);
3788         if (!bp->hw_rx_port_stats)
3789                 return -ENOMEM;
3790
3791         bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512;
3792         bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3793                                    sizeof(struct rx_port_stats) + 512;
3794         bp->flags |= BNXT_FLAG_PORT_STATS;
3795
3796 alloc_ext_stats:
3797         /* Display extended statistics only if FW supports it */
3798         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3799                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3800                         return 0;
3801
3802         if (bp->hw_rx_port_stats_ext)
3803                 goto alloc_tx_ext_stats;
3804
3805         bp->hw_rx_port_stats_ext =
3806                 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3807                                    &bp->hw_rx_port_stats_ext_map, GFP_KERNEL);
3808         if (!bp->hw_rx_port_stats_ext)
3809                 return 0;
3810
3811 alloc_tx_ext_stats:
3812         if (bp->hw_tx_port_stats_ext)
3813                 goto alloc_pcie_stats;
3814
3815         if (bp->hwrm_spec_code >= 0x10902 ||
3816             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3817                 bp->hw_tx_port_stats_ext =
3818                         dma_alloc_coherent(&pdev->dev,
3819                                            sizeof(struct tx_port_stats_ext),
3820                                            &bp->hw_tx_port_stats_ext_map,
3821                                            GFP_KERNEL);
3822         }
3823         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3824
3825 alloc_pcie_stats:
3826         if (bp->hw_pcie_stats ||
3827             !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED))
3828                 return 0;
3829
3830         bp->hw_pcie_stats =
3831                 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3832                                    &bp->hw_pcie_stats_map, GFP_KERNEL);
3833         if (!bp->hw_pcie_stats)
3834                 return 0;
3835
3836         bp->flags |= BNXT_FLAG_PCIE_STATS;
3837         return 0;
3838 }
3839
3840 static void bnxt_clear_ring_indices(struct bnxt *bp)
3841 {
3842         int i;
3843
3844         if (!bp->bnapi)
3845                 return;
3846
3847         for (i = 0; i < bp->cp_nr_rings; i++) {
3848                 struct bnxt_napi *bnapi = bp->bnapi[i];
3849                 struct bnxt_cp_ring_info *cpr;
3850                 struct bnxt_rx_ring_info *rxr;
3851                 struct bnxt_tx_ring_info *txr;
3852
3853                 if (!bnapi)
3854                         continue;
3855
3856                 cpr = &bnapi->cp_ring;
3857                 cpr->cp_raw_cons = 0;
3858
3859                 txr = bnapi->tx_ring;
3860                 if (txr) {
3861                         txr->tx_prod = 0;
3862                         txr->tx_cons = 0;
3863                 }
3864
3865                 rxr = bnapi->rx_ring;
3866                 if (rxr) {
3867                         rxr->rx_prod = 0;
3868                         rxr->rx_agg_prod = 0;
3869                         rxr->rx_sw_agg_prod = 0;
3870                         rxr->rx_next_cons = 0;
3871                 }
3872         }
3873 }
3874
3875 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3876 {
3877 #ifdef CONFIG_RFS_ACCEL
3878         int i;
3879
3880         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3881          * safe to delete the hash table.
3882          */
3883         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3884                 struct hlist_head *head;
3885                 struct hlist_node *tmp;
3886                 struct bnxt_ntuple_filter *fltr;
3887
3888                 head = &bp->ntp_fltr_hash_tbl[i];
3889                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3890                         hlist_del(&fltr->hash);
3891                         kfree(fltr);
3892                 }
3893         }
3894         if (irq_reinit) {
3895                 kfree(bp->ntp_fltr_bmap);
3896                 bp->ntp_fltr_bmap = NULL;
3897         }
3898         bp->ntp_fltr_count = 0;
3899 #endif
3900 }
3901
3902 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3903 {
3904 #ifdef CONFIG_RFS_ACCEL
3905         int i, rc = 0;
3906
3907         if (!(bp->flags & BNXT_FLAG_RFS))
3908                 return 0;
3909
3910         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3911                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3912
3913         bp->ntp_fltr_count = 0;
3914         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3915                                     sizeof(long),
3916                                     GFP_KERNEL);
3917
3918         if (!bp->ntp_fltr_bmap)
3919                 rc = -ENOMEM;
3920
3921         return rc;
3922 #else
3923         return 0;
3924 #endif
3925 }
3926
3927 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3928 {
3929         bnxt_free_vnic_attributes(bp);
3930         bnxt_free_tx_rings(bp);
3931         bnxt_free_rx_rings(bp);
3932         bnxt_free_cp_rings(bp);
3933         bnxt_free_ntp_fltrs(bp, irq_re_init);
3934         if (irq_re_init) {
3935                 bnxt_free_ring_stats(bp);
3936                 bnxt_free_ring_grps(bp);
3937                 bnxt_free_vnics(bp);
3938                 kfree(bp->tx_ring_map);
3939                 bp->tx_ring_map = NULL;
3940                 kfree(bp->tx_ring);
3941                 bp->tx_ring = NULL;
3942                 kfree(bp->rx_ring);
3943                 bp->rx_ring = NULL;
3944                 kfree(bp->bnapi);
3945                 bp->bnapi = NULL;
3946         } else {
3947                 bnxt_clear_ring_indices(bp);
3948         }
3949 }
3950
3951 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3952 {
3953         int i, j, rc, size, arr_size;
3954         void *bnapi;
3955
3956         if (irq_re_init) {
3957                 /* Allocate bnapi mem pointer array and mem block for
3958                  * all queues
3959                  */
3960                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3961                                 bp->cp_nr_rings);
3962                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3963                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3964                 if (!bnapi)
3965                         return -ENOMEM;
3966
3967                 bp->bnapi = bnapi;
3968                 bnapi += arr_size;
3969                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3970                         bp->bnapi[i] = bnapi;
3971                         bp->bnapi[i]->index = i;
3972                         bp->bnapi[i]->bp = bp;
3973                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3974                                 struct bnxt_cp_ring_info *cpr =
3975                                         &bp->bnapi[i]->cp_ring;
3976
3977                                 cpr->cp_ring_struct.ring_mem.flags =
3978                                         BNXT_RMEM_RING_PTE_FLAG;
3979                         }
3980                 }
3981
3982                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3983                                       sizeof(struct bnxt_rx_ring_info),
3984                                       GFP_KERNEL);
3985                 if (!bp->rx_ring)
3986                         return -ENOMEM;
3987
3988                 for (i = 0; i < bp->rx_nr_rings; i++) {
3989                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3990
3991                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3992                                 rxr->rx_ring_struct.ring_mem.flags =
3993                                         BNXT_RMEM_RING_PTE_FLAG;
3994                                 rxr->rx_agg_ring_struct.ring_mem.flags =
3995                                         BNXT_RMEM_RING_PTE_FLAG;
3996                         }
3997                         rxr->bnapi = bp->bnapi[i];
3998                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3999                 }
4000
4001                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4002                                       sizeof(struct bnxt_tx_ring_info),
4003                                       GFP_KERNEL);
4004                 if (!bp->tx_ring)
4005                         return -ENOMEM;
4006
4007                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4008                                           GFP_KERNEL);
4009
4010                 if (!bp->tx_ring_map)
4011                         return -ENOMEM;
4012
4013                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4014                         j = 0;
4015                 else
4016                         j = bp->rx_nr_rings;
4017
4018                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4019                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4020
4021                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4022                                 txr->tx_ring_struct.ring_mem.flags =
4023                                         BNXT_RMEM_RING_PTE_FLAG;
4024                         txr->bnapi = bp->bnapi[j];
4025                         bp->bnapi[j]->tx_ring = txr;
4026                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4027                         if (i >= bp->tx_nr_rings_xdp) {
4028                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4029                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4030                         } else {
4031                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4032                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4033                         }
4034                 }
4035
4036                 rc = bnxt_alloc_stats(bp);
4037                 if (rc)
4038                         goto alloc_mem_err;
4039
4040                 rc = bnxt_alloc_ntp_fltrs(bp);
4041                 if (rc)
4042                         goto alloc_mem_err;
4043
4044                 rc = bnxt_alloc_vnics(bp);
4045                 if (rc)
4046                         goto alloc_mem_err;
4047         }
4048
4049         bnxt_init_ring_struct(bp);
4050
4051         rc = bnxt_alloc_rx_rings(bp);
4052         if (rc)
4053                 goto alloc_mem_err;
4054
4055         rc = bnxt_alloc_tx_rings(bp);
4056         if (rc)
4057                 goto alloc_mem_err;
4058
4059         rc = bnxt_alloc_cp_rings(bp);
4060         if (rc)
4061                 goto alloc_mem_err;
4062
4063         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4064                                   BNXT_VNIC_UCAST_FLAG;
4065         rc = bnxt_alloc_vnic_attributes(bp);
4066         if (rc)
4067                 goto alloc_mem_err;
4068         return 0;
4069
4070 alloc_mem_err:
4071         bnxt_free_mem(bp, true);
4072         return rc;
4073 }
4074
4075 static void bnxt_disable_int(struct bnxt *bp)
4076 {
4077         int i;
4078
4079         if (!bp->bnapi)
4080                 return;
4081
4082         for (i = 0; i < bp->cp_nr_rings; i++) {
4083                 struct bnxt_napi *bnapi = bp->bnapi[i];
4084                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4085                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4086
4087                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4088                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4089         }
4090 }
4091
4092 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4093 {
4094         struct bnxt_napi *bnapi = bp->bnapi[n];
4095         struct bnxt_cp_ring_info *cpr;
4096
4097         cpr = &bnapi->cp_ring;
4098         return cpr->cp_ring_struct.map_idx;
4099 }
4100
4101 static void bnxt_disable_int_sync(struct bnxt *bp)
4102 {
4103         int i;
4104
4105         atomic_inc(&bp->intr_sem);
4106
4107         bnxt_disable_int(bp);
4108         for (i = 0; i < bp->cp_nr_rings; i++) {
4109                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4110
4111                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4112         }
4113 }
4114
4115 static void bnxt_enable_int(struct bnxt *bp)
4116 {
4117         int i;
4118
4119         atomic_set(&bp->intr_sem, 0);
4120         for (i = 0; i < bp->cp_nr_rings; i++) {
4121                 struct bnxt_napi *bnapi = bp->bnapi[i];
4122                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4123
4124                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4125         }
4126 }
4127
4128 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4129                             u16 cmpl_ring, u16 target_id)
4130 {
4131         struct input *req = request;
4132
4133         req->req_type = cpu_to_le16(req_type);
4134         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4135         req->target_id = cpu_to_le16(target_id);
4136         if (bnxt_kong_hwrm_message(bp, req))
4137                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4138         else
4139                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4140 }
4141
4142 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4143 {
4144         switch (hwrm_err) {
4145         case HWRM_ERR_CODE_SUCCESS:
4146                 return 0;
4147         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4148                 return -EACCES;
4149         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4150                 return -ENOSPC;
4151         case HWRM_ERR_CODE_INVALID_PARAMS:
4152         case HWRM_ERR_CODE_INVALID_FLAGS:
4153         case HWRM_ERR_CODE_INVALID_ENABLES:
4154         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4155         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4156                 return -EINVAL;
4157         case HWRM_ERR_CODE_NO_BUFFER:
4158                 return -ENOMEM;
4159         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4160                 return -EAGAIN;
4161         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4162                 return -EOPNOTSUPP;
4163         default:
4164                 return -EIO;
4165         }
4166 }
4167
4168 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4169                                  int timeout, bool silent)
4170 {
4171         int i, intr_process, rc, tmo_count;
4172         struct input *req = msg;
4173         u32 *data = msg;
4174         __le32 *resp_len;
4175         u8 *valid;
4176         u16 cp_ring_id, len = 0;
4177         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4178         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4179         struct hwrm_short_input short_input = {0};
4180         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4181         u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
4182         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4183         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4184
4185         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4186                 return -EBUSY;
4187
4188         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4189                 if (msg_len > bp->hwrm_max_ext_req_len ||
4190                     !bp->hwrm_short_cmd_req_addr)
4191                         return -EINVAL;
4192         }
4193
4194         if (bnxt_hwrm_kong_chnl(bp, req)) {
4195                 dst = BNXT_HWRM_CHNL_KONG;
4196                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4197                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4198                 resp = bp->hwrm_cmd_kong_resp_addr;
4199                 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
4200         }
4201
4202         memset(resp, 0, PAGE_SIZE);
4203         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4204         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4205
4206         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4207         /* currently supports only one outstanding message */
4208         if (intr_process)
4209                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4210
4211         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4212             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4213                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4214                 u16 max_msg_len;
4215
4216                 /* Set boundary for maximum extended request length for short
4217                  * cmd format. If passed up from device use the max supported
4218                  * internal req length.
4219                  */
4220                 max_msg_len = bp->hwrm_max_ext_req_len;
4221
4222                 memcpy(short_cmd_req, req, msg_len);
4223                 if (msg_len < max_msg_len)
4224                         memset(short_cmd_req + msg_len, 0,
4225                                max_msg_len - msg_len);
4226
4227                 short_input.req_type = req->req_type;
4228                 short_input.signature =
4229                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4230                 short_input.size = cpu_to_le16(msg_len);
4231                 short_input.req_addr =
4232                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4233
4234                 data = (u32 *)&short_input;
4235                 msg_len = sizeof(short_input);
4236
4237                 /* Sync memory write before updating doorbell */
4238                 wmb();
4239
4240                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4241         }
4242
4243         /* Write request msg to hwrm channel */
4244         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4245
4246         for (i = msg_len; i < max_req_len; i += 4)
4247                 writel(0, bp->bar0 + bar_offset + i);
4248
4249         /* Ring channel doorbell */
4250         writel(1, bp->bar0 + doorbell_offset);
4251
4252         if (!pci_is_enabled(bp->pdev))
4253                 return 0;
4254
4255         if (!timeout)
4256                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4257         /* convert timeout to usec */
4258         timeout *= 1000;
4259
4260         i = 0;
4261         /* Short timeout for the first few iterations:
4262          * number of loops = number of loops for short timeout +
4263          * number of loops for standard timeout.
4264          */
4265         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4266         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4267         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4268         resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
4269
4270         if (intr_process) {
4271                 u16 seq_id = bp->hwrm_intr_seq_id;
4272
4273                 /* Wait until hwrm response cmpl interrupt is processed */
4274                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4275                        i++ < tmo_count) {
4276                         /* Abort the wait for completion if the FW health
4277                          * check has failed.
4278                          */
4279                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4280                                 return -EBUSY;
4281                         /* on first few passes, just barely sleep */
4282                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4283                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4284                                              HWRM_SHORT_MAX_TIMEOUT);
4285                         else
4286                                 usleep_range(HWRM_MIN_TIMEOUT,
4287                                              HWRM_MAX_TIMEOUT);
4288                 }
4289
4290                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4291                         if (!silent)
4292                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4293                                            le16_to_cpu(req->req_type));
4294                         return -EBUSY;
4295                 }
4296                 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4297                       HWRM_RESP_LEN_SFT;
4298                 valid = resp_addr + len - 1;
4299         } else {
4300                 int j;
4301
4302                 /* Check if response len is updated */
4303                 for (i = 0; i < tmo_count; i++) {
4304                         /* Abort the wait for completion if the FW health
4305                          * check has failed.
4306                          */
4307                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4308                                 return -EBUSY;
4309                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4310                               HWRM_RESP_LEN_SFT;
4311                         if (len)
4312                                 break;
4313                         /* on first few passes, just barely sleep */
4314                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4315                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4316                                              HWRM_SHORT_MAX_TIMEOUT);
4317                         else
4318                                 usleep_range(HWRM_MIN_TIMEOUT,
4319                                              HWRM_MAX_TIMEOUT);
4320                 }
4321
4322                 if (i >= tmo_count) {
4323                         if (!silent)
4324                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4325                                            HWRM_TOTAL_TIMEOUT(i),
4326                                            le16_to_cpu(req->req_type),
4327                                            le16_to_cpu(req->seq_id), len);
4328                         return -EBUSY;
4329                 }
4330
4331                 /* Last byte of resp contains valid bit */
4332                 valid = resp_addr + len - 1;
4333                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4334                         /* make sure we read from updated DMA memory */
4335                         dma_rmb();
4336                         if (*valid)
4337                                 break;
4338                         usleep_range(1, 5);
4339                 }
4340
4341                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4342                         if (!silent)
4343                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4344                                            HWRM_TOTAL_TIMEOUT(i),
4345                                            le16_to_cpu(req->req_type),
4346                                            le16_to_cpu(req->seq_id), len,
4347                                            *valid);
4348                         return -EBUSY;
4349                 }
4350         }
4351
4352         /* Zero valid bit for compatibility.  Valid bit in an older spec
4353          * may become a new field in a newer spec.  We must make sure that
4354          * a new field not implemented by old spec will read zero.
4355          */
4356         *valid = 0;
4357         rc = le16_to_cpu(resp->error_code);
4358         if (rc && !silent)
4359                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4360                            le16_to_cpu(resp->req_type),
4361                            le16_to_cpu(resp->seq_id), rc);
4362         return bnxt_hwrm_to_stderr(rc);
4363 }
4364
4365 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4366 {
4367         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4368 }
4369
4370 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4371                               int timeout)
4372 {
4373         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4374 }
4375
4376 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4377 {
4378         int rc;
4379
4380         mutex_lock(&bp->hwrm_cmd_lock);
4381         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4382         mutex_unlock(&bp->hwrm_cmd_lock);
4383         return rc;
4384 }
4385
4386 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4387                              int timeout)
4388 {
4389         int rc;
4390
4391         mutex_lock(&bp->hwrm_cmd_lock);
4392         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4393         mutex_unlock(&bp->hwrm_cmd_lock);
4394         return rc;
4395 }
4396
4397 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
4398                                      int bmap_size)
4399 {
4400         struct hwrm_func_drv_rgtr_input req = {0};
4401         DECLARE_BITMAP(async_events_bmap, 256);
4402         u32 *events = (u32 *)async_events_bmap;
4403         int i;
4404
4405         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4406
4407         req.enables =
4408                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4409
4410         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4411         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4412                 u16 event_id = bnxt_async_events_arr[i];
4413
4414                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4415                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4416                         continue;
4417                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4418         }
4419         if (bmap && bmap_size) {
4420                 for (i = 0; i < bmap_size; i++) {
4421                         if (test_bit(i, bmap))
4422                                 __set_bit(i, async_events_bmap);
4423                 }
4424         }
4425
4426         for (i = 0; i < 8; i++)
4427                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4428
4429         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4430 }
4431
4432 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
4433 {
4434         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4435         struct hwrm_func_drv_rgtr_input req = {0};
4436         u32 flags;
4437         int rc;
4438
4439         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4440
4441         req.enables =
4442                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4443                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
4444
4445         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4446         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE |
4447                 FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4448         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4449                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4450                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4451         req.flags = cpu_to_le32(flags);
4452         req.ver_maj_8b = DRV_VER_MAJ;
4453         req.ver_min_8b = DRV_VER_MIN;
4454         req.ver_upd_8b = DRV_VER_UPD;
4455         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4456         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4457         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4458
4459         if (BNXT_PF(bp)) {
4460                 u32 data[8];
4461                 int i;
4462
4463                 memset(data, 0, sizeof(data));
4464                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4465                         u16 cmd = bnxt_vf_req_snif[i];
4466                         unsigned int bit, idx;
4467
4468                         idx = cmd / 32;
4469                         bit = cmd % 32;
4470                         data[idx] |= 1 << bit;
4471                 }
4472
4473                 for (i = 0; i < 8; i++)
4474                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4475
4476                 req.enables |=
4477                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4478         }
4479
4480         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4481                 req.flags |= cpu_to_le32(
4482                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4483
4484         mutex_lock(&bp->hwrm_cmd_lock);
4485         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4486         if (!rc && (resp->flags &
4487                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)))
4488                 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4489         mutex_unlock(&bp->hwrm_cmd_lock);
4490         return rc;
4491 }
4492
4493 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4494 {
4495         struct hwrm_func_drv_unrgtr_input req = {0};
4496
4497         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4498         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4499 }
4500
4501 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4502 {
4503         u32 rc = 0;
4504         struct hwrm_tunnel_dst_port_free_input req = {0};
4505
4506         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4507         req.tunnel_type = tunnel_type;
4508
4509         switch (tunnel_type) {
4510         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4511                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4512                 break;
4513         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4514                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4515                 break;
4516         default:
4517                 break;
4518         }
4519
4520         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4521         if (rc)
4522                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4523                            rc);
4524         return rc;
4525 }
4526
4527 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4528                                            u8 tunnel_type)
4529 {
4530         u32 rc = 0;
4531         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4532         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4533
4534         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4535
4536         req.tunnel_type = tunnel_type;
4537         req.tunnel_dst_port_val = port;
4538
4539         mutex_lock(&bp->hwrm_cmd_lock);
4540         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4541         if (rc) {
4542                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4543                            rc);
4544                 goto err_out;
4545         }
4546
4547         switch (tunnel_type) {
4548         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4549                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
4550                 break;
4551         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4552                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
4553                 break;
4554         default:
4555                 break;
4556         }
4557
4558 err_out:
4559         mutex_unlock(&bp->hwrm_cmd_lock);
4560         return rc;
4561 }
4562
4563 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4564 {
4565         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4566         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4567
4568         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4569         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4570
4571         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4572         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4573         req.mask = cpu_to_le32(vnic->rx_mask);
4574         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4575 }
4576
4577 #ifdef CONFIG_RFS_ACCEL
4578 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4579                                             struct bnxt_ntuple_filter *fltr)
4580 {
4581         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4582
4583         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4584         req.ntuple_filter_id = fltr->filter_id;
4585         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4586 }
4587
4588 #define BNXT_NTP_FLTR_FLAGS                                     \
4589         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4590          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4591          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4592          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4593          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4594          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4595          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4596          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4597          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4598          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4599          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4600          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4601          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4602          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4603
4604 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4605                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4606
4607 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4608                                              struct bnxt_ntuple_filter *fltr)
4609 {
4610         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4611         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4612         struct flow_keys *keys = &fltr->fkeys;
4613         struct bnxt_vnic_info *vnic;
4614         u32 flags = 0;
4615         int rc = 0;
4616
4617         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4618         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4619
4620         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4621                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4622                 req.dst_id = cpu_to_le16(fltr->rxq);
4623         } else {
4624                 vnic = &bp->vnic_info[fltr->rxq + 1];
4625                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4626         }
4627         req.flags = cpu_to_le32(flags);
4628         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4629
4630         req.ethertype = htons(ETH_P_IP);
4631         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4632         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4633         req.ip_protocol = keys->basic.ip_proto;
4634
4635         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4636                 int i;
4637
4638                 req.ethertype = htons(ETH_P_IPV6);
4639                 req.ip_addr_type =
4640                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4641                 *(struct in6_addr *)&req.src_ipaddr[0] =
4642                         keys->addrs.v6addrs.src;
4643                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4644                         keys->addrs.v6addrs.dst;
4645                 for (i = 0; i < 4; i++) {
4646                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4647                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4648                 }
4649         } else {
4650                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4651                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4652                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4653                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4654         }
4655         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4656                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4657                 req.tunnel_type =
4658                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4659         }
4660
4661         req.src_port = keys->ports.src;
4662         req.src_port_mask = cpu_to_be16(0xffff);
4663         req.dst_port = keys->ports.dst;
4664         req.dst_port_mask = cpu_to_be16(0xffff);
4665
4666         mutex_lock(&bp->hwrm_cmd_lock);
4667         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4668         if (!rc) {
4669                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4670                 fltr->filter_id = resp->ntuple_filter_id;
4671         }
4672         mutex_unlock(&bp->hwrm_cmd_lock);
4673         return rc;
4674 }
4675 #endif
4676
4677 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4678                                      u8 *mac_addr)
4679 {
4680         u32 rc = 0;
4681         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4682         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4683
4684         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4685         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4686         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4687                 req.flags |=
4688                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4689         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4690         req.enables =
4691                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4692                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4693                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4694         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4695         req.l2_addr_mask[0] = 0xff;
4696         req.l2_addr_mask[1] = 0xff;
4697         req.l2_addr_mask[2] = 0xff;
4698         req.l2_addr_mask[3] = 0xff;
4699         req.l2_addr_mask[4] = 0xff;
4700         req.l2_addr_mask[5] = 0xff;
4701
4702         mutex_lock(&bp->hwrm_cmd_lock);
4703         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4704         if (!rc)
4705                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4706                                                         resp->l2_filter_id;
4707         mutex_unlock(&bp->hwrm_cmd_lock);
4708         return rc;
4709 }
4710
4711 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4712 {
4713         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4714         int rc = 0;
4715
4716         /* Any associated ntuple filters will also be cleared by firmware. */
4717         mutex_lock(&bp->hwrm_cmd_lock);
4718         for (i = 0; i < num_of_vnics; i++) {
4719                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4720
4721                 for (j = 0; j < vnic->uc_filter_count; j++) {
4722                         struct hwrm_cfa_l2_filter_free_input req = {0};
4723
4724                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4725                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4726
4727                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4728
4729                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4730                                                 HWRM_CMD_TIMEOUT);
4731                 }
4732                 vnic->uc_filter_count = 0;
4733         }
4734         mutex_unlock(&bp->hwrm_cmd_lock);
4735
4736         return rc;
4737 }
4738
4739 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4740 {
4741         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4742         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4743         struct hwrm_vnic_tpa_cfg_input req = {0};
4744
4745         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4746                 return 0;
4747
4748         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4749
4750         if (tpa_flags) {
4751                 u16 mss = bp->dev->mtu - 40;
4752                 u32 nsegs, n, segs = 0, flags;
4753
4754                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4755                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4756                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4757                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4758                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4759                 if (tpa_flags & BNXT_FLAG_GRO)
4760                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4761
4762                 req.flags = cpu_to_le32(flags);
4763
4764                 req.enables =
4765                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4766                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4767                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4768
4769                 /* Number of segs are log2 units, and first packet is not
4770                  * included as part of this units.
4771                  */
4772                 if (mss <= BNXT_RX_PAGE_SIZE) {
4773                         n = BNXT_RX_PAGE_SIZE / mss;
4774                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4775                 } else {
4776                         n = mss / BNXT_RX_PAGE_SIZE;
4777                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4778                                 n++;
4779                         nsegs = (MAX_SKB_FRAGS - n) / n;
4780                 }
4781
4782                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4783                         segs = MAX_TPA_SEGS_P5;
4784                         max_aggs = bp->max_tpa;
4785                 } else {
4786                         segs = ilog2(nsegs);
4787                 }
4788                 req.max_agg_segs = cpu_to_le16(segs);
4789                 req.max_aggs = cpu_to_le16(max_aggs);
4790
4791                 req.min_agg_len = cpu_to_le32(512);
4792         }
4793         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4794
4795         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4796 }
4797
4798 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4799 {
4800         struct bnxt_ring_grp_info *grp_info;
4801
4802         grp_info = &bp->grp_info[ring->grp_idx];
4803         return grp_info->cp_fw_ring_id;
4804 }
4805
4806 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4807 {
4808         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4809                 struct bnxt_napi *bnapi = rxr->bnapi;
4810                 struct bnxt_cp_ring_info *cpr;
4811
4812                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4813                 return cpr->cp_ring_struct.fw_ring_id;
4814         } else {
4815                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4816         }
4817 }
4818
4819 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4820 {
4821         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4822                 struct bnxt_napi *bnapi = txr->bnapi;
4823                 struct bnxt_cp_ring_info *cpr;
4824
4825                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4826                 return cpr->cp_ring_struct.fw_ring_id;
4827         } else {
4828                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4829         }
4830 }
4831
4832 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4833 {
4834         u32 i, j, max_rings;
4835         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4836         struct hwrm_vnic_rss_cfg_input req = {0};
4837
4838         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4839             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
4840                 return 0;
4841
4842         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4843         if (set_rss) {
4844                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4845                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4846                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4847                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4848                                 max_rings = bp->rx_nr_rings - 1;
4849                         else
4850                                 max_rings = bp->rx_nr_rings;
4851                 } else {
4852                         max_rings = 1;
4853                 }
4854
4855                 /* Fill the RSS indirection table with ring group ids */
4856                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4857                         if (j == max_rings)
4858                                 j = 0;
4859                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4860                 }
4861
4862                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4863                 req.hash_key_tbl_addr =
4864                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
4865         }
4866         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4867         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4868 }
4869
4870 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4871 {
4872         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4873         u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4874         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4875         struct hwrm_vnic_rss_cfg_input req = {0};
4876
4877         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4878         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4879         if (!set_rss) {
4880                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4881                 return 0;
4882         }
4883         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4884         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4885         req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4886         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4887         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4888         for (i = 0, k = 0; i < nr_ctxs; i++) {
4889                 __le16 *ring_tbl = vnic->rss_table;
4890                 int rc;
4891
4892                 req.ring_table_pair_index = i;
4893                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4894                 for (j = 0; j < 64; j++) {
4895                         u16 ring_id;
4896
4897                         ring_id = rxr->rx_ring_struct.fw_ring_id;
4898                         *ring_tbl++ = cpu_to_le16(ring_id);
4899                         ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4900                         *ring_tbl++ = cpu_to_le16(ring_id);
4901                         rxr++;
4902                         k++;
4903                         if (k == max_rings) {
4904                                 k = 0;
4905                                 rxr = &bp->rx_ring[0];
4906                         }
4907                 }
4908                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4909                 if (rc)
4910                         return rc;
4911         }
4912         return 0;
4913 }
4914
4915 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4916 {
4917         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4918         struct hwrm_vnic_plcmodes_cfg_input req = {0};
4919
4920         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4921         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4922                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4923                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4924         req.enables =
4925                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4926                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4927         /* thresholds not implemented in firmware yet */
4928         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4929         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4930         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4931         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4932 }
4933
4934 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4935                                         u16 ctx_idx)
4936 {
4937         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4938
4939         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4940         req.rss_cos_lb_ctx_id =
4941                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4942
4943         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4944         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4945 }
4946
4947 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4948 {
4949         int i, j;
4950
4951         for (i = 0; i < bp->nr_vnics; i++) {
4952                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4953
4954                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4955                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4956                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4957                 }
4958         }
4959         bp->rsscos_nr_ctxs = 0;
4960 }
4961
4962 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4963 {
4964         int rc;
4965         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4966         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4967                                                 bp->hwrm_cmd_resp_addr;
4968
4969         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4970                                -1);
4971
4972         mutex_lock(&bp->hwrm_cmd_lock);
4973         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4974         if (!rc)
4975                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4976                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
4977         mutex_unlock(&bp->hwrm_cmd_lock);
4978
4979         return rc;
4980 }
4981
4982 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4983 {
4984         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4985                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4986         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4987 }
4988
4989 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
4990 {
4991         unsigned int ring = 0, grp_idx;
4992         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4993         struct hwrm_vnic_cfg_input req = {0};
4994         u16 def_vlan = 0;
4995
4996         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
4997
4998         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4999                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5000
5001                 req.default_rx_ring_id =
5002                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5003                 req.default_cmpl_ring_id =
5004                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5005                 req.enables =
5006                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5007                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5008                 goto vnic_mru;
5009         }
5010         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5011         /* Only RSS support for now TBD: COS & LB */
5012         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5013                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5014                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5015                                            VNIC_CFG_REQ_ENABLES_MRU);
5016         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5017                 req.rss_rule =
5018                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5019                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5020                                            VNIC_CFG_REQ_ENABLES_MRU);
5021                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5022         } else {
5023                 req.rss_rule = cpu_to_le16(0xffff);
5024         }
5025
5026         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5027             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5028                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5029                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5030         } else {
5031                 req.cos_rule = cpu_to_le16(0xffff);
5032         }
5033
5034         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5035                 ring = 0;
5036         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5037                 ring = vnic_id - 1;
5038         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5039                 ring = bp->rx_nr_rings - 1;
5040
5041         grp_idx = bp->rx_ring[ring].bnapi->index;
5042         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5043         req.lb_rule = cpu_to_le16(0xffff);
5044 vnic_mru:
5045         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
5046                               VLAN_HLEN);
5047
5048         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5049 #ifdef CONFIG_BNXT_SRIOV
5050         if (BNXT_VF(bp))
5051                 def_vlan = bp->vf.vlan;
5052 #endif
5053         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5054                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5055         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5056                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5057
5058         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5059 }
5060
5061 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5062 {
5063         u32 rc = 0;
5064
5065         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5066                 struct hwrm_vnic_free_input req = {0};
5067
5068                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5069                 req.vnic_id =
5070                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5071
5072                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5073                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5074         }
5075         return rc;
5076 }
5077
5078 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5079 {
5080         u16 i;
5081
5082         for (i = 0; i < bp->nr_vnics; i++)
5083                 bnxt_hwrm_vnic_free_one(bp, i);
5084 }
5085
5086 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5087                                 unsigned int start_rx_ring_idx,
5088                                 unsigned int nr_rings)
5089 {
5090         int rc = 0;
5091         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5092         struct hwrm_vnic_alloc_input req = {0};
5093         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5094         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5095
5096         if (bp->flags & BNXT_FLAG_CHIP_P5)
5097                 goto vnic_no_ring_grps;
5098
5099         /* map ring groups to this vnic */
5100         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5101                 grp_idx = bp->rx_ring[i].bnapi->index;
5102                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5103                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5104                                    j, nr_rings);
5105                         break;
5106                 }
5107                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5108         }
5109
5110 vnic_no_ring_grps:
5111         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5112                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5113         if (vnic_id == 0)
5114                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5115
5116         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5117
5118         mutex_lock(&bp->hwrm_cmd_lock);
5119         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5120         if (!rc)
5121                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5122         mutex_unlock(&bp->hwrm_cmd_lock);
5123         return rc;
5124 }
5125
5126 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5127 {
5128         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5129         struct hwrm_vnic_qcaps_input req = {0};
5130         int rc;
5131
5132         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5133         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5134         if (bp->hwrm_spec_code < 0x10600)
5135                 return 0;
5136
5137         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5138         mutex_lock(&bp->hwrm_cmd_lock);
5139         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5140         if (!rc) {
5141                 u32 flags = le32_to_cpu(resp->flags);
5142
5143                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5144                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5145                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5146                 if (flags &
5147                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5148                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5149                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5150                 if (bp->max_tpa_v2)
5151                         bp->hw_ring_stats_size =
5152                                 sizeof(struct ctx_hw_stats_ext);
5153         }
5154         mutex_unlock(&bp->hwrm_cmd_lock);
5155         return rc;
5156 }
5157
5158 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5159 {
5160         u16 i;
5161         u32 rc = 0;
5162
5163         if (bp->flags & BNXT_FLAG_CHIP_P5)
5164                 return 0;
5165
5166         mutex_lock(&bp->hwrm_cmd_lock);
5167         for (i = 0; i < bp->rx_nr_rings; i++) {
5168                 struct hwrm_ring_grp_alloc_input req = {0};
5169                 struct hwrm_ring_grp_alloc_output *resp =
5170                                         bp->hwrm_cmd_resp_addr;
5171                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5172
5173                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5174
5175                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5176                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5177                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5178                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5179
5180                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5181                                         HWRM_CMD_TIMEOUT);
5182                 if (rc)
5183                         break;
5184
5185                 bp->grp_info[grp_idx].fw_grp_id =
5186                         le32_to_cpu(resp->ring_group_id);
5187         }
5188         mutex_unlock(&bp->hwrm_cmd_lock);
5189         return rc;
5190 }
5191
5192 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5193 {
5194         u16 i;
5195         u32 rc = 0;
5196         struct hwrm_ring_grp_free_input req = {0};
5197
5198         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5199                 return 0;
5200
5201         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5202
5203         mutex_lock(&bp->hwrm_cmd_lock);
5204         for (i = 0; i < bp->cp_nr_rings; i++) {
5205                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5206                         continue;
5207                 req.ring_group_id =
5208                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5209
5210                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5211                                         HWRM_CMD_TIMEOUT);
5212                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5213         }
5214         mutex_unlock(&bp->hwrm_cmd_lock);
5215         return rc;
5216 }
5217
5218 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5219                                     struct bnxt_ring_struct *ring,
5220                                     u32 ring_type, u32 map_index)
5221 {
5222         int rc = 0, err = 0;
5223         struct hwrm_ring_alloc_input req = {0};
5224         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5225         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5226         struct bnxt_ring_grp_info *grp_info;
5227         u16 ring_id;
5228
5229         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5230
5231         req.enables = 0;
5232         if (rmem->nr_pages > 1) {
5233                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5234                 /* Page size is in log2 units */
5235                 req.page_size = BNXT_PAGE_SHIFT;
5236                 req.page_tbl_depth = 1;
5237         } else {
5238                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5239         }
5240         req.fbo = 0;
5241         /* Association of ring index with doorbell index and MSIX number */
5242         req.logical_id = cpu_to_le16(map_index);
5243
5244         switch (ring_type) {
5245         case HWRM_RING_ALLOC_TX: {
5246                 struct bnxt_tx_ring_info *txr;
5247
5248                 txr = container_of(ring, struct bnxt_tx_ring_info,
5249                                    tx_ring_struct);
5250                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5251                 /* Association of transmit ring with completion ring */
5252                 grp_info = &bp->grp_info[ring->grp_idx];
5253                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5254                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5255                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5256                 req.queue_id = cpu_to_le16(ring->queue_id);
5257                 break;
5258         }
5259         case HWRM_RING_ALLOC_RX:
5260                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5261                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5262                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5263                         u16 flags = 0;
5264
5265                         /* Association of rx ring with stats context */
5266                         grp_info = &bp->grp_info[ring->grp_idx];
5267                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5268                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5269                         req.enables |= cpu_to_le32(
5270                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5271                         if (NET_IP_ALIGN == 2)
5272                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5273                         req.flags = cpu_to_le16(flags);
5274                 }
5275                 break;
5276         case HWRM_RING_ALLOC_AGG:
5277                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5278                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5279                         /* Association of agg ring with rx ring */
5280                         grp_info = &bp->grp_info[ring->grp_idx];
5281                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5282                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5283                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5284                         req.enables |= cpu_to_le32(
5285                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5286                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5287                 } else {
5288                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5289                 }
5290                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5291                 break;
5292         case HWRM_RING_ALLOC_CMPL:
5293                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5294                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5295                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5296                         /* Association of cp ring with nq */
5297                         grp_info = &bp->grp_info[map_index];
5298                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5299                         req.cq_handle = cpu_to_le64(ring->handle);
5300                         req.enables |= cpu_to_le32(
5301                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5302                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5303                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5304                 }
5305                 break;
5306         case HWRM_RING_ALLOC_NQ:
5307                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5308                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5309                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5310                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5311                 break;
5312         default:
5313                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5314                            ring_type);
5315                 return -1;
5316         }
5317
5318         mutex_lock(&bp->hwrm_cmd_lock);
5319         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5320         err = le16_to_cpu(resp->error_code);
5321         ring_id = le16_to_cpu(resp->ring_id);
5322         mutex_unlock(&bp->hwrm_cmd_lock);
5323
5324         if (rc || err) {
5325                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5326                            ring_type, rc, err);
5327                 return -EIO;
5328         }
5329         ring->fw_ring_id = ring_id;
5330         return rc;
5331 }
5332
5333 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5334 {
5335         int rc;
5336
5337         if (BNXT_PF(bp)) {
5338                 struct hwrm_func_cfg_input req = {0};
5339
5340                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5341                 req.fid = cpu_to_le16(0xffff);
5342                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5343                 req.async_event_cr = cpu_to_le16(idx);
5344                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5345         } else {
5346                 struct hwrm_func_vf_cfg_input req = {0};
5347
5348                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5349                 req.enables =
5350                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5351                 req.async_event_cr = cpu_to_le16(idx);
5352                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5353         }
5354         return rc;
5355 }
5356
5357 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5358                         u32 map_idx, u32 xid)
5359 {
5360         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5361                 if (BNXT_PF(bp))
5362                         db->doorbell = bp->bar1 + 0x10000;
5363                 else
5364                         db->doorbell = bp->bar1 + 0x4000;
5365                 switch (ring_type) {
5366                 case HWRM_RING_ALLOC_TX:
5367                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5368                         break;
5369                 case HWRM_RING_ALLOC_RX:
5370                 case HWRM_RING_ALLOC_AGG:
5371                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5372                         break;
5373                 case HWRM_RING_ALLOC_CMPL:
5374                         db->db_key64 = DBR_PATH_L2;
5375                         break;
5376                 case HWRM_RING_ALLOC_NQ:
5377                         db->db_key64 = DBR_PATH_L2;
5378                         break;
5379                 }
5380                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5381         } else {
5382                 db->doorbell = bp->bar1 + map_idx * 0x80;
5383                 switch (ring_type) {
5384                 case HWRM_RING_ALLOC_TX:
5385                         db->db_key32 = DB_KEY_TX;
5386                         break;
5387                 case HWRM_RING_ALLOC_RX:
5388                 case HWRM_RING_ALLOC_AGG:
5389                         db->db_key32 = DB_KEY_RX;
5390                         break;
5391                 case HWRM_RING_ALLOC_CMPL:
5392                         db->db_key32 = DB_KEY_CP;
5393                         break;
5394                 }
5395         }
5396 }
5397
5398 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5399 {
5400         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5401         int i, rc = 0;
5402         u32 type;
5403
5404         if (bp->flags & BNXT_FLAG_CHIP_P5)
5405                 type = HWRM_RING_ALLOC_NQ;
5406         else
5407                 type = HWRM_RING_ALLOC_CMPL;
5408         for (i = 0; i < bp->cp_nr_rings; i++) {
5409                 struct bnxt_napi *bnapi = bp->bnapi[i];
5410                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5411                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5412                 u32 map_idx = ring->map_idx;
5413                 unsigned int vector;
5414
5415                 vector = bp->irq_tbl[map_idx].vector;
5416                 disable_irq_nosync(vector);
5417                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5418                 if (rc) {
5419                         enable_irq(vector);
5420                         goto err_out;
5421                 }
5422                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5423                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5424                 enable_irq(vector);
5425                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5426
5427                 if (!i) {
5428                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5429                         if (rc)
5430                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5431                 }
5432         }
5433
5434         type = HWRM_RING_ALLOC_TX;
5435         for (i = 0; i < bp->tx_nr_rings; i++) {
5436                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5437                 struct bnxt_ring_struct *ring;
5438                 u32 map_idx;
5439
5440                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5441                         struct bnxt_napi *bnapi = txr->bnapi;
5442                         struct bnxt_cp_ring_info *cpr, *cpr2;
5443                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5444
5445                         cpr = &bnapi->cp_ring;
5446                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5447                         ring = &cpr2->cp_ring_struct;
5448                         ring->handle = BNXT_TX_HDL;
5449                         map_idx = bnapi->index;
5450                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5451                         if (rc)
5452                                 goto err_out;
5453                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5454                                     ring->fw_ring_id);
5455                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5456                 }
5457                 ring = &txr->tx_ring_struct;
5458                 map_idx = i;
5459                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5460                 if (rc)
5461                         goto err_out;
5462                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5463         }
5464
5465         type = HWRM_RING_ALLOC_RX;
5466         for (i = 0; i < bp->rx_nr_rings; i++) {
5467                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5468                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5469                 struct bnxt_napi *bnapi = rxr->bnapi;
5470                 u32 map_idx = bnapi->index;
5471
5472                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5473                 if (rc)
5474                         goto err_out;
5475                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5476                 /* If we have agg rings, post agg buffers first. */
5477                 if (!agg_rings)
5478                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5479                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5480                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5481                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5482                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5483                         struct bnxt_cp_ring_info *cpr2;
5484
5485                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5486                         ring = &cpr2->cp_ring_struct;
5487                         ring->handle = BNXT_RX_HDL;
5488                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5489                         if (rc)
5490                                 goto err_out;
5491                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5492                                     ring->fw_ring_id);
5493                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5494                 }
5495         }
5496
5497         if (agg_rings) {
5498                 type = HWRM_RING_ALLOC_AGG;
5499                 for (i = 0; i < bp->rx_nr_rings; i++) {
5500                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5501                         struct bnxt_ring_struct *ring =
5502                                                 &rxr->rx_agg_ring_struct;
5503                         u32 grp_idx = ring->grp_idx;
5504                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5505
5506                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5507                         if (rc)
5508                                 goto err_out;
5509
5510                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5511                                     ring->fw_ring_id);
5512                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5513                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5514                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5515                 }
5516         }
5517 err_out:
5518         return rc;
5519 }
5520
5521 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5522                                    struct bnxt_ring_struct *ring,
5523                                    u32 ring_type, int cmpl_ring_id)
5524 {
5525         int rc;
5526         struct hwrm_ring_free_input req = {0};
5527         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5528         u16 error_code;
5529
5530         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
5531                 return 0;
5532
5533         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5534         req.ring_type = ring_type;
5535         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5536
5537         mutex_lock(&bp->hwrm_cmd_lock);
5538         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5539         error_code = le16_to_cpu(resp->error_code);
5540         mutex_unlock(&bp->hwrm_cmd_lock);
5541
5542         if (rc || error_code) {
5543                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5544                            ring_type, rc, error_code);
5545                 return -EIO;
5546         }
5547         return 0;
5548 }
5549
5550 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5551 {
5552         u32 type;
5553         int i;
5554
5555         if (!bp->bnapi)
5556                 return;
5557
5558         for (i = 0; i < bp->tx_nr_rings; i++) {
5559                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5560                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5561
5562                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5563                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5564
5565                         hwrm_ring_free_send_msg(bp, ring,
5566                                                 RING_FREE_REQ_RING_TYPE_TX,
5567                                                 close_path ? cmpl_ring_id :
5568                                                 INVALID_HW_RING_ID);
5569                         ring->fw_ring_id = INVALID_HW_RING_ID;
5570                 }
5571         }
5572
5573         for (i = 0; i < bp->rx_nr_rings; i++) {
5574                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5575                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5576                 u32 grp_idx = rxr->bnapi->index;
5577
5578                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5579                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5580
5581                         hwrm_ring_free_send_msg(bp, ring,
5582                                                 RING_FREE_REQ_RING_TYPE_RX,
5583                                                 close_path ? cmpl_ring_id :
5584                                                 INVALID_HW_RING_ID);
5585                         ring->fw_ring_id = INVALID_HW_RING_ID;
5586                         bp->grp_info[grp_idx].rx_fw_ring_id =
5587                                 INVALID_HW_RING_ID;
5588                 }
5589         }
5590
5591         if (bp->flags & BNXT_FLAG_CHIP_P5)
5592                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5593         else
5594                 type = RING_FREE_REQ_RING_TYPE_RX;
5595         for (i = 0; i < bp->rx_nr_rings; i++) {
5596                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5597                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5598                 u32 grp_idx = rxr->bnapi->index;
5599
5600                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5601                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5602
5603                         hwrm_ring_free_send_msg(bp, ring, type,
5604                                                 close_path ? cmpl_ring_id :
5605                                                 INVALID_HW_RING_ID);
5606                         ring->fw_ring_id = INVALID_HW_RING_ID;
5607                         bp->grp_info[grp_idx].agg_fw_ring_id =
5608                                 INVALID_HW_RING_ID;
5609                 }
5610         }
5611
5612         /* The completion rings are about to be freed.  After that the
5613          * IRQ doorbell will not work anymore.  So we need to disable
5614          * IRQ here.
5615          */
5616         bnxt_disable_int_sync(bp);
5617
5618         if (bp->flags & BNXT_FLAG_CHIP_P5)
5619                 type = RING_FREE_REQ_RING_TYPE_NQ;
5620         else
5621                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5622         for (i = 0; i < bp->cp_nr_rings; i++) {
5623                 struct bnxt_napi *bnapi = bp->bnapi[i];
5624                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5625                 struct bnxt_ring_struct *ring;
5626                 int j;
5627
5628                 for (j = 0; j < 2; j++) {
5629                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5630
5631                         if (cpr2) {
5632                                 ring = &cpr2->cp_ring_struct;
5633                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5634                                         continue;
5635                                 hwrm_ring_free_send_msg(bp, ring,
5636                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5637                                         INVALID_HW_RING_ID);
5638                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5639                         }
5640                 }
5641                 ring = &cpr->cp_ring_struct;
5642                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5643                         hwrm_ring_free_send_msg(bp, ring, type,
5644                                                 INVALID_HW_RING_ID);
5645                         ring->fw_ring_id = INVALID_HW_RING_ID;
5646                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5647                 }
5648         }
5649 }
5650
5651 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5652                            bool shared);
5653
5654 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5655 {
5656         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5657         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5658         struct hwrm_func_qcfg_input req = {0};
5659         int rc;
5660
5661         if (bp->hwrm_spec_code < 0x10601)
5662                 return 0;
5663
5664         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5665         req.fid = cpu_to_le16(0xffff);
5666         mutex_lock(&bp->hwrm_cmd_lock);
5667         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5668         if (rc) {
5669                 mutex_unlock(&bp->hwrm_cmd_lock);
5670                 return rc;
5671         }
5672
5673         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5674         if (BNXT_NEW_RM(bp)) {
5675                 u16 cp, stats;
5676
5677                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5678                 hw_resc->resv_hw_ring_grps =
5679                         le32_to_cpu(resp->alloc_hw_ring_grps);
5680                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5681                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5682                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5683                 hw_resc->resv_irqs = cp;
5684                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5685                         int rx = hw_resc->resv_rx_rings;
5686                         int tx = hw_resc->resv_tx_rings;
5687
5688                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5689                                 rx >>= 1;
5690                         if (cp < (rx + tx)) {
5691                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5692                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5693                                         rx <<= 1;
5694                                 hw_resc->resv_rx_rings = rx;
5695                                 hw_resc->resv_tx_rings = tx;
5696                         }
5697                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5698                         hw_resc->resv_hw_ring_grps = rx;
5699                 }
5700                 hw_resc->resv_cp_rings = cp;
5701                 hw_resc->resv_stat_ctxs = stats;
5702         }
5703         mutex_unlock(&bp->hwrm_cmd_lock);
5704         return 0;
5705 }
5706
5707 /* Caller must hold bp->hwrm_cmd_lock */
5708 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5709 {
5710         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5711         struct hwrm_func_qcfg_input req = {0};
5712         int rc;
5713
5714         if (bp->hwrm_spec_code < 0x10601)
5715                 return 0;
5716
5717         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5718         req.fid = cpu_to_le16(fid);
5719         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5720         if (!rc)
5721                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5722
5723         return rc;
5724 }
5725
5726 static bool bnxt_rfs_supported(struct bnxt *bp);
5727
5728 static void
5729 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5730                              int tx_rings, int rx_rings, int ring_grps,
5731                              int cp_rings, int stats, int vnics)
5732 {
5733         u32 enables = 0;
5734
5735         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5736         req->fid = cpu_to_le16(0xffff);
5737         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5738         req->num_tx_rings = cpu_to_le16(tx_rings);
5739         if (BNXT_NEW_RM(bp)) {
5740                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5741                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5742                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5743                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5744                         enables |= tx_rings + ring_grps ?
5745                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5746                         enables |= rx_rings ?
5747                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5748                 } else {
5749                         enables |= cp_rings ?
5750                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5751                         enables |= ring_grps ?
5752                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5753                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5754                 }
5755                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5756
5757                 req->num_rx_rings = cpu_to_le16(rx_rings);
5758                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5759                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5760                         req->num_msix = cpu_to_le16(cp_rings);
5761                         req->num_rsscos_ctxs =
5762                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5763                 } else {
5764                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
5765                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5766                         req->num_rsscos_ctxs = cpu_to_le16(1);
5767                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5768                             bnxt_rfs_supported(bp))
5769                                 req->num_rsscos_ctxs =
5770                                         cpu_to_le16(ring_grps + 1);
5771                 }
5772                 req->num_stat_ctxs = cpu_to_le16(stats);
5773                 req->num_vnics = cpu_to_le16(vnics);
5774         }
5775         req->enables = cpu_to_le32(enables);
5776 }
5777
5778 static void
5779 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5780                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
5781                              int rx_rings, int ring_grps, int cp_rings,
5782                              int stats, int vnics)
5783 {
5784         u32 enables = 0;
5785
5786         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5787         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5788         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5789                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5790         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5791         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5792                 enables |= tx_rings + ring_grps ?
5793                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5794         } else {
5795                 enables |= cp_rings ?
5796                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5797                 enables |= ring_grps ?
5798                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5799         }
5800         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5801         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5802
5803         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
5804         req->num_tx_rings = cpu_to_le16(tx_rings);
5805         req->num_rx_rings = cpu_to_le16(rx_rings);
5806         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5807                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5808                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5809         } else {
5810                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5811                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5812                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5813         }
5814         req->num_stat_ctxs = cpu_to_le16(stats);
5815         req->num_vnics = cpu_to_le16(vnics);
5816
5817         req->enables = cpu_to_le32(enables);
5818 }
5819
5820 static int
5821 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5822                            int ring_grps, int cp_rings, int stats, int vnics)
5823 {
5824         struct hwrm_func_cfg_input req = {0};
5825         int rc;
5826
5827         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5828                                      cp_rings, stats, vnics);
5829         if (!req.enables)
5830                 return 0;
5831
5832         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5833         if (rc)
5834                 return rc;
5835
5836         if (bp->hwrm_spec_code < 0x10601)
5837                 bp->hw_resc.resv_tx_rings = tx_rings;
5838
5839         rc = bnxt_hwrm_get_rings(bp);
5840         return rc;
5841 }
5842
5843 static int
5844 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5845                            int ring_grps, int cp_rings, int stats, int vnics)
5846 {
5847         struct hwrm_func_vf_cfg_input req = {0};
5848         int rc;
5849
5850         if (!BNXT_NEW_RM(bp)) {
5851                 bp->hw_resc.resv_tx_rings = tx_rings;
5852                 return 0;
5853         }
5854
5855         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5856                                      cp_rings, stats, vnics);
5857         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5858         if (rc)
5859                 return rc;
5860
5861         rc = bnxt_hwrm_get_rings(bp);
5862         return rc;
5863 }
5864
5865 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
5866                                    int cp, int stat, int vnic)
5867 {
5868         if (BNXT_PF(bp))
5869                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5870                                                   vnic);
5871         else
5872                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5873                                                   vnic);
5874 }
5875
5876 int bnxt_nq_rings_in_use(struct bnxt *bp)
5877 {
5878         int cp = bp->cp_nr_rings;
5879         int ulp_msix, ulp_base;
5880
5881         ulp_msix = bnxt_get_ulp_msix_num(bp);
5882         if (ulp_msix) {
5883                 ulp_base = bnxt_get_ulp_msix_base(bp);
5884                 cp += ulp_msix;
5885                 if ((ulp_base + ulp_msix) > cp)
5886                         cp = ulp_base + ulp_msix;
5887         }
5888         return cp;
5889 }
5890
5891 static int bnxt_cp_rings_in_use(struct bnxt *bp)
5892 {
5893         int cp;
5894
5895         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5896                 return bnxt_nq_rings_in_use(bp);
5897
5898         cp = bp->tx_nr_rings + bp->rx_nr_rings;
5899         return cp;
5900 }
5901
5902 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5903 {
5904         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
5905         int cp = bp->cp_nr_rings;
5906
5907         if (!ulp_stat)
5908                 return cp;
5909
5910         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
5911                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
5912
5913         return cp + ulp_stat;
5914 }
5915
5916 static bool bnxt_need_reserve_rings(struct bnxt *bp)
5917 {
5918         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5919         int cp = bnxt_cp_rings_in_use(bp);
5920         int nq = bnxt_nq_rings_in_use(bp);
5921         int rx = bp->rx_nr_rings, stat;
5922         int vnic = 1, grp = rx;
5923
5924         if (bp->hwrm_spec_code < 0x10601)
5925                 return false;
5926
5927         if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5928                 return true;
5929
5930         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5931                 vnic = rx + 1;
5932         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5933                 rx <<= 1;
5934         stat = bnxt_get_func_stat_ctxs(bp);
5935         if (BNXT_NEW_RM(bp) &&
5936             (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
5937              hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
5938              (hw_resc->resv_hw_ring_grps != grp &&
5939               !(bp->flags & BNXT_FLAG_CHIP_P5))))
5940                 return true;
5941         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
5942             hw_resc->resv_irqs != nq)
5943                 return true;
5944         return false;
5945 }
5946
5947 static int __bnxt_reserve_rings(struct bnxt *bp)
5948 {
5949         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5950         int cp = bnxt_nq_rings_in_use(bp);
5951         int tx = bp->tx_nr_rings;
5952         int rx = bp->rx_nr_rings;
5953         int grp, rx_rings, rc;
5954         int vnic = 1, stat;
5955         bool sh = false;
5956
5957         if (!bnxt_need_reserve_rings(bp))
5958                 return 0;
5959
5960         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5961                 sh = true;
5962         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5963                 vnic = rx + 1;
5964         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5965                 rx <<= 1;
5966         grp = bp->rx_nr_rings;
5967         stat = bnxt_get_func_stat_ctxs(bp);
5968
5969         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
5970         if (rc)
5971                 return rc;
5972
5973         tx = hw_resc->resv_tx_rings;
5974         if (BNXT_NEW_RM(bp)) {
5975                 rx = hw_resc->resv_rx_rings;
5976                 cp = hw_resc->resv_irqs;
5977                 grp = hw_resc->resv_hw_ring_grps;
5978                 vnic = hw_resc->resv_vnics;
5979                 stat = hw_resc->resv_stat_ctxs;
5980         }
5981
5982         rx_rings = rx;
5983         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5984                 if (rx >= 2) {
5985                         rx_rings = rx >> 1;
5986                 } else {
5987                         if (netif_running(bp->dev))
5988                                 return -ENOMEM;
5989
5990                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5991                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5992                         bp->dev->hw_features &= ~NETIF_F_LRO;
5993                         bp->dev->features &= ~NETIF_F_LRO;
5994                         bnxt_set_ring_params(bp);
5995                 }
5996         }
5997         rx_rings = min_t(int, rx_rings, grp);
5998         cp = min_t(int, cp, bp->cp_nr_rings);
5999         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6000                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6001         cp = min_t(int, cp, stat);
6002         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6003         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6004                 rx = rx_rings << 1;
6005         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6006         bp->tx_nr_rings = tx;
6007         bp->rx_nr_rings = rx_rings;
6008         bp->cp_nr_rings = cp;
6009
6010         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6011                 return -ENOMEM;
6012
6013         return rc;
6014 }
6015
6016 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6017                                     int ring_grps, int cp_rings, int stats,
6018                                     int vnics)
6019 {
6020         struct hwrm_func_vf_cfg_input req = {0};
6021         u32 flags;
6022         int rc;
6023
6024         if (!BNXT_NEW_RM(bp))
6025                 return 0;
6026
6027         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6028                                      cp_rings, stats, vnics);
6029         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6030                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6031                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6032                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6033                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6034                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6035         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6036                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6037
6038         req.flags = cpu_to_le32(flags);
6039         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6040         return rc;
6041 }
6042
6043 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6044                                     int ring_grps, int cp_rings, int stats,
6045                                     int vnics)
6046 {
6047         struct hwrm_func_cfg_input req = {0};
6048         u32 flags;
6049         int rc;
6050
6051         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6052                                      cp_rings, stats, vnics);
6053         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6054         if (BNXT_NEW_RM(bp)) {
6055                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6056                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6057                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6058                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6059                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6060                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6061                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6062                 else
6063                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6064         }
6065
6066         req.flags = cpu_to_le32(flags);
6067         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6068         return rc;
6069 }
6070
6071 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6072                                  int ring_grps, int cp_rings, int stats,
6073                                  int vnics)
6074 {
6075         if (bp->hwrm_spec_code < 0x10801)
6076                 return 0;
6077
6078         if (BNXT_PF(bp))
6079                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6080                                                 ring_grps, cp_rings, stats,
6081                                                 vnics);
6082
6083         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6084                                         cp_rings, stats, vnics);
6085 }
6086
6087 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6088 {
6089         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6090         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6091         struct hwrm_ring_aggint_qcaps_input req = {0};
6092         int rc;
6093
6094         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6095         coal_cap->num_cmpl_dma_aggr_max = 63;
6096         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6097         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6098         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6099         coal_cap->int_lat_tmr_min_max = 65535;
6100         coal_cap->int_lat_tmr_max_max = 65535;
6101         coal_cap->num_cmpl_aggr_int_max = 65535;
6102         coal_cap->timer_units = 80;
6103
6104         if (bp->hwrm_spec_code < 0x10902)
6105                 return;
6106
6107         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6108         mutex_lock(&bp->hwrm_cmd_lock);
6109         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6110         if (!rc) {
6111                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6112                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6113                 coal_cap->num_cmpl_dma_aggr_max =
6114                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6115                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6116                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6117                 coal_cap->cmpl_aggr_dma_tmr_max =
6118                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6119                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6120                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6121                 coal_cap->int_lat_tmr_min_max =
6122                         le16_to_cpu(resp->int_lat_tmr_min_max);
6123                 coal_cap->int_lat_tmr_max_max =
6124                         le16_to_cpu(resp->int_lat_tmr_max_max);
6125                 coal_cap->num_cmpl_aggr_int_max =
6126                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6127                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6128         }
6129         mutex_unlock(&bp->hwrm_cmd_lock);
6130 }
6131
6132 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6133 {
6134         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6135
6136         return usec * 1000 / coal_cap->timer_units;
6137 }
6138
6139 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6140         struct bnxt_coal *hw_coal,
6141         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6142 {
6143         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6144         u32 cmpl_params = coal_cap->cmpl_params;
6145         u16 val, tmr, max, flags = 0;
6146
6147         max = hw_coal->bufs_per_record * 128;
6148         if (hw_coal->budget)
6149                 max = hw_coal->bufs_per_record * hw_coal->budget;
6150         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6151
6152         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6153         req->num_cmpl_aggr_int = cpu_to_le16(val);
6154
6155         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6156         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6157
6158         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6159                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6160         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6161
6162         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6163         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6164         req->int_lat_tmr_max = cpu_to_le16(tmr);
6165
6166         /* min timer set to 1/2 of interrupt timer */
6167         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6168                 val = tmr / 2;
6169                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6170                 req->int_lat_tmr_min = cpu_to_le16(val);
6171                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6172         }
6173
6174         /* buf timer set to 1/4 of interrupt timer */
6175         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6176         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6177
6178         if (cmpl_params &
6179             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6180                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6181                 val = clamp_t(u16, tmr, 1,
6182                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6183                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
6184                 req->enables |=
6185                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6186         }
6187
6188         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6189                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6190         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6191             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6192                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6193         req->flags = cpu_to_le16(flags);
6194         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6195 }
6196
6197 /* Caller holds bp->hwrm_cmd_lock */
6198 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6199                                    struct bnxt_coal *hw_coal)
6200 {
6201         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6202         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6203         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6204         u32 nq_params = coal_cap->nq_params;
6205         u16 tmr;
6206
6207         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6208                 return 0;
6209
6210         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6211                                -1, -1);
6212         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6213         req.flags =
6214                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6215
6216         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6217         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6218         req.int_lat_tmr_min = cpu_to_le16(tmr);
6219         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6220         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6221 }
6222
6223 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6224 {
6225         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6226         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6227         struct bnxt_coal coal;
6228
6229         /* Tick values in micro seconds.
6230          * 1 coal_buf x bufs_per_record = 1 completion record.
6231          */
6232         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6233
6234         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6235         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6236
6237         if (!bnapi->rx_ring)
6238                 return -ENODEV;
6239
6240         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6241                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6242
6243         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6244
6245         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6246
6247         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6248                                  HWRM_CMD_TIMEOUT);
6249 }
6250
6251 int bnxt_hwrm_set_coal(struct bnxt *bp)
6252 {
6253         int i, rc = 0;
6254         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6255                                                            req_tx = {0}, *req;
6256
6257         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6258                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6259         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6260                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6261
6262         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6263         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6264
6265         mutex_lock(&bp->hwrm_cmd_lock);
6266         for (i = 0; i < bp->cp_nr_rings; i++) {
6267                 struct bnxt_napi *bnapi = bp->bnapi[i];
6268                 struct bnxt_coal *hw_coal;
6269                 u16 ring_id;
6270
6271                 req = &req_rx;
6272                 if (!bnapi->rx_ring) {
6273                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6274                         req = &req_tx;
6275                 } else {
6276                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6277                 }
6278                 req->ring_id = cpu_to_le16(ring_id);
6279
6280                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6281                                         HWRM_CMD_TIMEOUT);
6282                 if (rc)
6283                         break;
6284
6285                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6286                         continue;
6287
6288                 if (bnapi->rx_ring && bnapi->tx_ring) {
6289                         req = &req_tx;
6290                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6291                         req->ring_id = cpu_to_le16(ring_id);
6292                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6293                                                 HWRM_CMD_TIMEOUT);
6294                         if (rc)
6295                                 break;
6296                 }
6297                 if (bnapi->rx_ring)
6298                         hw_coal = &bp->rx_coal;
6299                 else
6300                         hw_coal = &bp->tx_coal;
6301                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6302         }
6303         mutex_unlock(&bp->hwrm_cmd_lock);
6304         return rc;
6305 }
6306
6307 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6308 {
6309         int rc = 0, i;
6310         struct hwrm_stat_ctx_free_input req = {0};
6311
6312         if (!bp->bnapi)
6313                 return 0;
6314
6315         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6316                 return 0;
6317
6318         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6319
6320         mutex_lock(&bp->hwrm_cmd_lock);
6321         for (i = 0; i < bp->cp_nr_rings; i++) {
6322                 struct bnxt_napi *bnapi = bp->bnapi[i];
6323                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6324
6325                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6326                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6327
6328                         rc = _hwrm_send_message(bp, &req, sizeof(req),
6329                                                 HWRM_CMD_TIMEOUT);
6330
6331                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6332                 }
6333         }
6334         mutex_unlock(&bp->hwrm_cmd_lock);
6335         return rc;
6336 }
6337
6338 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6339 {
6340         int rc = 0, i;
6341         struct hwrm_stat_ctx_alloc_input req = {0};
6342         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6343
6344         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6345                 return 0;
6346
6347         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6348
6349         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6350         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6351
6352         mutex_lock(&bp->hwrm_cmd_lock);
6353         for (i = 0; i < bp->cp_nr_rings; i++) {
6354                 struct bnxt_napi *bnapi = bp->bnapi[i];
6355                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6356
6357                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
6358
6359                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6360                                         HWRM_CMD_TIMEOUT);
6361                 if (rc)
6362                         break;
6363
6364                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6365
6366                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6367         }
6368         mutex_unlock(&bp->hwrm_cmd_lock);
6369         return rc;
6370 }
6371
6372 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6373 {
6374         struct hwrm_func_qcfg_input req = {0};
6375         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6376         u16 flags;
6377         int rc;
6378
6379         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6380         req.fid = cpu_to_le16(0xffff);
6381         mutex_lock(&bp->hwrm_cmd_lock);
6382         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6383         if (rc)
6384                 goto func_qcfg_exit;
6385
6386 #ifdef CONFIG_BNXT_SRIOV
6387         if (BNXT_VF(bp)) {
6388                 struct bnxt_vf_info *vf = &bp->vf;
6389
6390                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6391         } else {
6392                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6393         }
6394 #endif
6395         flags = le16_to_cpu(resp->flags);
6396         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6397                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6398                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6399                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6400                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6401         }
6402         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6403                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6404
6405         switch (resp->port_partition_type) {
6406         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6407         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6408         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6409                 bp->port_partition_type = resp->port_partition_type;
6410                 break;
6411         }
6412         if (bp->hwrm_spec_code < 0x10707 ||
6413             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6414                 bp->br_mode = BRIDGE_MODE_VEB;
6415         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6416                 bp->br_mode = BRIDGE_MODE_VEPA;
6417         else
6418                 bp->br_mode = BRIDGE_MODE_UNDEF;
6419
6420         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6421         if (!bp->max_mtu)
6422                 bp->max_mtu = BNXT_MAX_MTU;
6423
6424 func_qcfg_exit:
6425         mutex_unlock(&bp->hwrm_cmd_lock);
6426         return rc;
6427 }
6428
6429 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6430 {
6431         struct hwrm_func_backing_store_qcaps_input req = {0};
6432         struct hwrm_func_backing_store_qcaps_output *resp =
6433                 bp->hwrm_cmd_resp_addr;
6434         int rc;
6435
6436         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6437                 return 0;
6438
6439         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6440         mutex_lock(&bp->hwrm_cmd_lock);
6441         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6442         if (!rc) {
6443                 struct bnxt_ctx_pg_info *ctx_pg;
6444                 struct bnxt_ctx_mem_info *ctx;
6445                 int i;
6446
6447                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6448                 if (!ctx) {
6449                         rc = -ENOMEM;
6450                         goto ctx_err;
6451                 }
6452                 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6453                 if (!ctx_pg) {
6454                         kfree(ctx);
6455                         rc = -ENOMEM;
6456                         goto ctx_err;
6457                 }
6458                 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6459                         ctx->tqm_mem[i] = ctx_pg;
6460
6461                 bp->ctx = ctx;
6462                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6463                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6464                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6465                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6466                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6467                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6468                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6469                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6470                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6471                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6472                 ctx->vnic_max_vnic_entries =
6473                         le16_to_cpu(resp->vnic_max_vnic_entries);
6474                 ctx->vnic_max_ring_table_entries =
6475                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6476                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6477                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6478                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6479                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6480                 ctx->tqm_min_entries_per_ring =
6481                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6482                 ctx->tqm_max_entries_per_ring =
6483                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6484                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6485                 if (!ctx->tqm_entries_multiple)
6486                         ctx->tqm_entries_multiple = 1;
6487                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6488                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6489                 ctx->mrav_num_entries_units =
6490                         le16_to_cpu(resp->mrav_num_entries_units);
6491                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6492                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6493         } else {
6494                 rc = 0;
6495         }
6496 ctx_err:
6497         mutex_unlock(&bp->hwrm_cmd_lock);
6498         return rc;
6499 }
6500
6501 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6502                                   __le64 *pg_dir)
6503 {
6504         u8 pg_size = 0;
6505
6506         if (BNXT_PAGE_SHIFT == 13)
6507                 pg_size = 1 << 4;
6508         else if (BNXT_PAGE_SIZE == 16)
6509                 pg_size = 2 << 4;
6510
6511         *pg_attr = pg_size;
6512         if (rmem->depth >= 1) {
6513                 if (rmem->depth == 2)
6514                         *pg_attr |= 2;
6515                 else
6516                         *pg_attr |= 1;
6517                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6518         } else {
6519                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6520         }
6521 }
6522
6523 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6524         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6525          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6526          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6527          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6528          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6529
6530 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6531 {
6532         struct hwrm_func_backing_store_cfg_input req = {0};
6533         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6534         struct bnxt_ctx_pg_info *ctx_pg;
6535         __le32 *num_entries;
6536         __le64 *pg_dir;
6537         u32 flags = 0;
6538         u8 *pg_attr;
6539         int i, rc;
6540         u32 ena;
6541
6542         if (!ctx)
6543                 return 0;
6544
6545         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6546         req.enables = cpu_to_le32(enables);
6547
6548         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6549                 ctx_pg = &ctx->qp_mem;
6550                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6551                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6552                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6553                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6554                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6555                                       &req.qpc_pg_size_qpc_lvl,
6556                                       &req.qpc_page_dir);
6557         }
6558         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6559                 ctx_pg = &ctx->srq_mem;
6560                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6561                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6562                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6563                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6564                                       &req.srq_pg_size_srq_lvl,
6565                                       &req.srq_page_dir);
6566         }
6567         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6568                 ctx_pg = &ctx->cq_mem;
6569                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6570                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6571                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6572                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6573                                       &req.cq_page_dir);
6574         }
6575         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6576                 ctx_pg = &ctx->vnic_mem;
6577                 req.vnic_num_vnic_entries =
6578                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6579                 req.vnic_num_ring_table_entries =
6580                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6581                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6582                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6583                                       &req.vnic_pg_size_vnic_lvl,
6584                                       &req.vnic_page_dir);
6585         }
6586         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6587                 ctx_pg = &ctx->stat_mem;
6588                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6589                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6590                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6591                                       &req.stat_pg_size_stat_lvl,
6592                                       &req.stat_page_dir);
6593         }
6594         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6595                 ctx_pg = &ctx->mrav_mem;
6596                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6597                 if (ctx->mrav_num_entries_units)
6598                         flags |=
6599                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6600                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6601                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6602                                       &req.mrav_pg_size_mrav_lvl,
6603                                       &req.mrav_page_dir);
6604         }
6605         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6606                 ctx_pg = &ctx->tim_mem;
6607                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6608                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6609                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6610                                       &req.tim_pg_size_tim_lvl,
6611                                       &req.tim_page_dir);
6612         }
6613         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6614              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6615              pg_dir = &req.tqm_sp_page_dir,
6616              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6617              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6618                 if (!(enables & ena))
6619                         continue;
6620
6621                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6622                 ctx_pg = ctx->tqm_mem[i];
6623                 *num_entries = cpu_to_le32(ctx_pg->entries);
6624                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6625         }
6626         req.flags = cpu_to_le32(flags);
6627         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6628         return rc;
6629 }
6630
6631 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6632                                   struct bnxt_ctx_pg_info *ctx_pg)
6633 {
6634         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6635
6636         rmem->page_size = BNXT_PAGE_SIZE;
6637         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6638         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6639         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6640         if (rmem->depth >= 1)
6641                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6642         return bnxt_alloc_ring(bp, rmem);
6643 }
6644
6645 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6646                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6647                                   u8 depth)
6648 {
6649         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6650         int rc;
6651
6652         if (!mem_size)
6653                 return 0;
6654
6655         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6656         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6657                 ctx_pg->nr_pages = 0;
6658                 return -EINVAL;
6659         }
6660         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6661                 int nr_tbls, i;
6662
6663                 rmem->depth = 2;
6664                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6665                                              GFP_KERNEL);
6666                 if (!ctx_pg->ctx_pg_tbl)
6667                         return -ENOMEM;
6668                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6669                 rmem->nr_pages = nr_tbls;
6670                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6671                 if (rc)
6672                         return rc;
6673                 for (i = 0; i < nr_tbls; i++) {
6674                         struct bnxt_ctx_pg_info *pg_tbl;
6675
6676                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6677                         if (!pg_tbl)
6678                                 return -ENOMEM;
6679                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6680                         rmem = &pg_tbl->ring_mem;
6681                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6682                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6683                         rmem->depth = 1;
6684                         rmem->nr_pages = MAX_CTX_PAGES;
6685                         if (i == (nr_tbls - 1)) {
6686                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6687
6688                                 if (rem)
6689                                         rmem->nr_pages = rem;
6690                         }
6691                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6692                         if (rc)
6693                                 break;
6694                 }
6695         } else {
6696                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6697                 if (rmem->nr_pages > 1 || depth)
6698                         rmem->depth = 1;
6699                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6700         }
6701         return rc;
6702 }
6703
6704 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6705                                   struct bnxt_ctx_pg_info *ctx_pg)
6706 {
6707         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6708
6709         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6710             ctx_pg->ctx_pg_tbl) {
6711                 int i, nr_tbls = rmem->nr_pages;
6712
6713                 for (i = 0; i < nr_tbls; i++) {
6714                         struct bnxt_ctx_pg_info *pg_tbl;
6715                         struct bnxt_ring_mem_info *rmem2;
6716
6717                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
6718                         if (!pg_tbl)
6719                                 continue;
6720                         rmem2 = &pg_tbl->ring_mem;
6721                         bnxt_free_ring(bp, rmem2);
6722                         ctx_pg->ctx_pg_arr[i] = NULL;
6723                         kfree(pg_tbl);
6724                         ctx_pg->ctx_pg_tbl[i] = NULL;
6725                 }
6726                 kfree(ctx_pg->ctx_pg_tbl);
6727                 ctx_pg->ctx_pg_tbl = NULL;
6728         }
6729         bnxt_free_ring(bp, rmem);
6730         ctx_pg->nr_pages = 0;
6731 }
6732
6733 static void bnxt_free_ctx_mem(struct bnxt *bp)
6734 {
6735         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6736         int i;
6737
6738         if (!ctx)
6739                 return;
6740
6741         if (ctx->tqm_mem[0]) {
6742                 for (i = 0; i < bp->max_q + 1; i++)
6743                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
6744                 kfree(ctx->tqm_mem[0]);
6745                 ctx->tqm_mem[0] = NULL;
6746         }
6747
6748         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6749         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
6750         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6751         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6752         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6753         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6754         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
6755         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6756 }
6757
6758 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6759 {
6760         struct bnxt_ctx_pg_info *ctx_pg;
6761         struct bnxt_ctx_mem_info *ctx;
6762         u32 mem_size, ena, entries;
6763         u32 num_mr, num_ah;
6764         u32 extra_srqs = 0;
6765         u32 extra_qps = 0;
6766         u8 pg_lvl = 1;
6767         int i, rc;
6768
6769         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6770         if (rc) {
6771                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6772                            rc);
6773                 return rc;
6774         }
6775         ctx = bp->ctx;
6776         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6777                 return 0;
6778
6779         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
6780                 pg_lvl = 2;
6781                 extra_qps = 65536;
6782                 extra_srqs = 8192;
6783         }
6784
6785         ctx_pg = &ctx->qp_mem;
6786         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6787                           extra_qps;
6788         mem_size = ctx->qp_entry_size * ctx_pg->entries;
6789         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6790         if (rc)
6791                 return rc;
6792
6793         ctx_pg = &ctx->srq_mem;
6794         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
6795         mem_size = ctx->srq_entry_size * ctx_pg->entries;
6796         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6797         if (rc)
6798                 return rc;
6799
6800         ctx_pg = &ctx->cq_mem;
6801         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
6802         mem_size = ctx->cq_entry_size * ctx_pg->entries;
6803         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl);
6804         if (rc)
6805                 return rc;
6806
6807         ctx_pg = &ctx->vnic_mem;
6808         ctx_pg->entries = ctx->vnic_max_vnic_entries +
6809                           ctx->vnic_max_ring_table_entries;
6810         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
6811         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6812         if (rc)
6813                 return rc;
6814
6815         ctx_pg = &ctx->stat_mem;
6816         ctx_pg->entries = ctx->stat_max_entries;
6817         mem_size = ctx->stat_entry_size * ctx_pg->entries;
6818         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6819         if (rc)
6820                 return rc;
6821
6822         ena = 0;
6823         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6824                 goto skip_rdma;
6825
6826         ctx_pg = &ctx->mrav_mem;
6827         /* 128K extra is needed to accommodate static AH context
6828          * allocation by f/w.
6829          */
6830         num_mr = 1024 * 256;
6831         num_ah = 1024 * 128;
6832         ctx_pg->entries = num_mr + num_ah;
6833         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6834         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2);
6835         if (rc)
6836                 return rc;
6837         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
6838         if (ctx->mrav_num_entries_units)
6839                 ctx_pg->entries =
6840                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
6841                          (num_ah / ctx->mrav_num_entries_units);
6842
6843         ctx_pg = &ctx->tim_mem;
6844         ctx_pg->entries = ctx->qp_mem.entries;
6845         mem_size = ctx->tim_entry_size * ctx_pg->entries;
6846         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6847         if (rc)
6848                 return rc;
6849         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6850
6851 skip_rdma:
6852         entries = ctx->qp_max_l2_entries + extra_qps;
6853         entries = roundup(entries, ctx->tqm_entries_multiple);
6854         entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6855                           ctx->tqm_max_entries_per_ring);
6856         for (i = 0; i < bp->max_q + 1; i++) {
6857                 ctx_pg = ctx->tqm_mem[i];
6858                 ctx_pg->entries = entries;
6859                 mem_size = ctx->tqm_entry_size * entries;
6860                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1);
6861                 if (rc)
6862                         return rc;
6863                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
6864         }
6865         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6866         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6867         if (rc)
6868                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6869                            rc);
6870         else
6871                 ctx->flags |= BNXT_CTX_FLAG_INITED;
6872
6873         return 0;
6874 }
6875
6876 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
6877 {
6878         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6879         struct hwrm_func_resource_qcaps_input req = {0};
6880         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6881         int rc;
6882
6883         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6884         req.fid = cpu_to_le16(0xffff);
6885
6886         mutex_lock(&bp->hwrm_cmd_lock);
6887         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6888                                        HWRM_CMD_TIMEOUT);
6889         if (rc)
6890                 goto hwrm_func_resc_qcaps_exit;
6891
6892         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6893         if (!all)
6894                 goto hwrm_func_resc_qcaps_exit;
6895
6896         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6897         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6898         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6899         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6900         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6901         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6902         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6903         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6904         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6905         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6906         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6907         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6908         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6909         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6910         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6911         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6912
6913         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6914                 u16 max_msix = le16_to_cpu(resp->max_msix);
6915
6916                 hw_resc->max_nqs = max_msix;
6917                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6918         }
6919
6920         if (BNXT_PF(bp)) {
6921                 struct bnxt_pf_info *pf = &bp->pf;
6922
6923                 pf->vf_resv_strategy =
6924                         le16_to_cpu(resp->vf_reservation_strategy);
6925                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
6926                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6927         }
6928 hwrm_func_resc_qcaps_exit:
6929         mutex_unlock(&bp->hwrm_cmd_lock);
6930         return rc;
6931 }
6932
6933 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
6934 {
6935         int rc = 0;
6936         struct hwrm_func_qcaps_input req = {0};
6937         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6938         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6939         u32 flags;
6940
6941         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6942         req.fid = cpu_to_le16(0xffff);
6943
6944         mutex_lock(&bp->hwrm_cmd_lock);
6945         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6946         if (rc)
6947                 goto hwrm_func_qcaps_exit;
6948
6949         flags = le32_to_cpu(resp->flags);
6950         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
6951                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6952         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
6953                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
6954         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
6955                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
6956         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
6957                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
6958         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
6959                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
6960         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
6961                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
6962         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
6963                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
6964
6965         bp->tx_push_thresh = 0;
6966         if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
6967                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6968
6969         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6970         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6971         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6972         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6973         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6974         if (!hw_resc->max_hw_ring_grps)
6975                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6976         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6977         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6978         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6979
6980         if (BNXT_PF(bp)) {
6981                 struct bnxt_pf_info *pf = &bp->pf;
6982
6983                 pf->fw_fid = le16_to_cpu(resp->fid);
6984                 pf->port_id = le16_to_cpu(resp->port_id);
6985                 bp->dev->dev_port = pf->port_id;
6986                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
6987                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6988                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6989                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6990                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6991                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6992                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6993                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6994                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
6995                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
6996                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
6997                         bp->flags |= BNXT_FLAG_WOL_CAP;
6998         } else {
6999 #ifdef CONFIG_BNXT_SRIOV
7000                 struct bnxt_vf_info *vf = &bp->vf;
7001
7002                 vf->fw_fid = le16_to_cpu(resp->fid);
7003                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7004 #endif
7005         }
7006
7007 hwrm_func_qcaps_exit:
7008         mutex_unlock(&bp->hwrm_cmd_lock);
7009         return rc;
7010 }
7011
7012 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7013
7014 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7015 {
7016         int rc;
7017
7018         rc = __bnxt_hwrm_func_qcaps(bp);
7019         if (rc)
7020                 return rc;
7021         rc = bnxt_hwrm_queue_qportcfg(bp);
7022         if (rc) {
7023                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7024                 return rc;
7025         }
7026         if (bp->hwrm_spec_code >= 0x10803) {
7027                 rc = bnxt_alloc_ctx_mem(bp);
7028                 if (rc)
7029                         return rc;
7030                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7031                 if (!rc)
7032                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7033         }
7034         return 0;
7035 }
7036
7037 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7038 {
7039         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7040         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7041         int rc = 0;
7042         u32 flags;
7043
7044         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7045                 return 0;
7046
7047         resp = bp->hwrm_cmd_resp_addr;
7048         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7049
7050         mutex_lock(&bp->hwrm_cmd_lock);
7051         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7052         if (rc)
7053                 goto hwrm_cfa_adv_qcaps_exit;
7054
7055         flags = le32_to_cpu(resp->flags);
7056         if (flags &
7057             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7058                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7059
7060 hwrm_cfa_adv_qcaps_exit:
7061         mutex_unlock(&bp->hwrm_cmd_lock);
7062         return rc;
7063 }
7064
7065 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7066 {
7067         struct bnxt_fw_health *fw_health = bp->fw_health;
7068         u32 reg_base = 0xffffffff;
7069         int i;
7070
7071         /* Only pre-map the monitoring GRC registers using window 3 */
7072         for (i = 0; i < 4; i++) {
7073                 u32 reg = fw_health->regs[i];
7074
7075                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7076                         continue;
7077                 if (reg_base == 0xffffffff)
7078                         reg_base = reg & BNXT_GRC_BASE_MASK;
7079                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7080                         return -ERANGE;
7081                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
7082                                             (reg & BNXT_GRC_OFFSET_MASK);
7083         }
7084         if (reg_base == 0xffffffff)
7085                 return 0;
7086
7087         writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7088                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7089         return 0;
7090 }
7091
7092 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7093 {
7094         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7095         struct bnxt_fw_health *fw_health = bp->fw_health;
7096         struct hwrm_error_recovery_qcfg_input req = {0};
7097         int rc, i;
7098
7099         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7100                 return 0;
7101
7102         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7103         mutex_lock(&bp->hwrm_cmd_lock);
7104         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7105         if (rc)
7106                 goto err_recovery_out;
7107         if (!fw_health) {
7108                 fw_health = kzalloc(sizeof(*fw_health), GFP_KERNEL);
7109                 bp->fw_health = fw_health;
7110                 if (!fw_health) {
7111                         rc = -ENOMEM;
7112                         goto err_recovery_out;
7113                 }
7114         }
7115         fw_health->flags = le32_to_cpu(resp->flags);
7116         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7117             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7118                 rc = -EINVAL;
7119                 goto err_recovery_out;
7120         }
7121         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7122         fw_health->master_func_wait_dsecs =
7123                 le32_to_cpu(resp->master_func_wait_period);
7124         fw_health->normal_func_wait_dsecs =
7125                 le32_to_cpu(resp->normal_func_wait_period);
7126         fw_health->post_reset_wait_dsecs =
7127                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7128         fw_health->post_reset_max_wait_dsecs =
7129                 le32_to_cpu(resp->max_bailout_time_after_reset);
7130         fw_health->regs[BNXT_FW_HEALTH_REG] =
7131                 le32_to_cpu(resp->fw_health_status_reg);
7132         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7133                 le32_to_cpu(resp->fw_heartbeat_reg);
7134         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7135                 le32_to_cpu(resp->fw_reset_cnt_reg);
7136         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7137                 le32_to_cpu(resp->reset_inprogress_reg);
7138         fw_health->fw_reset_inprog_reg_mask =
7139                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7140         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7141         if (fw_health->fw_reset_seq_cnt >= 16) {
7142                 rc = -EINVAL;
7143                 goto err_recovery_out;
7144         }
7145         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7146                 fw_health->fw_reset_seq_regs[i] =
7147                         le32_to_cpu(resp->reset_reg[i]);
7148                 fw_health->fw_reset_seq_vals[i] =
7149                         le32_to_cpu(resp->reset_reg_val[i]);
7150                 fw_health->fw_reset_seq_delay_msec[i] =
7151                         resp->delay_after_reset[i];
7152         }
7153 err_recovery_out:
7154         mutex_unlock(&bp->hwrm_cmd_lock);
7155         if (!rc)
7156                 rc = bnxt_map_fw_health_regs(bp);
7157         if (rc)
7158                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7159         return rc;
7160 }
7161
7162 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7163 {
7164         struct hwrm_func_reset_input req = {0};
7165
7166         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7167         req.enables = 0;
7168
7169         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7170 }
7171
7172 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7173 {
7174         int rc = 0;
7175         struct hwrm_queue_qportcfg_input req = {0};
7176         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7177         u8 i, j, *qptr;
7178         bool no_rdma;
7179
7180         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7181
7182         mutex_lock(&bp->hwrm_cmd_lock);
7183         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7184         if (rc)
7185                 goto qportcfg_exit;
7186
7187         if (!resp->max_configurable_queues) {
7188                 rc = -EINVAL;
7189                 goto qportcfg_exit;
7190         }
7191         bp->max_tc = resp->max_configurable_queues;
7192         bp->max_lltc = resp->max_configurable_lossless_queues;
7193         if (bp->max_tc > BNXT_MAX_QUEUE)
7194                 bp->max_tc = BNXT_MAX_QUEUE;
7195
7196         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7197         qptr = &resp->queue_id0;
7198         for (i = 0, j = 0; i < bp->max_tc; i++) {
7199                 bp->q_info[j].queue_id = *qptr;
7200                 bp->q_ids[i] = *qptr++;
7201                 bp->q_info[j].queue_profile = *qptr++;
7202                 bp->tc_to_qidx[j] = j;
7203                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7204                     (no_rdma && BNXT_PF(bp)))
7205                         j++;
7206         }
7207         bp->max_q = bp->max_tc;
7208         bp->max_tc = max_t(u8, j, 1);
7209
7210         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7211                 bp->max_tc = 1;
7212
7213         if (bp->max_lltc > bp->max_tc)
7214                 bp->max_lltc = bp->max_tc;
7215
7216 qportcfg_exit:
7217         mutex_unlock(&bp->hwrm_cmd_lock);
7218         return rc;
7219 }
7220
7221 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7222 {
7223         struct hwrm_ver_get_input req = {0};
7224         int rc;
7225
7226         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7227         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7228         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7229         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7230
7231         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7232                                    silent);
7233         return rc;
7234 }
7235
7236 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7237 {
7238         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7239         u32 dev_caps_cfg;
7240         int rc;
7241
7242         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7243         mutex_lock(&bp->hwrm_cmd_lock);
7244         rc = __bnxt_hwrm_ver_get(bp, false);
7245         if (rc)
7246                 goto hwrm_ver_get_exit;
7247
7248         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7249
7250         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7251                              resp->hwrm_intf_min_8b << 8 |
7252                              resp->hwrm_intf_upd_8b;
7253         if (resp->hwrm_intf_maj_8b < 1) {
7254                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7255                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7256                             resp->hwrm_intf_upd_8b);
7257                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7258         }
7259         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
7260                  resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
7261                  resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
7262
7263         if (strlen(resp->active_pkg_name)) {
7264                 int fw_ver_len = strlen(bp->fw_ver_str);
7265
7266                 snprintf(bp->fw_ver_str + fw_ver_len,
7267                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7268                          resp->active_pkg_name);
7269                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7270         }
7271
7272         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7273         if (!bp->hwrm_cmd_timeout)
7274                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7275
7276         if (resp->hwrm_intf_maj_8b >= 1) {
7277                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7278                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7279         }
7280         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7281                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7282
7283         bp->chip_num = le16_to_cpu(resp->chip_num);
7284         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7285             !resp->chip_metal)
7286                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7287
7288         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7289         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7290             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7291                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7292
7293         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7294                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7295
7296         if (dev_caps_cfg &
7297             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7298                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7299
7300         if (dev_caps_cfg &
7301             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7302                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7303
7304         if (dev_caps_cfg &
7305             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7306                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7307
7308 hwrm_ver_get_exit:
7309         mutex_unlock(&bp->hwrm_cmd_lock);
7310         return rc;
7311 }
7312
7313 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7314 {
7315         struct hwrm_fw_set_time_input req = {0};
7316         struct tm tm;
7317         time64_t now = ktime_get_real_seconds();
7318
7319         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7320             bp->hwrm_spec_code < 0x10400)
7321                 return -EOPNOTSUPP;
7322
7323         time64_to_tm(now, 0, &tm);
7324         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7325         req.year = cpu_to_le16(1900 + tm.tm_year);
7326         req.month = 1 + tm.tm_mon;
7327         req.day = tm.tm_mday;
7328         req.hour = tm.tm_hour;
7329         req.minute = tm.tm_min;
7330         req.second = tm.tm_sec;
7331         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7332 }
7333
7334 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
7335 {
7336         int rc;
7337         struct bnxt_pf_info *pf = &bp->pf;
7338         struct hwrm_port_qstats_input req = {0};
7339
7340         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7341                 return 0;
7342
7343         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7344         req.port_id = cpu_to_le16(pf->port_id);
7345         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
7346         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
7347         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7348         return rc;
7349 }
7350
7351 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
7352 {
7353         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7354         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7355         struct hwrm_port_qstats_ext_input req = {0};
7356         struct bnxt_pf_info *pf = &bp->pf;
7357         u32 tx_stat_size;
7358         int rc;
7359
7360         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7361                 return 0;
7362
7363         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7364         req.port_id = cpu_to_le16(pf->port_id);
7365         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7366         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
7367         tx_stat_size = bp->hw_tx_port_stats_ext ?
7368                        sizeof(*bp->hw_tx_port_stats_ext) : 0;
7369         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7370         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
7371         mutex_lock(&bp->hwrm_cmd_lock);
7372         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7373         if (!rc) {
7374                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7375                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7376                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7377         } else {
7378                 bp->fw_rx_stats_ext_size = 0;
7379                 bp->fw_tx_stats_ext_size = 0;
7380         }
7381         if (bp->fw_tx_stats_ext_size <=
7382             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7383                 mutex_unlock(&bp->hwrm_cmd_lock);
7384                 bp->pri2cos_valid = 0;
7385                 return rc;
7386         }
7387
7388         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7389         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7390
7391         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7392         if (!rc) {
7393                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7394                 u8 *pri2cos;
7395                 int i, j;
7396
7397                 resp2 = bp->hwrm_cmd_resp_addr;
7398                 pri2cos = &resp2->pri0_cos_queue_id;
7399                 for (i = 0; i < 8; i++) {
7400                         u8 queue_id = pri2cos[i];
7401
7402                         for (j = 0; j < bp->max_q; j++) {
7403                                 if (bp->q_ids[j] == queue_id)
7404                                         bp->pri2cos[i] = j;
7405                         }
7406                 }
7407                 bp->pri2cos_valid = 1;
7408         }
7409         mutex_unlock(&bp->hwrm_cmd_lock);
7410         return rc;
7411 }
7412
7413 static int bnxt_hwrm_pcie_qstats(struct bnxt *bp)
7414 {
7415         struct hwrm_pcie_qstats_input req = {0};
7416
7417         if (!(bp->flags & BNXT_FLAG_PCIE_STATS))
7418                 return 0;
7419
7420         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1);
7421         req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats));
7422         req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map);
7423         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7424 }
7425
7426 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7427 {
7428         if (bp->vxlan_port_cnt) {
7429                 bnxt_hwrm_tunnel_dst_port_free(
7430                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7431         }
7432         bp->vxlan_port_cnt = 0;
7433         if (bp->nge_port_cnt) {
7434                 bnxt_hwrm_tunnel_dst_port_free(
7435                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7436         }
7437         bp->nge_port_cnt = 0;
7438 }
7439
7440 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7441 {
7442         int rc, i;
7443         u32 tpa_flags = 0;
7444
7445         if (set_tpa)
7446                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7447         else if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
7448                 return 0;
7449         for (i = 0; i < bp->nr_vnics; i++) {
7450                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7451                 if (rc) {
7452                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7453                                    i, rc);
7454                         return rc;
7455                 }
7456         }
7457         return 0;
7458 }
7459
7460 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7461 {
7462         int i;
7463
7464         for (i = 0; i < bp->nr_vnics; i++)
7465                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7466 }
7467
7468 static void bnxt_clear_vnic(struct bnxt *bp)
7469 {
7470         if (!bp->vnic_info)
7471                 return;
7472
7473         bnxt_hwrm_clear_vnic_filter(bp);
7474         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7475                 /* clear all RSS setting before free vnic ctx */
7476                 bnxt_hwrm_clear_vnic_rss(bp);
7477                 bnxt_hwrm_vnic_ctx_free(bp);
7478         }
7479         /* before free the vnic, undo the vnic tpa settings */
7480         if (bp->flags & BNXT_FLAG_TPA)
7481                 bnxt_set_tpa(bp, false);
7482         bnxt_hwrm_vnic_free(bp);
7483         if (bp->flags & BNXT_FLAG_CHIP_P5)
7484                 bnxt_hwrm_vnic_ctx_free(bp);
7485 }
7486
7487 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7488                                     bool irq_re_init)
7489 {
7490         bnxt_clear_vnic(bp);
7491         bnxt_hwrm_ring_free(bp, close_path);
7492         bnxt_hwrm_ring_grp_free(bp);
7493         if (irq_re_init) {
7494                 bnxt_hwrm_stat_ctx_free(bp);
7495                 bnxt_hwrm_free_tunnel_ports(bp);
7496         }
7497 }
7498
7499 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7500 {
7501         struct hwrm_func_cfg_input req = {0};
7502         int rc;
7503
7504         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7505         req.fid = cpu_to_le16(0xffff);
7506         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7507         if (br_mode == BRIDGE_MODE_VEB)
7508                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7509         else if (br_mode == BRIDGE_MODE_VEPA)
7510                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7511         else
7512                 return -EINVAL;
7513         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7514         return rc;
7515 }
7516
7517 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7518 {
7519         struct hwrm_func_cfg_input req = {0};
7520         int rc;
7521
7522         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7523                 return 0;
7524
7525         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7526         req.fid = cpu_to_le16(0xffff);
7527         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7528         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7529         if (size == 128)
7530                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7531
7532         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7533         return rc;
7534 }
7535
7536 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7537 {
7538         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7539         int rc;
7540
7541         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7542                 goto skip_rss_ctx;
7543
7544         /* allocate context for vnic */
7545         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7546         if (rc) {
7547                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7548                            vnic_id, rc);
7549                 goto vnic_setup_err;
7550         }
7551         bp->rsscos_nr_ctxs++;
7552
7553         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7554                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7555                 if (rc) {
7556                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7557                                    vnic_id, rc);
7558                         goto vnic_setup_err;
7559                 }
7560                 bp->rsscos_nr_ctxs++;
7561         }
7562
7563 skip_rss_ctx:
7564         /* configure default vnic, ring grp */
7565         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7566         if (rc) {
7567                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7568                            vnic_id, rc);
7569                 goto vnic_setup_err;
7570         }
7571
7572         /* Enable RSS hashing on vnic */
7573         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7574         if (rc) {
7575                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7576                            vnic_id, rc);
7577                 goto vnic_setup_err;
7578         }
7579
7580         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7581                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7582                 if (rc) {
7583                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7584                                    vnic_id, rc);
7585                 }
7586         }
7587
7588 vnic_setup_err:
7589         return rc;
7590 }
7591
7592 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7593 {
7594         int rc, i, nr_ctxs;
7595
7596         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
7597         for (i = 0; i < nr_ctxs; i++) {
7598                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7599                 if (rc) {
7600                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7601                                    vnic_id, i, rc);
7602                         break;
7603                 }
7604                 bp->rsscos_nr_ctxs++;
7605         }
7606         if (i < nr_ctxs)
7607                 return -ENOMEM;
7608
7609         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7610         if (rc) {
7611                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7612                            vnic_id, rc);
7613                 return rc;
7614         }
7615         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7616         if (rc) {
7617                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7618                            vnic_id, rc);
7619                 return rc;
7620         }
7621         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7622                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7623                 if (rc) {
7624                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7625                                    vnic_id, rc);
7626                 }
7627         }
7628         return rc;
7629 }
7630
7631 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7632 {
7633         if (bp->flags & BNXT_FLAG_CHIP_P5)
7634                 return __bnxt_setup_vnic_p5(bp, vnic_id);
7635         else
7636                 return __bnxt_setup_vnic(bp, vnic_id);
7637 }
7638
7639 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
7640 {
7641 #ifdef CONFIG_RFS_ACCEL
7642         int i, rc = 0;
7643
7644         if (bp->flags & BNXT_FLAG_CHIP_P5)
7645                 return 0;
7646
7647         for (i = 0; i < bp->rx_nr_rings; i++) {
7648                 struct bnxt_vnic_info *vnic;
7649                 u16 vnic_id = i + 1;
7650                 u16 ring_id = i;
7651
7652                 if (vnic_id >= bp->nr_vnics)
7653                         break;
7654
7655                 vnic = &bp->vnic_info[vnic_id];
7656                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7657                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7658                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
7659                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
7660                 if (rc) {
7661                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7662                                    vnic_id, rc);
7663                         break;
7664                 }
7665                 rc = bnxt_setup_vnic(bp, vnic_id);
7666                 if (rc)
7667                         break;
7668         }
7669         return rc;
7670 #else
7671         return 0;
7672 #endif
7673 }
7674
7675 /* Allow PF and VF with default VLAN to be in promiscuous mode */
7676 static bool bnxt_promisc_ok(struct bnxt *bp)
7677 {
7678 #ifdef CONFIG_BNXT_SRIOV
7679         if (BNXT_VF(bp) && !bp->vf.vlan)
7680                 return false;
7681 #endif
7682         return true;
7683 }
7684
7685 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7686 {
7687         unsigned int rc = 0;
7688
7689         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7690         if (rc) {
7691                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7692                            rc);
7693                 return rc;
7694         }
7695
7696         rc = bnxt_hwrm_vnic_cfg(bp, 1);
7697         if (rc) {
7698                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7699                            rc);
7700                 return rc;
7701         }
7702         return rc;
7703 }
7704
7705 static int bnxt_cfg_rx_mode(struct bnxt *);
7706 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
7707
7708 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7709 {
7710         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7711         int rc = 0;
7712         unsigned int rx_nr_rings = bp->rx_nr_rings;
7713
7714         if (irq_re_init) {
7715                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7716                 if (rc) {
7717                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7718                                    rc);
7719                         goto err_out;
7720                 }
7721         }
7722
7723         rc = bnxt_hwrm_ring_alloc(bp);
7724         if (rc) {
7725                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7726                 goto err_out;
7727         }
7728
7729         rc = bnxt_hwrm_ring_grp_alloc(bp);
7730         if (rc) {
7731                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7732                 goto err_out;
7733         }
7734
7735         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7736                 rx_nr_rings--;
7737
7738         /* default vnic 0 */
7739         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
7740         if (rc) {
7741                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7742                 goto err_out;
7743         }
7744
7745         rc = bnxt_setup_vnic(bp, 0);
7746         if (rc)
7747                 goto err_out;
7748
7749         if (bp->flags & BNXT_FLAG_RFS) {
7750                 rc = bnxt_alloc_rfs_vnics(bp);
7751                 if (rc)
7752                         goto err_out;
7753         }
7754
7755         if (bp->flags & BNXT_FLAG_TPA) {
7756                 rc = bnxt_set_tpa(bp, true);
7757                 if (rc)
7758                         goto err_out;
7759         }
7760
7761         if (BNXT_VF(bp))
7762                 bnxt_update_vf_mac(bp);
7763
7764         /* Filter for default vnic 0 */
7765         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7766         if (rc) {
7767                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7768                 goto err_out;
7769         }
7770         vnic->uc_filter_count = 1;
7771
7772         vnic->rx_mask = 0;
7773         if (bp->dev->flags & IFF_BROADCAST)
7774                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
7775
7776         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7777                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7778
7779         if (bp->dev->flags & IFF_ALLMULTI) {
7780                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7781                 vnic->mc_list_count = 0;
7782         } else {
7783                 u32 mask = 0;
7784
7785                 bnxt_mc_list_updated(bp, &mask);
7786                 vnic->rx_mask |= mask;
7787         }
7788
7789         rc = bnxt_cfg_rx_mode(bp);
7790         if (rc)
7791                 goto err_out;
7792
7793         rc = bnxt_hwrm_set_coal(bp);
7794         if (rc)
7795                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
7796                                 rc);
7797
7798         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7799                 rc = bnxt_setup_nitroa0_vnic(bp);
7800                 if (rc)
7801                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7802                                    rc);
7803         }
7804
7805         if (BNXT_VF(bp)) {
7806                 bnxt_hwrm_func_qcfg(bp);
7807                 netdev_update_features(bp->dev);
7808         }
7809
7810         return 0;
7811
7812 err_out:
7813         bnxt_hwrm_resource_free(bp, 0, true);
7814
7815         return rc;
7816 }
7817
7818 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7819 {
7820         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7821         return 0;
7822 }
7823
7824 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7825 {
7826         bnxt_init_cp_rings(bp);
7827         bnxt_init_rx_rings(bp);
7828         bnxt_init_tx_rings(bp);
7829         bnxt_init_ring_grps(bp, irq_re_init);
7830         bnxt_init_vnics(bp);
7831
7832         return bnxt_init_chip(bp, irq_re_init);
7833 }
7834
7835 static int bnxt_set_real_num_queues(struct bnxt *bp)
7836 {
7837         int rc;
7838         struct net_device *dev = bp->dev;
7839
7840         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7841                                           bp->tx_nr_rings_xdp);
7842         if (rc)
7843                 return rc;
7844
7845         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7846         if (rc)
7847                 return rc;
7848
7849 #ifdef CONFIG_RFS_ACCEL
7850         if (bp->flags & BNXT_FLAG_RFS)
7851                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
7852 #endif
7853
7854         return rc;
7855 }
7856
7857 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7858                            bool shared)
7859 {
7860         int _rx = *rx, _tx = *tx;
7861
7862         if (shared) {
7863                 *rx = min_t(int, _rx, max);
7864                 *tx = min_t(int, _tx, max);
7865         } else {
7866                 if (max < 2)
7867                         return -ENOMEM;
7868
7869                 while (_rx + _tx > max) {
7870                         if (_rx > _tx && _rx > 1)
7871                                 _rx--;
7872                         else if (_tx > 1)
7873                                 _tx--;
7874                 }
7875                 *rx = _rx;
7876                 *tx = _tx;
7877         }
7878         return 0;
7879 }
7880
7881 static void bnxt_setup_msix(struct bnxt *bp)
7882 {
7883         const int len = sizeof(bp->irq_tbl[0].name);
7884         struct net_device *dev = bp->dev;
7885         int tcs, i;
7886
7887         tcs = netdev_get_num_tc(dev);
7888         if (tcs > 1) {
7889                 int i, off, count;
7890
7891                 for (i = 0; i < tcs; i++) {
7892                         count = bp->tx_nr_rings_per_tc;
7893                         off = i * count;
7894                         netdev_set_tc_queue(dev, i, count, off);
7895                 }
7896         }
7897
7898         for (i = 0; i < bp->cp_nr_rings; i++) {
7899                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7900                 char *attr;
7901
7902                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7903                         attr = "TxRx";
7904                 else if (i < bp->rx_nr_rings)
7905                         attr = "rx";
7906                 else
7907                         attr = "tx";
7908
7909                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7910                          attr, i);
7911                 bp->irq_tbl[map_idx].handler = bnxt_msix;
7912         }
7913 }
7914
7915 static void bnxt_setup_inta(struct bnxt *bp)
7916 {
7917         const int len = sizeof(bp->irq_tbl[0].name);
7918
7919         if (netdev_get_num_tc(bp->dev))
7920                 netdev_reset_tc(bp->dev);
7921
7922         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7923                  0);
7924         bp->irq_tbl[0].handler = bnxt_inta;
7925 }
7926
7927 static int bnxt_setup_int_mode(struct bnxt *bp)
7928 {
7929         int rc;
7930
7931         if (bp->flags & BNXT_FLAG_USING_MSIX)
7932                 bnxt_setup_msix(bp);
7933         else
7934                 bnxt_setup_inta(bp);
7935
7936         rc = bnxt_set_real_num_queues(bp);
7937         return rc;
7938 }
7939
7940 #ifdef CONFIG_RFS_ACCEL
7941 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7942 {
7943         return bp->hw_resc.max_rsscos_ctxs;
7944 }
7945
7946 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7947 {
7948         return bp->hw_resc.max_vnics;
7949 }
7950 #endif
7951
7952 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7953 {
7954         return bp->hw_resc.max_stat_ctxs;
7955 }
7956
7957 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7958 {
7959         return bp->hw_resc.max_cp_rings;
7960 }
7961
7962 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
7963 {
7964         unsigned int cp = bp->hw_resc.max_cp_rings;
7965
7966         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7967                 cp -= bnxt_get_ulp_msix_num(bp);
7968
7969         return cp;
7970 }
7971
7972 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7973 {
7974         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7975
7976         if (bp->flags & BNXT_FLAG_CHIP_P5)
7977                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7978
7979         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7980 }
7981
7982 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
7983 {
7984         bp->hw_resc.max_irqs = max_irqs;
7985 }
7986
7987 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7988 {
7989         unsigned int cp;
7990
7991         cp = bnxt_get_max_func_cp_rings_for_en(bp);
7992         if (bp->flags & BNXT_FLAG_CHIP_P5)
7993                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
7994         else
7995                 return cp - bp->cp_nr_rings;
7996 }
7997
7998 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
7999 {
8000         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8001 }
8002
8003 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8004 {
8005         int max_cp = bnxt_get_max_func_cp_rings(bp);
8006         int max_irq = bnxt_get_max_func_irqs(bp);
8007         int total_req = bp->cp_nr_rings + num;
8008         int max_idx, avail_msix;
8009
8010         max_idx = bp->total_irqs;
8011         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8012                 max_idx = min_t(int, bp->total_irqs, max_cp);
8013         avail_msix = max_idx - bp->cp_nr_rings;
8014         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8015                 return avail_msix;
8016
8017         if (max_irq < total_req) {
8018                 num = max_irq - bp->cp_nr_rings;
8019                 if (num <= 0)
8020                         return 0;
8021         }
8022         return num;
8023 }
8024
8025 static int bnxt_get_num_msix(struct bnxt *bp)
8026 {
8027         if (!BNXT_NEW_RM(bp))
8028                 return bnxt_get_max_func_irqs(bp);
8029
8030         return bnxt_nq_rings_in_use(bp);
8031 }
8032
8033 static int bnxt_init_msix(struct bnxt *bp)
8034 {
8035         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8036         struct msix_entry *msix_ent;
8037
8038         total_vecs = bnxt_get_num_msix(bp);
8039         max = bnxt_get_max_func_irqs(bp);
8040         if (total_vecs > max)
8041                 total_vecs = max;
8042
8043         if (!total_vecs)
8044                 return 0;
8045
8046         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8047         if (!msix_ent)
8048                 return -ENOMEM;
8049
8050         for (i = 0; i < total_vecs; i++) {
8051                 msix_ent[i].entry = i;
8052                 msix_ent[i].vector = 0;
8053         }
8054
8055         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8056                 min = 2;
8057
8058         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8059         ulp_msix = bnxt_get_ulp_msix_num(bp);
8060         if (total_vecs < 0 || total_vecs < ulp_msix) {
8061                 rc = -ENODEV;
8062                 goto msix_setup_exit;
8063         }
8064
8065         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8066         if (bp->irq_tbl) {
8067                 for (i = 0; i < total_vecs; i++)
8068                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8069
8070                 bp->total_irqs = total_vecs;
8071                 /* Trim rings based upon num of vectors allocated */
8072                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8073                                      total_vecs - ulp_msix, min == 1);
8074                 if (rc)
8075                         goto msix_setup_exit;
8076
8077                 bp->cp_nr_rings = (min == 1) ?
8078                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8079                                   bp->tx_nr_rings + bp->rx_nr_rings;
8080
8081         } else {
8082                 rc = -ENOMEM;
8083                 goto msix_setup_exit;
8084         }
8085         bp->flags |= BNXT_FLAG_USING_MSIX;
8086         kfree(msix_ent);
8087         return 0;
8088
8089 msix_setup_exit:
8090         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8091         kfree(bp->irq_tbl);
8092         bp->irq_tbl = NULL;
8093         pci_disable_msix(bp->pdev);
8094         kfree(msix_ent);
8095         return rc;
8096 }
8097
8098 static int bnxt_init_inta(struct bnxt *bp)
8099 {
8100         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8101         if (!bp->irq_tbl)
8102                 return -ENOMEM;
8103
8104         bp->total_irqs = 1;
8105         bp->rx_nr_rings = 1;
8106         bp->tx_nr_rings = 1;
8107         bp->cp_nr_rings = 1;
8108         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8109         bp->irq_tbl[0].vector = bp->pdev->irq;
8110         return 0;
8111 }
8112
8113 static int bnxt_init_int_mode(struct bnxt *bp)
8114 {
8115         int rc = 0;
8116
8117         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8118                 rc = bnxt_init_msix(bp);
8119
8120         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8121                 /* fallback to INTA */
8122                 rc = bnxt_init_inta(bp);
8123         }
8124         return rc;
8125 }
8126
8127 static void bnxt_clear_int_mode(struct bnxt *bp)
8128 {
8129         if (bp->flags & BNXT_FLAG_USING_MSIX)
8130                 pci_disable_msix(bp->pdev);
8131
8132         kfree(bp->irq_tbl);
8133         bp->irq_tbl = NULL;
8134         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8135 }
8136
8137 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8138 {
8139         int tcs = netdev_get_num_tc(bp->dev);
8140         bool irq_cleared = false;
8141         int rc;
8142
8143         if (!bnxt_need_reserve_rings(bp))
8144                 return 0;
8145
8146         if (irq_re_init && BNXT_NEW_RM(bp) &&
8147             bnxt_get_num_msix(bp) != bp->total_irqs) {
8148                 bnxt_ulp_irq_stop(bp);
8149                 bnxt_clear_int_mode(bp);
8150                 irq_cleared = true;
8151         }
8152         rc = __bnxt_reserve_rings(bp);
8153         if (irq_cleared) {
8154                 if (!rc)
8155                         rc = bnxt_init_int_mode(bp);
8156                 bnxt_ulp_irq_restart(bp, rc);
8157         }
8158         if (rc) {
8159                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8160                 return rc;
8161         }
8162         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8163                 netdev_err(bp->dev, "tx ring reservation failure\n");
8164                 netdev_reset_tc(bp->dev);
8165                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8166                 return -ENOMEM;
8167         }
8168         return 0;
8169 }
8170
8171 static void bnxt_free_irq(struct bnxt *bp)
8172 {
8173         struct bnxt_irq *irq;
8174         int i;
8175
8176 #ifdef CONFIG_RFS_ACCEL
8177         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8178         bp->dev->rx_cpu_rmap = NULL;
8179 #endif
8180         if (!bp->irq_tbl || !bp->bnapi)
8181                 return;
8182
8183         for (i = 0; i < bp->cp_nr_rings; i++) {
8184                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8185
8186                 irq = &bp->irq_tbl[map_idx];
8187                 if (irq->requested) {
8188                         if (irq->have_cpumask) {
8189                                 irq_set_affinity_hint(irq->vector, NULL);
8190                                 free_cpumask_var(irq->cpu_mask);
8191                                 irq->have_cpumask = 0;
8192                         }
8193                         free_irq(irq->vector, bp->bnapi[i]);
8194                 }
8195
8196                 irq->requested = 0;
8197         }
8198 }
8199
8200 static int bnxt_request_irq(struct bnxt *bp)
8201 {
8202         int i, j, rc = 0;
8203         unsigned long flags = 0;
8204 #ifdef CONFIG_RFS_ACCEL
8205         struct cpu_rmap *rmap;
8206 #endif
8207
8208         rc = bnxt_setup_int_mode(bp);
8209         if (rc) {
8210                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8211                            rc);
8212                 return rc;
8213         }
8214 #ifdef CONFIG_RFS_ACCEL
8215         rmap = bp->dev->rx_cpu_rmap;
8216 #endif
8217         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8218                 flags = IRQF_SHARED;
8219
8220         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8221                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8222                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8223
8224 #ifdef CONFIG_RFS_ACCEL
8225                 if (rmap && bp->bnapi[i]->rx_ring) {
8226                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8227                         if (rc)
8228                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8229                                             j);
8230                         j++;
8231                 }
8232 #endif
8233                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8234                                  bp->bnapi[i]);
8235                 if (rc)
8236                         break;
8237
8238                 irq->requested = 1;
8239
8240                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8241                         int numa_node = dev_to_node(&bp->pdev->dev);
8242
8243                         irq->have_cpumask = 1;
8244                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8245                                         irq->cpu_mask);
8246                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8247                         if (rc) {
8248                                 netdev_warn(bp->dev,
8249                                             "Set affinity failed, IRQ = %d\n",
8250                                             irq->vector);
8251                                 break;
8252                         }
8253                 }
8254         }
8255         return rc;
8256 }
8257
8258 static void bnxt_del_napi(struct bnxt *bp)
8259 {
8260         int i;
8261
8262         if (!bp->bnapi)
8263                 return;
8264
8265         for (i = 0; i < bp->cp_nr_rings; i++) {
8266                 struct bnxt_napi *bnapi = bp->bnapi[i];
8267
8268                 napi_hash_del(&bnapi->napi);
8269                 netif_napi_del(&bnapi->napi);
8270         }
8271         /* We called napi_hash_del() before netif_napi_del(), we need
8272          * to respect an RCU grace period before freeing napi structures.
8273          */
8274         synchronize_net();
8275 }
8276
8277 static void bnxt_init_napi(struct bnxt *bp)
8278 {
8279         int i;
8280         unsigned int cp_nr_rings = bp->cp_nr_rings;
8281         struct bnxt_napi *bnapi;
8282
8283         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8284                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8285
8286                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8287                         poll_fn = bnxt_poll_p5;
8288                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8289                         cp_nr_rings--;
8290                 for (i = 0; i < cp_nr_rings; i++) {
8291                         bnapi = bp->bnapi[i];
8292                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8293                 }
8294                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8295                         bnapi = bp->bnapi[cp_nr_rings];
8296                         netif_napi_add(bp->dev, &bnapi->napi,
8297                                        bnxt_poll_nitroa0, 64);
8298                 }
8299         } else {
8300                 bnapi = bp->bnapi[0];
8301                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8302         }
8303 }
8304
8305 static void bnxt_disable_napi(struct bnxt *bp)
8306 {
8307         int i;
8308
8309         if (!bp->bnapi)
8310                 return;
8311
8312         for (i = 0; i < bp->cp_nr_rings; i++) {
8313                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8314
8315                 if (bp->bnapi[i]->rx_ring)
8316                         cancel_work_sync(&cpr->dim.work);
8317
8318                 napi_disable(&bp->bnapi[i]->napi);
8319         }
8320 }
8321
8322 static void bnxt_enable_napi(struct bnxt *bp)
8323 {
8324         int i;
8325
8326         for (i = 0; i < bp->cp_nr_rings; i++) {
8327                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8328                 bp->bnapi[i]->in_reset = false;
8329
8330                 if (bp->bnapi[i]->rx_ring) {
8331                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8332                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8333                 }
8334                 napi_enable(&bp->bnapi[i]->napi);
8335         }
8336 }
8337
8338 void bnxt_tx_disable(struct bnxt *bp)
8339 {
8340         int i;
8341         struct bnxt_tx_ring_info *txr;
8342
8343         if (bp->tx_ring) {
8344                 for (i = 0; i < bp->tx_nr_rings; i++) {
8345                         txr = &bp->tx_ring[i];
8346                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8347                 }
8348         }
8349         /* Stop all TX queues */
8350         netif_tx_disable(bp->dev);
8351         netif_carrier_off(bp->dev);
8352 }
8353
8354 void bnxt_tx_enable(struct bnxt *bp)
8355 {
8356         int i;
8357         struct bnxt_tx_ring_info *txr;
8358
8359         for (i = 0; i < bp->tx_nr_rings; i++) {
8360                 txr = &bp->tx_ring[i];
8361                 txr->dev_state = 0;
8362         }
8363         netif_tx_wake_all_queues(bp->dev);
8364         if (bp->link_info.link_up)
8365                 netif_carrier_on(bp->dev);
8366 }
8367
8368 static void bnxt_report_link(struct bnxt *bp)
8369 {
8370         if (bp->link_info.link_up) {
8371                 const char *duplex;
8372                 const char *flow_ctrl;
8373                 u32 speed;
8374                 u16 fec;
8375
8376                 netif_carrier_on(bp->dev);
8377                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8378                         duplex = "full";
8379                 else
8380                         duplex = "half";
8381                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8382                         flow_ctrl = "ON - receive & transmit";
8383                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8384                         flow_ctrl = "ON - transmit";
8385                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8386                         flow_ctrl = "ON - receive";
8387                 else
8388                         flow_ctrl = "none";
8389                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8390                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8391                             speed, duplex, flow_ctrl);
8392                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8393                         netdev_info(bp->dev, "EEE is %s\n",
8394                                     bp->eee.eee_active ? "active" :
8395                                                          "not active");
8396                 fec = bp->link_info.fec_cfg;
8397                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8398                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8399                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8400                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8401                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8402         } else {
8403                 netif_carrier_off(bp->dev);
8404                 netdev_err(bp->dev, "NIC Link is Down\n");
8405         }
8406 }
8407
8408 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8409 {
8410         int rc = 0;
8411         struct hwrm_port_phy_qcaps_input req = {0};
8412         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8413         struct bnxt_link_info *link_info = &bp->link_info;
8414
8415         bp->flags &= ~BNXT_FLAG_EEE_CAP;
8416         if (bp->test_info)
8417                 bp->test_info->flags &= ~BNXT_TEST_FL_EXT_LPBK;
8418         if (bp->hwrm_spec_code < 0x10201)
8419                 return 0;
8420
8421         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8422
8423         mutex_lock(&bp->hwrm_cmd_lock);
8424         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8425         if (rc)
8426                 goto hwrm_phy_qcaps_exit;
8427
8428         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8429                 struct ethtool_eee *eee = &bp->eee;
8430                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8431
8432                 bp->flags |= BNXT_FLAG_EEE_CAP;
8433                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8434                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8435                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8436                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8437                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8438         }
8439         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8440                 if (bp->test_info)
8441                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8442         }
8443         if (resp->supported_speeds_auto_mode)
8444                 link_info->support_auto_speeds =
8445                         le16_to_cpu(resp->supported_speeds_auto_mode);
8446
8447         bp->port_count = resp->port_cnt;
8448
8449 hwrm_phy_qcaps_exit:
8450         mutex_unlock(&bp->hwrm_cmd_lock);
8451         return rc;
8452 }
8453
8454 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8455 {
8456         int rc = 0;
8457         struct bnxt_link_info *link_info = &bp->link_info;
8458         struct hwrm_port_phy_qcfg_input req = {0};
8459         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8460         u8 link_up = link_info->link_up;
8461         u16 diff;
8462
8463         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8464
8465         mutex_lock(&bp->hwrm_cmd_lock);
8466         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8467         if (rc) {
8468                 mutex_unlock(&bp->hwrm_cmd_lock);
8469                 return rc;
8470         }
8471
8472         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8473         link_info->phy_link_status = resp->link;
8474         link_info->duplex = resp->duplex_cfg;
8475         if (bp->hwrm_spec_code >= 0x10800)
8476                 link_info->duplex = resp->duplex_state;
8477         link_info->pause = resp->pause;
8478         link_info->auto_mode = resp->auto_mode;
8479         link_info->auto_pause_setting = resp->auto_pause;
8480         link_info->lp_pause = resp->link_partner_adv_pause;
8481         link_info->force_pause_setting = resp->force_pause;
8482         link_info->duplex_setting = resp->duplex_cfg;
8483         if (link_info->phy_link_status == BNXT_LINK_LINK)
8484                 link_info->link_speed = le16_to_cpu(resp->link_speed);
8485         else
8486                 link_info->link_speed = 0;
8487         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8488         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8489         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8490         link_info->lp_auto_link_speeds =
8491                 le16_to_cpu(resp->link_partner_adv_speeds);
8492         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8493         link_info->phy_ver[0] = resp->phy_maj;
8494         link_info->phy_ver[1] = resp->phy_min;
8495         link_info->phy_ver[2] = resp->phy_bld;
8496         link_info->media_type = resp->media_type;
8497         link_info->phy_type = resp->phy_type;
8498         link_info->transceiver = resp->xcvr_pkg_type;
8499         link_info->phy_addr = resp->eee_config_phy_addr &
8500                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8501         link_info->module_status = resp->module_status;
8502
8503         if (bp->flags & BNXT_FLAG_EEE_CAP) {
8504                 struct ethtool_eee *eee = &bp->eee;
8505                 u16 fw_speeds;
8506
8507                 eee->eee_active = 0;
8508                 if (resp->eee_config_phy_addr &
8509                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8510                         eee->eee_active = 1;
8511                         fw_speeds = le16_to_cpu(
8512                                 resp->link_partner_adv_eee_link_speed_mask);
8513                         eee->lp_advertised =
8514                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8515                 }
8516
8517                 /* Pull initial EEE config */
8518                 if (!chng_link_state) {
8519                         if (resp->eee_config_phy_addr &
8520                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8521                                 eee->eee_enabled = 1;
8522
8523                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8524                         eee->advertised =
8525                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8526
8527                         if (resp->eee_config_phy_addr &
8528                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8529                                 __le32 tmr;
8530
8531                                 eee->tx_lpi_enabled = 1;
8532                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8533                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8534                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8535                         }
8536                 }
8537         }
8538
8539         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8540         if (bp->hwrm_spec_code >= 0x10504)
8541                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8542
8543         /* TODO: need to add more logic to report VF link */
8544         if (chng_link_state) {
8545                 if (link_info->phy_link_status == BNXT_LINK_LINK)
8546                         link_info->link_up = 1;
8547                 else
8548                         link_info->link_up = 0;
8549                 if (link_up != link_info->link_up)
8550                         bnxt_report_link(bp);
8551         } else {
8552                 /* alwasy link down if not require to update link state */
8553                 link_info->link_up = 0;
8554         }
8555         mutex_unlock(&bp->hwrm_cmd_lock);
8556
8557         if (!BNXT_SINGLE_PF(bp))
8558                 return 0;
8559
8560         diff = link_info->support_auto_speeds ^ link_info->advertising;
8561         if ((link_info->support_auto_speeds | diff) !=
8562             link_info->support_auto_speeds) {
8563                 /* An advertised speed is no longer supported, so we need to
8564                  * update the advertisement settings.  Caller holds RTNL
8565                  * so we can modify link settings.
8566                  */
8567                 link_info->advertising = link_info->support_auto_speeds;
8568                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
8569                         bnxt_hwrm_set_link_setting(bp, true, false);
8570         }
8571         return 0;
8572 }
8573
8574 static void bnxt_get_port_module_status(struct bnxt *bp)
8575 {
8576         struct bnxt_link_info *link_info = &bp->link_info;
8577         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8578         u8 module_status;
8579
8580         if (bnxt_update_link(bp, true))
8581                 return;
8582
8583         module_status = link_info->module_status;
8584         switch (module_status) {
8585         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8586         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8587         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8588                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8589                             bp->pf.port_id);
8590                 if (bp->hwrm_spec_code >= 0x10201) {
8591                         netdev_warn(bp->dev, "Module part number %s\n",
8592                                     resp->phy_vendor_partnumber);
8593                 }
8594                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8595                         netdev_warn(bp->dev, "TX is disabled\n");
8596                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8597                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8598         }
8599 }
8600
8601 static void
8602 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8603 {
8604         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
8605                 if (bp->hwrm_spec_code >= 0x10201)
8606                         req->auto_pause =
8607                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
8608                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8609                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8610                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8611                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
8612                 req->enables |=
8613                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8614         } else {
8615                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8616                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8617                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8618                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
8619                 req->enables |=
8620                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
8621                 if (bp->hwrm_spec_code >= 0x10201) {
8622                         req->auto_pause = req->force_pause;
8623                         req->enables |= cpu_to_le32(
8624                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8625                 }
8626         }
8627 }
8628
8629 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
8630                                       struct hwrm_port_phy_cfg_input *req)
8631 {
8632         u8 autoneg = bp->link_info.autoneg;
8633         u16 fw_link_speed = bp->link_info.req_link_speed;
8634         u16 advertising = bp->link_info.advertising;
8635
8636         if (autoneg & BNXT_AUTONEG_SPEED) {
8637                 req->auto_mode |=
8638                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
8639
8640                 req->enables |= cpu_to_le32(
8641                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
8642                 req->auto_link_speed_mask = cpu_to_le16(advertising);
8643
8644                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
8645                 req->flags |=
8646                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
8647         } else {
8648                 req->force_link_speed = cpu_to_le16(fw_link_speed);
8649                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
8650         }
8651
8652         /* tell chimp that the setting takes effect immediately */
8653         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8654 }
8655
8656 int bnxt_hwrm_set_pause(struct bnxt *bp)
8657 {
8658         struct hwrm_port_phy_cfg_input req = {0};
8659         int rc;
8660
8661         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8662         bnxt_hwrm_set_pause_common(bp, &req);
8663
8664         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8665             bp->link_info.force_link_chng)
8666                 bnxt_hwrm_set_link_common(bp, &req);
8667
8668         mutex_lock(&bp->hwrm_cmd_lock);
8669         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8670         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8671                 /* since changing of pause setting doesn't trigger any link
8672                  * change event, the driver needs to update the current pause
8673                  * result upon successfully return of the phy_cfg command
8674                  */
8675                 bp->link_info.pause =
8676                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8677                 bp->link_info.auto_pause_setting = 0;
8678                 if (!bp->link_info.force_link_chng)
8679                         bnxt_report_link(bp);
8680         }
8681         bp->link_info.force_link_chng = false;
8682         mutex_unlock(&bp->hwrm_cmd_lock);
8683         return rc;
8684 }
8685
8686 static void bnxt_hwrm_set_eee(struct bnxt *bp,
8687                               struct hwrm_port_phy_cfg_input *req)
8688 {
8689         struct ethtool_eee *eee = &bp->eee;
8690
8691         if (eee->eee_enabled) {
8692                 u16 eee_speeds;
8693                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8694
8695                 if (eee->tx_lpi_enabled)
8696                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8697                 else
8698                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8699
8700                 req->flags |= cpu_to_le32(flags);
8701                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8702                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8703                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8704         } else {
8705                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8706         }
8707 }
8708
8709 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
8710 {
8711         struct hwrm_port_phy_cfg_input req = {0};
8712
8713         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8714         if (set_pause)
8715                 bnxt_hwrm_set_pause_common(bp, &req);
8716
8717         bnxt_hwrm_set_link_common(bp, &req);
8718
8719         if (set_eee)
8720                 bnxt_hwrm_set_eee(bp, &req);
8721         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8722 }
8723
8724 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8725 {
8726         struct hwrm_port_phy_cfg_input req = {0};
8727
8728         if (!BNXT_SINGLE_PF(bp))
8729                 return 0;
8730
8731         if (pci_num_vf(bp->pdev))
8732                 return 0;
8733
8734         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8735         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
8736         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8737 }
8738
8739 static int bnxt_fw_init_one(struct bnxt *bp);
8740
8741 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8742 {
8743         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8744         struct hwrm_func_drv_if_change_input req = {0};
8745         bool resc_reinit = false, fw_reset = false;
8746         u32 flags = 0;
8747         int rc;
8748
8749         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8750                 return 0;
8751
8752         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8753         if (up)
8754                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8755         mutex_lock(&bp->hwrm_cmd_lock);
8756         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8757         if (!rc)
8758                 flags = le32_to_cpu(resp->flags);
8759         mutex_unlock(&bp->hwrm_cmd_lock);
8760         if (rc)
8761                 return rc;
8762
8763         if (!up)
8764                 return 0;
8765
8766         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
8767                 resc_reinit = true;
8768         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
8769                 fw_reset = true;
8770
8771         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
8772                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
8773                 return -ENODEV;
8774         }
8775         if (resc_reinit || fw_reset) {
8776                 if (fw_reset) {
8777                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
8778                                 bnxt_ulp_stop(bp);
8779                         rc = bnxt_fw_init_one(bp);
8780                         if (rc) {
8781                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
8782                                 return rc;
8783                         }
8784                         bnxt_clear_int_mode(bp);
8785                         rc = bnxt_init_int_mode(bp);
8786                         if (rc) {
8787                                 netdev_err(bp->dev, "init int mode failed\n");
8788                                 return rc;
8789                         }
8790                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
8791                 }
8792                 if (BNXT_NEW_RM(bp)) {
8793                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8794
8795                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8796                         hw_resc->resv_cp_rings = 0;
8797                         hw_resc->resv_stat_ctxs = 0;
8798                         hw_resc->resv_irqs = 0;
8799                         hw_resc->resv_tx_rings = 0;
8800                         hw_resc->resv_rx_rings = 0;
8801                         hw_resc->resv_hw_ring_grps = 0;
8802                         hw_resc->resv_vnics = 0;
8803                         if (!fw_reset) {
8804                                 bp->tx_nr_rings = 0;
8805                                 bp->rx_nr_rings = 0;
8806                         }
8807                 }
8808         }
8809         return 0;
8810 }
8811
8812 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8813 {
8814         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8815         struct hwrm_port_led_qcaps_input req = {0};
8816         struct bnxt_pf_info *pf = &bp->pf;
8817         int rc;
8818
8819         bp->num_leds = 0;
8820         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8821                 return 0;
8822
8823         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8824         req.port_id = cpu_to_le16(pf->port_id);
8825         mutex_lock(&bp->hwrm_cmd_lock);
8826         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8827         if (rc) {
8828                 mutex_unlock(&bp->hwrm_cmd_lock);
8829                 return rc;
8830         }
8831         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8832                 int i;
8833
8834                 bp->num_leds = resp->num_leds;
8835                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8836                                                  bp->num_leds);
8837                 for (i = 0; i < bp->num_leds; i++) {
8838                         struct bnxt_led_info *led = &bp->leds[i];
8839                         __le16 caps = led->led_state_caps;
8840
8841                         if (!led->led_group_id ||
8842                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
8843                                 bp->num_leds = 0;
8844                                 break;
8845                         }
8846                 }
8847         }
8848         mutex_unlock(&bp->hwrm_cmd_lock);
8849         return 0;
8850 }
8851
8852 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8853 {
8854         struct hwrm_wol_filter_alloc_input req = {0};
8855         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8856         int rc;
8857
8858         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8859         req.port_id = cpu_to_le16(bp->pf.port_id);
8860         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8861         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8862         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8863         mutex_lock(&bp->hwrm_cmd_lock);
8864         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8865         if (!rc)
8866                 bp->wol_filter_id = resp->wol_filter_id;
8867         mutex_unlock(&bp->hwrm_cmd_lock);
8868         return rc;
8869 }
8870
8871 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8872 {
8873         struct hwrm_wol_filter_free_input req = {0};
8874         int rc;
8875
8876         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8877         req.port_id = cpu_to_le16(bp->pf.port_id);
8878         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8879         req.wol_filter_id = bp->wol_filter_id;
8880         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8881         return rc;
8882 }
8883
8884 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8885 {
8886         struct hwrm_wol_filter_qcfg_input req = {0};
8887         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8888         u16 next_handle = 0;
8889         int rc;
8890
8891         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8892         req.port_id = cpu_to_le16(bp->pf.port_id);
8893         req.handle = cpu_to_le16(handle);
8894         mutex_lock(&bp->hwrm_cmd_lock);
8895         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8896         if (!rc) {
8897                 next_handle = le16_to_cpu(resp->next_handle);
8898                 if (next_handle != 0) {
8899                         if (resp->wol_type ==
8900                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8901                                 bp->wol = 1;
8902                                 bp->wol_filter_id = resp->wol_filter_id;
8903                         }
8904                 }
8905         }
8906         mutex_unlock(&bp->hwrm_cmd_lock);
8907         return next_handle;
8908 }
8909
8910 static void bnxt_get_wol_settings(struct bnxt *bp)
8911 {
8912         u16 handle = 0;
8913
8914         bp->wol = 0;
8915         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8916                 return;
8917
8918         do {
8919                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8920         } while (handle && handle != 0xffff);
8921 }
8922
8923 #ifdef CONFIG_BNXT_HWMON
8924 static ssize_t bnxt_show_temp(struct device *dev,
8925                               struct device_attribute *devattr, char *buf)
8926 {
8927         struct hwrm_temp_monitor_query_input req = {0};
8928         struct hwrm_temp_monitor_query_output *resp;
8929         struct bnxt *bp = dev_get_drvdata(dev);
8930         u32 temp = 0;
8931
8932         resp = bp->hwrm_cmd_resp_addr;
8933         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8934         mutex_lock(&bp->hwrm_cmd_lock);
8935         if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8936                 temp = resp->temp * 1000; /* display millidegree */
8937         mutex_unlock(&bp->hwrm_cmd_lock);
8938
8939         return sprintf(buf, "%u\n", temp);
8940 }
8941 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8942
8943 static struct attribute *bnxt_attrs[] = {
8944         &sensor_dev_attr_temp1_input.dev_attr.attr,
8945         NULL
8946 };
8947 ATTRIBUTE_GROUPS(bnxt);
8948
8949 static void bnxt_hwmon_close(struct bnxt *bp)
8950 {
8951         if (bp->hwmon_dev) {
8952                 hwmon_device_unregister(bp->hwmon_dev);
8953                 bp->hwmon_dev = NULL;
8954         }
8955 }
8956
8957 static void bnxt_hwmon_open(struct bnxt *bp)
8958 {
8959         struct pci_dev *pdev = bp->pdev;
8960
8961         if (bp->hwmon_dev)
8962                 return;
8963
8964         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8965                                                           DRV_MODULE_NAME, bp,
8966                                                           bnxt_groups);
8967         if (IS_ERR(bp->hwmon_dev)) {
8968                 bp->hwmon_dev = NULL;
8969                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8970         }
8971 }
8972 #else
8973 static void bnxt_hwmon_close(struct bnxt *bp)
8974 {
8975 }
8976
8977 static void bnxt_hwmon_open(struct bnxt *bp)
8978 {
8979 }
8980 #endif
8981
8982 static bool bnxt_eee_config_ok(struct bnxt *bp)
8983 {
8984         struct ethtool_eee *eee = &bp->eee;
8985         struct bnxt_link_info *link_info = &bp->link_info;
8986
8987         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8988                 return true;
8989
8990         if (eee->eee_enabled) {
8991                 u32 advertising =
8992                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8993
8994                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8995                         eee->eee_enabled = 0;
8996                         return false;
8997                 }
8998                 if (eee->advertised & ~advertising) {
8999                         eee->advertised = advertising & eee->supported;
9000                         return false;
9001                 }
9002         }
9003         return true;
9004 }
9005
9006 static int bnxt_update_phy_setting(struct bnxt *bp)
9007 {
9008         int rc;
9009         bool update_link = false;
9010         bool update_pause = false;
9011         bool update_eee = false;
9012         struct bnxt_link_info *link_info = &bp->link_info;
9013
9014         rc = bnxt_update_link(bp, true);
9015         if (rc) {
9016                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9017                            rc);
9018                 return rc;
9019         }
9020         if (!BNXT_SINGLE_PF(bp))
9021                 return 0;
9022
9023         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9024             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9025             link_info->req_flow_ctrl)
9026                 update_pause = true;
9027         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9028             link_info->force_pause_setting != link_info->req_flow_ctrl)
9029                 update_pause = true;
9030         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9031                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9032                         update_link = true;
9033                 if (link_info->req_link_speed != link_info->force_link_speed)
9034                         update_link = true;
9035                 if (link_info->req_duplex != link_info->duplex_setting)
9036                         update_link = true;
9037         } else {
9038                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9039                         update_link = true;
9040                 if (link_info->advertising != link_info->auto_link_speeds)
9041                         update_link = true;
9042         }
9043
9044         /* The last close may have shutdown the link, so need to call
9045          * PHY_CFG to bring it back up.
9046          */
9047         if (!netif_carrier_ok(bp->dev))
9048                 update_link = true;
9049
9050         if (!bnxt_eee_config_ok(bp))
9051                 update_eee = true;
9052
9053         if (update_link)
9054                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9055         else if (update_pause)
9056                 rc = bnxt_hwrm_set_pause(bp);
9057         if (rc) {
9058                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9059                            rc);
9060                 return rc;
9061         }
9062
9063         return rc;
9064 }
9065
9066 /* Common routine to pre-map certain register block to different GRC window.
9067  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9068  * in PF and 3 windows in VF that can be customized to map in different
9069  * register blocks.
9070  */
9071 static void bnxt_preset_reg_win(struct bnxt *bp)
9072 {
9073         if (BNXT_PF(bp)) {
9074                 /* CAG registers map to GRC window #4 */
9075                 writel(BNXT_CAG_REG_BASE,
9076                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9077         }
9078 }
9079
9080 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9081
9082 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9083 {
9084         int rc = 0;
9085
9086         bnxt_preset_reg_win(bp);
9087         netif_carrier_off(bp->dev);
9088         if (irq_re_init) {
9089                 /* Reserve rings now if none were reserved at driver probe. */
9090                 rc = bnxt_init_dflt_ring_mode(bp);
9091                 if (rc) {
9092                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9093                         return rc;
9094                 }
9095         }
9096         rc = bnxt_reserve_rings(bp, irq_re_init);
9097         if (rc)
9098                 return rc;
9099         if ((bp->flags & BNXT_FLAG_RFS) &&
9100             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9101                 /* disable RFS if falling back to INTA */
9102                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9103                 bp->flags &= ~BNXT_FLAG_RFS;
9104         }
9105
9106         rc = bnxt_alloc_mem(bp, irq_re_init);
9107         if (rc) {
9108                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9109                 goto open_err_free_mem;
9110         }
9111
9112         if (irq_re_init) {
9113                 bnxt_init_napi(bp);
9114                 rc = bnxt_request_irq(bp);
9115                 if (rc) {
9116                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9117                         goto open_err_irq;
9118                 }
9119         }
9120
9121         bnxt_enable_napi(bp);
9122         bnxt_debug_dev_init(bp);
9123
9124         rc = bnxt_init_nic(bp, irq_re_init);
9125         if (rc) {
9126                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9127                 goto open_err;
9128         }
9129
9130         if (link_re_init) {
9131                 mutex_lock(&bp->link_lock);
9132                 rc = bnxt_update_phy_setting(bp);
9133                 mutex_unlock(&bp->link_lock);
9134                 if (rc) {
9135                         netdev_warn(bp->dev, "failed to update phy settings\n");
9136                         if (BNXT_SINGLE_PF(bp)) {
9137                                 bp->link_info.phy_retry = true;
9138                                 bp->link_info.phy_retry_expires =
9139                                         jiffies + 5 * HZ;
9140                         }
9141                 }
9142         }
9143
9144         if (irq_re_init)
9145                 udp_tunnel_get_rx_info(bp->dev);
9146
9147         set_bit(BNXT_STATE_OPEN, &bp->state);
9148         bnxt_enable_int(bp);
9149         /* Enable TX queues */
9150         bnxt_tx_enable(bp);
9151         mod_timer(&bp->timer, jiffies + bp->current_interval);
9152         /* Poll link status and check for SFP+ module status */
9153         bnxt_get_port_module_status(bp);
9154
9155         /* VF-reps may need to be re-opened after the PF is re-opened */
9156         if (BNXT_PF(bp))
9157                 bnxt_vf_reps_open(bp);
9158         return 0;
9159
9160 open_err:
9161         bnxt_debug_dev_exit(bp);
9162         bnxt_disable_napi(bp);
9163
9164 open_err_irq:
9165         bnxt_del_napi(bp);
9166
9167 open_err_free_mem:
9168         bnxt_free_skbs(bp);
9169         bnxt_free_irq(bp);
9170         bnxt_free_mem(bp, true);
9171         return rc;
9172 }
9173
9174 /* rtnl_lock held */
9175 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9176 {
9177         int rc = 0;
9178
9179         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9180         if (rc) {
9181                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9182                 dev_close(bp->dev);
9183         }
9184         return rc;
9185 }
9186
9187 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9188  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9189  * self tests.
9190  */
9191 int bnxt_half_open_nic(struct bnxt *bp)
9192 {
9193         int rc = 0;
9194
9195         rc = bnxt_alloc_mem(bp, false);
9196         if (rc) {
9197                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9198                 goto half_open_err;
9199         }
9200         rc = bnxt_init_nic(bp, false);
9201         if (rc) {
9202                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9203                 goto half_open_err;
9204         }
9205         return 0;
9206
9207 half_open_err:
9208         bnxt_free_skbs(bp);
9209         bnxt_free_mem(bp, false);
9210         dev_close(bp->dev);
9211         return rc;
9212 }
9213
9214 /* rtnl_lock held, this call can only be made after a previous successful
9215  * call to bnxt_half_open_nic().
9216  */
9217 void bnxt_half_close_nic(struct bnxt *bp)
9218 {
9219         bnxt_hwrm_resource_free(bp, false, false);
9220         bnxt_free_skbs(bp);
9221         bnxt_free_mem(bp, false);
9222 }
9223
9224 static int bnxt_open(struct net_device *dev)
9225 {
9226         struct bnxt *bp = netdev_priv(dev);
9227         int rc;
9228
9229         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9230                 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9231                 return -ENODEV;
9232         }
9233
9234         rc = bnxt_hwrm_if_change(bp, true);
9235         if (rc)
9236                 return rc;
9237         rc = __bnxt_open_nic(bp, true, true);
9238         if (rc) {
9239                 bnxt_hwrm_if_change(bp, false);
9240         } else {
9241                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9242                         if (BNXT_PF(bp)) {
9243                                 struct bnxt_pf_info *pf = &bp->pf;
9244                                 int n = pf->active_vfs;
9245
9246                                 if (n)
9247                                         bnxt_cfg_hw_sriov(bp, &n, true);
9248                         }
9249                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9250                                 bnxt_ulp_start(bp, 0);
9251                 }
9252                 bnxt_hwmon_open(bp);
9253         }
9254
9255         return rc;
9256 }
9257
9258 static bool bnxt_drv_busy(struct bnxt *bp)
9259 {
9260         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9261                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9262 }
9263
9264 static void bnxt_get_ring_stats(struct bnxt *bp,
9265                                 struct rtnl_link_stats64 *stats);
9266
9267 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9268                              bool link_re_init)
9269 {
9270         /* Close the VF-reps before closing PF */
9271         if (BNXT_PF(bp))
9272                 bnxt_vf_reps_close(bp);
9273
9274         /* Change device state to avoid TX queue wake up's */
9275         bnxt_tx_disable(bp);
9276
9277         clear_bit(BNXT_STATE_OPEN, &bp->state);
9278         smp_mb__after_atomic();
9279         while (bnxt_drv_busy(bp))
9280                 msleep(20);
9281
9282         /* Flush rings and and disable interrupts */
9283         bnxt_shutdown_nic(bp, irq_re_init);
9284
9285         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9286
9287         bnxt_debug_dev_exit(bp);
9288         bnxt_disable_napi(bp);
9289         del_timer_sync(&bp->timer);
9290         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) &&
9291             pci_is_enabled(bp->pdev))
9292                 pci_disable_device(bp->pdev);
9293
9294         bnxt_free_skbs(bp);
9295
9296         /* Save ring stats before shutdown */
9297         if (bp->bnapi)
9298                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9299         if (irq_re_init) {
9300                 bnxt_free_irq(bp);
9301                 bnxt_del_napi(bp);
9302         }
9303         bnxt_free_mem(bp, irq_re_init);
9304 }
9305
9306 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9307 {
9308         int rc = 0;
9309
9310         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9311                 /* If we get here, it means firmware reset is in progress
9312                  * while we are trying to close.  We can safely proceed with
9313                  * the close because we are holding rtnl_lock().  Some firmware
9314                  * messages may fail as we proceed to close.  We set the
9315                  * ABORT_ERR flag here so that the FW reset thread will later
9316                  * abort when it gets the rtnl_lock() and sees the flag.
9317                  */
9318                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9319                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9320         }
9321
9322 #ifdef CONFIG_BNXT_SRIOV
9323         if (bp->sriov_cfg) {
9324                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9325                                                       !bp->sriov_cfg,
9326                                                       BNXT_SRIOV_CFG_WAIT_TMO);
9327                 if (rc)
9328                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9329         }
9330 #endif
9331         __bnxt_close_nic(bp, irq_re_init, link_re_init);
9332         return rc;
9333 }
9334
9335 static int bnxt_close(struct net_device *dev)
9336 {
9337         struct bnxt *bp = netdev_priv(dev);
9338
9339         bnxt_hwmon_close(bp);
9340         bnxt_close_nic(bp, true, true);
9341         bnxt_hwrm_shutdown_link(bp);
9342         bnxt_hwrm_if_change(bp, false);
9343         return 0;
9344 }
9345
9346 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9347                                    u16 *val)
9348 {
9349         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9350         struct hwrm_port_phy_mdio_read_input req = {0};
9351         int rc;
9352
9353         if (bp->hwrm_spec_code < 0x10a00)
9354                 return -EOPNOTSUPP;
9355
9356         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9357         req.port_id = cpu_to_le16(bp->pf.port_id);
9358         req.phy_addr = phy_addr;
9359         req.reg_addr = cpu_to_le16(reg & 0x1f);
9360         if (mdio_phy_id_is_c45(phy_addr)) {
9361                 req.cl45_mdio = 1;
9362                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9363                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9364                 req.reg_addr = cpu_to_le16(reg);
9365         }
9366
9367         mutex_lock(&bp->hwrm_cmd_lock);
9368         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9369         if (!rc)
9370                 *val = le16_to_cpu(resp->reg_data);
9371         mutex_unlock(&bp->hwrm_cmd_lock);
9372         return rc;
9373 }
9374
9375 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9376                                     u16 val)
9377 {
9378         struct hwrm_port_phy_mdio_write_input req = {0};
9379
9380         if (bp->hwrm_spec_code < 0x10a00)
9381                 return -EOPNOTSUPP;
9382
9383         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9384         req.port_id = cpu_to_le16(bp->pf.port_id);
9385         req.phy_addr = phy_addr;
9386         req.reg_addr = cpu_to_le16(reg & 0x1f);
9387         if (mdio_phy_id_is_c45(phy_addr)) {
9388                 req.cl45_mdio = 1;
9389                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9390                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9391                 req.reg_addr = cpu_to_le16(reg);
9392         }
9393         req.reg_data = cpu_to_le16(val);
9394
9395         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9396 }
9397
9398 /* rtnl_lock held */
9399 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9400 {
9401         struct mii_ioctl_data *mdio = if_mii(ifr);
9402         struct bnxt *bp = netdev_priv(dev);
9403         int rc;
9404
9405         switch (cmd) {
9406         case SIOCGMIIPHY:
9407                 mdio->phy_id = bp->link_info.phy_addr;
9408
9409                 /* fallthru */
9410         case SIOCGMIIREG: {
9411                 u16 mii_regval = 0;
9412
9413                 if (!netif_running(dev))
9414                         return -EAGAIN;
9415
9416                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9417                                              &mii_regval);
9418                 mdio->val_out = mii_regval;
9419                 return rc;
9420         }
9421
9422         case SIOCSMIIREG:
9423                 if (!netif_running(dev))
9424                         return -EAGAIN;
9425
9426                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9427                                                 mdio->val_in);
9428
9429         default:
9430                 /* do nothing */
9431                 break;
9432         }
9433         return -EOPNOTSUPP;
9434 }
9435
9436 static void bnxt_get_ring_stats(struct bnxt *bp,
9437                                 struct rtnl_link_stats64 *stats)
9438 {
9439         int i;
9440
9441
9442         for (i = 0; i < bp->cp_nr_rings; i++) {
9443                 struct bnxt_napi *bnapi = bp->bnapi[i];
9444                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9445                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
9446
9447                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
9448                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
9449                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
9450
9451                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
9452                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
9453                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
9454
9455                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
9456                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
9457                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
9458
9459                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
9460                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
9461                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
9462
9463                 stats->rx_missed_errors +=
9464                         le64_to_cpu(hw_stats->rx_discard_pkts);
9465
9466                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
9467
9468                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
9469         }
9470 }
9471
9472 static void bnxt_add_prev_stats(struct bnxt *bp,
9473                                 struct rtnl_link_stats64 *stats)
9474 {
9475         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9476
9477         stats->rx_packets += prev_stats->rx_packets;
9478         stats->tx_packets += prev_stats->tx_packets;
9479         stats->rx_bytes += prev_stats->rx_bytes;
9480         stats->tx_bytes += prev_stats->tx_bytes;
9481         stats->rx_missed_errors += prev_stats->rx_missed_errors;
9482         stats->multicast += prev_stats->multicast;
9483         stats->tx_dropped += prev_stats->tx_dropped;
9484 }
9485
9486 static void
9487 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9488 {
9489         struct bnxt *bp = netdev_priv(dev);
9490
9491         set_bit(BNXT_STATE_READ_STATS, &bp->state);
9492         /* Make sure bnxt_close_nic() sees that we are reading stats before
9493          * we check the BNXT_STATE_OPEN flag.
9494          */
9495         smp_mb__after_atomic();
9496         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9497                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9498                 *stats = bp->net_stats_prev;
9499                 return;
9500         }
9501
9502         bnxt_get_ring_stats(bp, stats);
9503         bnxt_add_prev_stats(bp, stats);
9504
9505         if (bp->flags & BNXT_FLAG_PORT_STATS) {
9506                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
9507                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
9508
9509                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
9510                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
9511                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
9512                                           le64_to_cpu(rx->rx_ovrsz_frames) +
9513                                           le64_to_cpu(rx->rx_runt_frames);
9514                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
9515                                    le64_to_cpu(rx->rx_jbr_frames);
9516                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
9517                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
9518                 stats->tx_errors = le64_to_cpu(tx->tx_err);
9519         }
9520         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9521 }
9522
9523 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9524 {
9525         struct net_device *dev = bp->dev;
9526         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9527         struct netdev_hw_addr *ha;
9528         u8 *haddr;
9529         int mc_count = 0;
9530         bool update = false;
9531         int off = 0;
9532
9533         netdev_for_each_mc_addr(ha, dev) {
9534                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9535                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9536                         vnic->mc_list_count = 0;
9537                         return false;
9538                 }
9539                 haddr = ha->addr;
9540                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9541                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9542                         update = true;
9543                 }
9544                 off += ETH_ALEN;
9545                 mc_count++;
9546         }
9547         if (mc_count)
9548                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9549
9550         if (mc_count != vnic->mc_list_count) {
9551                 vnic->mc_list_count = mc_count;
9552                 update = true;
9553         }
9554         return update;
9555 }
9556
9557 static bool bnxt_uc_list_updated(struct bnxt *bp)
9558 {
9559         struct net_device *dev = bp->dev;
9560         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9561         struct netdev_hw_addr *ha;
9562         int off = 0;
9563
9564         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9565                 return true;
9566
9567         netdev_for_each_uc_addr(ha, dev) {
9568                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9569                         return true;
9570
9571                 off += ETH_ALEN;
9572         }
9573         return false;
9574 }
9575
9576 static void bnxt_set_rx_mode(struct net_device *dev)
9577 {
9578         struct bnxt *bp = netdev_priv(dev);
9579         struct bnxt_vnic_info *vnic;
9580         bool mc_update = false;
9581         bool uc_update;
9582         u32 mask;
9583
9584         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
9585                 return;
9586
9587         vnic = &bp->vnic_info[0];
9588         mask = vnic->rx_mask;
9589         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9590                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
9591                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9592                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
9593
9594         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
9595                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9596
9597         uc_update = bnxt_uc_list_updated(bp);
9598
9599         if (dev->flags & IFF_BROADCAST)
9600                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9601         if (dev->flags & IFF_ALLMULTI) {
9602                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9603                 vnic->mc_list_count = 0;
9604         } else {
9605                 mc_update = bnxt_mc_list_updated(bp, &mask);
9606         }
9607
9608         if (mask != vnic->rx_mask || uc_update || mc_update) {
9609                 vnic->rx_mask = mask;
9610
9611                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
9612                 bnxt_queue_sp_work(bp);
9613         }
9614 }
9615
9616 static int bnxt_cfg_rx_mode(struct bnxt *bp)
9617 {
9618         struct net_device *dev = bp->dev;
9619         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9620         struct netdev_hw_addr *ha;
9621         int i, off = 0, rc;
9622         bool uc_update;
9623
9624         netif_addr_lock_bh(dev);
9625         uc_update = bnxt_uc_list_updated(bp);
9626         netif_addr_unlock_bh(dev);
9627
9628         if (!uc_update)
9629                 goto skip_uc;
9630
9631         mutex_lock(&bp->hwrm_cmd_lock);
9632         for (i = 1; i < vnic->uc_filter_count; i++) {
9633                 struct hwrm_cfa_l2_filter_free_input req = {0};
9634
9635                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
9636                                        -1);
9637
9638                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
9639
9640                 rc = _hwrm_send_message(bp, &req, sizeof(req),
9641                                         HWRM_CMD_TIMEOUT);
9642         }
9643         mutex_unlock(&bp->hwrm_cmd_lock);
9644
9645         vnic->uc_filter_count = 1;
9646
9647         netif_addr_lock_bh(dev);
9648         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
9649                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9650         } else {
9651                 netdev_for_each_uc_addr(ha, dev) {
9652                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
9653                         off += ETH_ALEN;
9654                         vnic->uc_filter_count++;
9655                 }
9656         }
9657         netif_addr_unlock_bh(dev);
9658
9659         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
9660                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
9661                 if (rc) {
9662                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
9663                                    rc);
9664                         vnic->uc_filter_count = i;
9665                         return rc;
9666                 }
9667         }
9668
9669 skip_uc:
9670         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9671         if (rc && vnic->mc_list_count) {
9672                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
9673                             rc);
9674                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9675                 vnic->mc_list_count = 0;
9676                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9677         }
9678         if (rc)
9679                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
9680                            rc);
9681
9682         return rc;
9683 }
9684
9685 static bool bnxt_can_reserve_rings(struct bnxt *bp)
9686 {
9687 #ifdef CONFIG_BNXT_SRIOV
9688         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
9689                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9690
9691                 /* No minimum rings were provisioned by the PF.  Don't
9692                  * reserve rings by default when device is down.
9693                  */
9694                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
9695                         return true;
9696
9697                 if (!netif_running(bp->dev))
9698                         return false;
9699         }
9700 #endif
9701         return true;
9702 }
9703
9704 /* If the chip and firmware supports RFS */
9705 static bool bnxt_rfs_supported(struct bnxt *bp)
9706 {
9707         if (bp->flags & BNXT_FLAG_CHIP_P5) {
9708                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
9709                         return true;
9710                 return false;
9711         }
9712         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
9713                 return true;
9714         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9715                 return true;
9716         return false;
9717 }
9718
9719 /* If runtime conditions support RFS */
9720 static bool bnxt_rfs_capable(struct bnxt *bp)
9721 {
9722 #ifdef CONFIG_RFS_ACCEL
9723         int vnics, max_vnics, max_rss_ctxs;
9724
9725         if (bp->flags & BNXT_FLAG_CHIP_P5)
9726                 return bnxt_rfs_supported(bp);
9727         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
9728                 return false;
9729
9730         vnics = 1 + bp->rx_nr_rings;
9731         max_vnics = bnxt_get_max_func_vnics(bp);
9732         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
9733
9734         /* RSS contexts not a limiting factor */
9735         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9736                 max_rss_ctxs = max_vnics;
9737         if (vnics > max_vnics || vnics > max_rss_ctxs) {
9738                 if (bp->rx_nr_rings > 1)
9739                         netdev_warn(bp->dev,
9740                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
9741                                     min(max_rss_ctxs - 1, max_vnics - 1));
9742                 return false;
9743         }
9744
9745         if (!BNXT_NEW_RM(bp))
9746                 return true;
9747
9748         if (vnics == bp->hw_resc.resv_vnics)
9749                 return true;
9750
9751         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
9752         if (vnics <= bp->hw_resc.resv_vnics)
9753                 return true;
9754
9755         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
9756         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
9757         return false;
9758 #else
9759         return false;
9760 #endif
9761 }
9762
9763 static netdev_features_t bnxt_fix_features(struct net_device *dev,
9764                                            netdev_features_t features)
9765 {
9766         struct bnxt *bp = netdev_priv(dev);
9767
9768         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
9769                 features &= ~NETIF_F_NTUPLE;
9770
9771         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9772                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9773
9774         if (!(features & NETIF_F_GRO))
9775                 features &= ~NETIF_F_GRO_HW;
9776
9777         if (features & NETIF_F_GRO_HW)
9778                 features &= ~NETIF_F_LRO;
9779
9780         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
9781          * turned on or off together.
9782          */
9783         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
9784             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
9785                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
9786                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9787                                       NETIF_F_HW_VLAN_STAG_RX);
9788                 else
9789                         features |= NETIF_F_HW_VLAN_CTAG_RX |
9790                                     NETIF_F_HW_VLAN_STAG_RX;
9791         }
9792 #ifdef CONFIG_BNXT_SRIOV
9793         if (BNXT_VF(bp)) {
9794                 if (bp->vf.vlan) {
9795                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9796                                       NETIF_F_HW_VLAN_STAG_RX);
9797                 }
9798         }
9799 #endif
9800         return features;
9801 }
9802
9803 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
9804 {
9805         struct bnxt *bp = netdev_priv(dev);
9806         u32 flags = bp->flags;
9807         u32 changes;
9808         int rc = 0;
9809         bool re_init = false;
9810         bool update_tpa = false;
9811
9812         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
9813         if (features & NETIF_F_GRO_HW)
9814                 flags |= BNXT_FLAG_GRO;
9815         else if (features & NETIF_F_LRO)
9816                 flags |= BNXT_FLAG_LRO;
9817
9818         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9819                 flags &= ~BNXT_FLAG_TPA;
9820
9821         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9822                 flags |= BNXT_FLAG_STRIP_VLAN;
9823
9824         if (features & NETIF_F_NTUPLE)
9825                 flags |= BNXT_FLAG_RFS;
9826
9827         changes = flags ^ bp->flags;
9828         if (changes & BNXT_FLAG_TPA) {
9829                 update_tpa = true;
9830                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
9831                     (flags & BNXT_FLAG_TPA) == 0 ||
9832                     (bp->flags & BNXT_FLAG_CHIP_P5))
9833                         re_init = true;
9834         }
9835
9836         if (changes & ~BNXT_FLAG_TPA)
9837                 re_init = true;
9838
9839         if (flags != bp->flags) {
9840                 u32 old_flags = bp->flags;
9841
9842                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9843                         bp->flags = flags;
9844                         if (update_tpa)
9845                                 bnxt_set_ring_params(bp);
9846                         return rc;
9847                 }
9848
9849                 if (re_init) {
9850                         bnxt_close_nic(bp, false, false);
9851                         bp->flags = flags;
9852                         if (update_tpa)
9853                                 bnxt_set_ring_params(bp);
9854
9855                         return bnxt_open_nic(bp, false, false);
9856                 }
9857                 if (update_tpa) {
9858                         bp->flags = flags;
9859                         rc = bnxt_set_tpa(bp,
9860                                           (flags & BNXT_FLAG_TPA) ?
9861                                           true : false);
9862                         if (rc)
9863                                 bp->flags = old_flags;
9864                 }
9865         }
9866         return rc;
9867 }
9868
9869 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9870                                        u32 ring_id, u32 *prod, u32 *cons)
9871 {
9872         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9873         struct hwrm_dbg_ring_info_get_input req = {0};
9874         int rc;
9875
9876         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9877         req.ring_type = ring_type;
9878         req.fw_ring_id = cpu_to_le32(ring_id);
9879         mutex_lock(&bp->hwrm_cmd_lock);
9880         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9881         if (!rc) {
9882                 *prod = le32_to_cpu(resp->producer_index);
9883                 *cons = le32_to_cpu(resp->consumer_index);
9884         }
9885         mutex_unlock(&bp->hwrm_cmd_lock);
9886         return rc;
9887 }
9888
9889 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9890 {
9891         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9892         int i = bnapi->index;
9893
9894         if (!txr)
9895                 return;
9896
9897         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9898                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9899                     txr->tx_cons);
9900 }
9901
9902 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9903 {
9904         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9905         int i = bnapi->index;
9906
9907         if (!rxr)
9908                 return;
9909
9910         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9911                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9912                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9913                     rxr->rx_sw_agg_prod);
9914 }
9915
9916 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9917 {
9918         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9919         int i = bnapi->index;
9920
9921         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9922                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9923 }
9924
9925 static void bnxt_dbg_dump_states(struct bnxt *bp)
9926 {
9927         int i;
9928         struct bnxt_napi *bnapi;
9929
9930         for (i = 0; i < bp->cp_nr_rings; i++) {
9931                 bnapi = bp->bnapi[i];
9932                 if (netif_msg_drv(bp)) {
9933                         bnxt_dump_tx_sw_state(bnapi);
9934                         bnxt_dump_rx_sw_state(bnapi);
9935                         bnxt_dump_cp_sw_state(bnapi);
9936                 }
9937         }
9938 }
9939
9940 static void bnxt_reset_task(struct bnxt *bp, bool silent)
9941 {
9942         if (!silent)
9943                 bnxt_dbg_dump_states(bp);
9944         if (netif_running(bp->dev)) {
9945                 int rc;
9946
9947                 if (silent) {
9948                         bnxt_close_nic(bp, false, false);
9949                         bnxt_open_nic(bp, false, false);
9950                 } else {
9951                         bnxt_ulp_stop(bp);
9952                         bnxt_close_nic(bp, true, false);
9953                         rc = bnxt_open_nic(bp, true, false);
9954                         bnxt_ulp_start(bp, rc);
9955                 }
9956         }
9957 }
9958
9959 static void bnxt_tx_timeout(struct net_device *dev)
9960 {
9961         struct bnxt *bp = netdev_priv(dev);
9962
9963         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
9964         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
9965         bnxt_queue_sp_work(bp);
9966 }
9967
9968 static void bnxt_fw_health_check(struct bnxt *bp)
9969 {
9970         struct bnxt_fw_health *fw_health = bp->fw_health;
9971         u32 val;
9972
9973         if (!fw_health || !fw_health->enabled ||
9974             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9975                 return;
9976
9977         if (fw_health->tmr_counter) {
9978                 fw_health->tmr_counter--;
9979                 return;
9980         }
9981
9982         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
9983         if (val == fw_health->last_fw_heartbeat)
9984                 goto fw_reset;
9985
9986         fw_health->last_fw_heartbeat = val;
9987
9988         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
9989         if (val != fw_health->last_fw_reset_cnt)
9990                 goto fw_reset;
9991
9992         fw_health->tmr_counter = fw_health->tmr_multiplier;
9993         return;
9994
9995 fw_reset:
9996         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
9997         bnxt_queue_sp_work(bp);
9998 }
9999
10000 static void bnxt_timer(struct timer_list *t)
10001 {
10002         struct bnxt *bp = from_timer(bp, t, timer);
10003         struct net_device *dev = bp->dev;
10004
10005         if (!netif_running(dev))
10006                 return;
10007
10008         if (atomic_read(&bp->intr_sem) != 0)
10009                 goto bnxt_restart_timer;
10010
10011         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10012                 bnxt_fw_health_check(bp);
10013
10014         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
10015             bp->stats_coal_ticks) {
10016                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10017                 bnxt_queue_sp_work(bp);
10018         }
10019
10020         if (bnxt_tc_flower_enabled(bp)) {
10021                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10022                 bnxt_queue_sp_work(bp);
10023         }
10024
10025         if (bp->link_info.phy_retry) {
10026                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10027                         bp->link_info.phy_retry = false;
10028                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10029                 } else {
10030                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10031                         bnxt_queue_sp_work(bp);
10032                 }
10033         }
10034
10035         if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
10036                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10037                 bnxt_queue_sp_work(bp);
10038         }
10039 bnxt_restart_timer:
10040         mod_timer(&bp->timer, jiffies + bp->current_interval);
10041 }
10042
10043 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10044 {
10045         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10046          * set.  If the device is being closed, bnxt_close() may be holding
10047          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10048          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10049          */
10050         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10051         rtnl_lock();
10052 }
10053
10054 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10055 {
10056         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10057         rtnl_unlock();
10058 }
10059
10060 /* Only called from bnxt_sp_task() */
10061 static void bnxt_reset(struct bnxt *bp, bool silent)
10062 {
10063         bnxt_rtnl_lock_sp(bp);
10064         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10065                 bnxt_reset_task(bp, silent);
10066         bnxt_rtnl_unlock_sp(bp);
10067 }
10068
10069 static void bnxt_fw_reset_close(struct bnxt *bp)
10070 {
10071         bnxt_ulp_stop(bp);
10072         __bnxt_close_nic(bp, true, false);
10073         bnxt_clear_int_mode(bp);
10074         bnxt_hwrm_func_drv_unrgtr(bp);
10075         bnxt_free_ctx_mem(bp);
10076         kfree(bp->ctx);
10077         bp->ctx = NULL;
10078 }
10079
10080 static bool is_bnxt_fw_ok(struct bnxt *bp)
10081 {
10082         struct bnxt_fw_health *fw_health = bp->fw_health;
10083         bool no_heartbeat = false, has_reset = false;
10084         u32 val;
10085
10086         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10087         if (val == fw_health->last_fw_heartbeat)
10088                 no_heartbeat = true;
10089
10090         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10091         if (val != fw_health->last_fw_reset_cnt)
10092                 has_reset = true;
10093
10094         if (!no_heartbeat && has_reset)
10095                 return true;
10096
10097         return false;
10098 }
10099
10100 /* rtnl_lock is acquired before calling this function */
10101 static void bnxt_force_fw_reset(struct bnxt *bp)
10102 {
10103         struct bnxt_fw_health *fw_health = bp->fw_health;
10104         u32 wait_dsecs;
10105
10106         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10107             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10108                 return;
10109
10110         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10111         bnxt_fw_reset_close(bp);
10112         wait_dsecs = fw_health->master_func_wait_dsecs;
10113         if (fw_health->master) {
10114                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10115                         wait_dsecs = 0;
10116                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10117         } else {
10118                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10119                 wait_dsecs = fw_health->normal_func_wait_dsecs;
10120                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10121         }
10122
10123         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10124         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10125         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10126 }
10127
10128 void bnxt_fw_exception(struct bnxt *bp)
10129 {
10130         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10131         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10132         bnxt_rtnl_lock_sp(bp);
10133         bnxt_force_fw_reset(bp);
10134         bnxt_rtnl_unlock_sp(bp);
10135 }
10136
10137 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10138  * < 0 on error.
10139  */
10140 static int bnxt_get_registered_vfs(struct bnxt *bp)
10141 {
10142 #ifdef CONFIG_BNXT_SRIOV
10143         int rc;
10144
10145         if (!BNXT_PF(bp))
10146                 return 0;
10147
10148         rc = bnxt_hwrm_func_qcfg(bp);
10149         if (rc) {
10150                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10151                 return rc;
10152         }
10153         if (bp->pf.registered_vfs)
10154                 return bp->pf.registered_vfs;
10155         if (bp->sriov_cfg)
10156                 return 1;
10157 #endif
10158         return 0;
10159 }
10160
10161 void bnxt_fw_reset(struct bnxt *bp)
10162 {
10163         bnxt_rtnl_lock_sp(bp);
10164         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10165             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10166                 int n = 0, tmo;
10167
10168                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10169                 if (bp->pf.active_vfs &&
10170                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10171                         n = bnxt_get_registered_vfs(bp);
10172                 if (n < 0) {
10173                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10174                                    n);
10175                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10176                         dev_close(bp->dev);
10177                         goto fw_reset_exit;
10178                 } else if (n > 0) {
10179                         u16 vf_tmo_dsecs = n * 10;
10180
10181                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10182                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10183                         bp->fw_reset_state =
10184                                 BNXT_FW_RESET_STATE_POLL_VF;
10185                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10186                         goto fw_reset_exit;
10187                 }
10188                 bnxt_fw_reset_close(bp);
10189                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10190                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10191                         tmo = HZ / 10;
10192                 } else {
10193                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10194                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10195                 }
10196                 bnxt_queue_fw_reset_work(bp, tmo);
10197         }
10198 fw_reset_exit:
10199         bnxt_rtnl_unlock_sp(bp);
10200 }
10201
10202 static void bnxt_chk_missed_irq(struct bnxt *bp)
10203 {
10204         int i;
10205
10206         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10207                 return;
10208
10209         for (i = 0; i < bp->cp_nr_rings; i++) {
10210                 struct bnxt_napi *bnapi = bp->bnapi[i];
10211                 struct bnxt_cp_ring_info *cpr;
10212                 u32 fw_ring_id;
10213                 int j;
10214
10215                 if (!bnapi)
10216                         continue;
10217
10218                 cpr = &bnapi->cp_ring;
10219                 for (j = 0; j < 2; j++) {
10220                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10221                         u32 val[2];
10222
10223                         if (!cpr2 || cpr2->has_more_work ||
10224                             !bnxt_has_work(bp, cpr2))
10225                                 continue;
10226
10227                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10228                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10229                                 continue;
10230                         }
10231                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10232                         bnxt_dbg_hwrm_ring_info_get(bp,
10233                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10234                                 fw_ring_id, &val[0], &val[1]);
10235                         cpr->missed_irqs++;
10236                 }
10237         }
10238 }
10239
10240 static void bnxt_cfg_ntp_filters(struct bnxt *);
10241
10242 static void bnxt_sp_task(struct work_struct *work)
10243 {
10244         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10245
10246         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10247         smp_mb__after_atomic();
10248         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10249                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10250                 return;
10251         }
10252
10253         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
10254                 bnxt_cfg_rx_mode(bp);
10255
10256         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
10257                 bnxt_cfg_ntp_filters(bp);
10258         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
10259                 bnxt_hwrm_exec_fwd_req(bp);
10260         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
10261                 bnxt_hwrm_tunnel_dst_port_alloc(
10262                         bp, bp->vxlan_port,
10263                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
10264         }
10265         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
10266                 bnxt_hwrm_tunnel_dst_port_free(
10267                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
10268         }
10269         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
10270                 bnxt_hwrm_tunnel_dst_port_alloc(
10271                         bp, bp->nge_port,
10272                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
10273         }
10274         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
10275                 bnxt_hwrm_tunnel_dst_port_free(
10276                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
10277         }
10278         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
10279                 bnxt_hwrm_port_qstats(bp);
10280                 bnxt_hwrm_port_qstats_ext(bp);
10281                 bnxt_hwrm_pcie_qstats(bp);
10282         }
10283
10284         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
10285                 int rc;
10286
10287                 mutex_lock(&bp->link_lock);
10288                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
10289                                        &bp->sp_event))
10290                         bnxt_hwrm_phy_qcaps(bp);
10291
10292                 rc = bnxt_update_link(bp, true);
10293                 mutex_unlock(&bp->link_lock);
10294                 if (rc)
10295                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
10296                                    rc);
10297         }
10298         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
10299                 int rc;
10300
10301                 mutex_lock(&bp->link_lock);
10302                 rc = bnxt_update_phy_setting(bp);
10303                 mutex_unlock(&bp->link_lock);
10304                 if (rc) {
10305                         netdev_warn(bp->dev, "update phy settings retry failed\n");
10306                 } else {
10307                         bp->link_info.phy_retry = false;
10308                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
10309                 }
10310         }
10311         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
10312                 mutex_lock(&bp->link_lock);
10313                 bnxt_get_port_module_status(bp);
10314                 mutex_unlock(&bp->link_lock);
10315         }
10316
10317         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10318                 bnxt_tc_flow_stats_work(bp);
10319
10320         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10321                 bnxt_chk_missed_irq(bp);
10322
10323         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
10324          * must be the last functions to be called before exiting.
10325          */
10326         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10327                 bnxt_reset(bp, false);
10328
10329         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10330                 bnxt_reset(bp, true);
10331
10332         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
10333                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
10334
10335         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
10336                 if (!is_bnxt_fw_ok(bp))
10337                         bnxt_devlink_health_report(bp,
10338                                                    BNXT_FW_EXCEPTION_SP_EVENT);
10339         }
10340
10341         smp_mb__before_atomic();
10342         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10343 }
10344
10345 /* Under rtnl_lock */
10346 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10347                      int tx_xdp)
10348 {
10349         int max_rx, max_tx, tx_sets = 1;
10350         int tx_rings_needed, stats;
10351         int rx_rings = rx;
10352         int cp, vnics, rc;
10353
10354         if (tcs)
10355                 tx_sets = tcs;
10356
10357         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10358         if (rc)
10359                 return rc;
10360
10361         if (max_rx < rx)
10362                 return -ENOMEM;
10363
10364         tx_rings_needed = tx * tx_sets + tx_xdp;
10365         if (max_tx < tx_rings_needed)
10366                 return -ENOMEM;
10367
10368         vnics = 1;
10369         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
10370                 vnics += rx_rings;
10371
10372         if (bp->flags & BNXT_FLAG_AGG_RINGS)
10373                 rx_rings <<= 1;
10374         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
10375         stats = cp;
10376         if (BNXT_NEW_RM(bp)) {
10377                 cp += bnxt_get_ulp_msix_num(bp);
10378                 stats += bnxt_get_ulp_stat_ctxs(bp);
10379         }
10380         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
10381                                      stats, vnics);
10382 }
10383
10384 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10385 {
10386         if (bp->bar2) {
10387                 pci_iounmap(pdev, bp->bar2);
10388                 bp->bar2 = NULL;
10389         }
10390
10391         if (bp->bar1) {
10392                 pci_iounmap(pdev, bp->bar1);
10393                 bp->bar1 = NULL;
10394         }
10395
10396         if (bp->bar0) {
10397                 pci_iounmap(pdev, bp->bar0);
10398                 bp->bar0 = NULL;
10399         }
10400 }
10401
10402 static void bnxt_cleanup_pci(struct bnxt *bp)
10403 {
10404         bnxt_unmap_bars(bp, bp->pdev);
10405         pci_release_regions(bp->pdev);
10406         if (pci_is_enabled(bp->pdev))
10407                 pci_disable_device(bp->pdev);
10408 }
10409
10410 static void bnxt_init_dflt_coal(struct bnxt *bp)
10411 {
10412         struct bnxt_coal *coal;
10413
10414         /* Tick values in micro seconds.
10415          * 1 coal_buf x bufs_per_record = 1 completion record.
10416          */
10417         coal = &bp->rx_coal;
10418         coal->coal_ticks = 10;
10419         coal->coal_bufs = 30;
10420         coal->coal_ticks_irq = 1;
10421         coal->coal_bufs_irq = 2;
10422         coal->idle_thresh = 50;
10423         coal->bufs_per_record = 2;
10424         coal->budget = 64;              /* NAPI budget */
10425
10426         coal = &bp->tx_coal;
10427         coal->coal_ticks = 28;
10428         coal->coal_bufs = 30;
10429         coal->coal_ticks_irq = 2;
10430         coal->coal_bufs_irq = 2;
10431         coal->bufs_per_record = 1;
10432
10433         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10434 }
10435
10436 static int bnxt_fw_init_one_p1(struct bnxt *bp)
10437 {
10438         int rc;
10439
10440         bp->fw_cap = 0;
10441         rc = bnxt_hwrm_ver_get(bp);
10442         if (rc)
10443                 return rc;
10444
10445         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10446                 rc = bnxt_alloc_kong_hwrm_resources(bp);
10447                 if (rc)
10448                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10449         }
10450
10451         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10452             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10453                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10454                 if (rc)
10455                         return rc;
10456         }
10457         rc = bnxt_hwrm_func_reset(bp);
10458         if (rc)
10459                 return -ENODEV;
10460
10461         bnxt_hwrm_fw_set_time(bp);
10462         return 0;
10463 }
10464
10465 static int bnxt_fw_init_one_p2(struct bnxt *bp)
10466 {
10467         int rc;
10468
10469         /* Get the MAX capabilities for this function */
10470         rc = bnxt_hwrm_func_qcaps(bp);
10471         if (rc) {
10472                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10473                            rc);
10474                 return -ENODEV;
10475         }
10476
10477         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10478         if (rc)
10479                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10480                             rc);
10481
10482         rc = bnxt_hwrm_error_recovery_qcfg(bp);
10483         if (rc)
10484                 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
10485                             rc);
10486
10487         rc = bnxt_hwrm_func_drv_rgtr(bp);
10488         if (rc)
10489                 return -ENODEV;
10490
10491         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
10492         if (rc)
10493                 return -ENODEV;
10494
10495         bnxt_hwrm_func_qcfg(bp);
10496         bnxt_hwrm_vnic_qcaps(bp);
10497         bnxt_hwrm_port_led_qcaps(bp);
10498         bnxt_ethtool_init(bp);
10499         bnxt_dcb_init(bp);
10500         return 0;
10501 }
10502
10503 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
10504 {
10505         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
10506         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10507                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10508                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10509                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10510         if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
10511                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10512                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10513                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10514         }
10515 }
10516
10517 static void bnxt_set_dflt_rfs(struct bnxt *bp)
10518 {
10519         struct net_device *dev = bp->dev;
10520
10521         dev->hw_features &= ~NETIF_F_NTUPLE;
10522         dev->features &= ~NETIF_F_NTUPLE;
10523         bp->flags &= ~BNXT_FLAG_RFS;
10524         if (bnxt_rfs_supported(bp)) {
10525                 dev->hw_features |= NETIF_F_NTUPLE;
10526                 if (bnxt_rfs_capable(bp)) {
10527                         bp->flags |= BNXT_FLAG_RFS;
10528                         dev->features |= NETIF_F_NTUPLE;
10529                 }
10530         }
10531 }
10532
10533 static void bnxt_fw_init_one_p3(struct bnxt *bp)
10534 {
10535         struct pci_dev *pdev = bp->pdev;
10536
10537         bnxt_set_dflt_rss_hash_type(bp);
10538         bnxt_set_dflt_rfs(bp);
10539
10540         bnxt_get_wol_settings(bp);
10541         if (bp->flags & BNXT_FLAG_WOL_CAP)
10542                 device_set_wakeup_enable(&pdev->dev, bp->wol);
10543         else
10544                 device_set_wakeup_capable(&pdev->dev, false);
10545
10546         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10547         bnxt_hwrm_coal_params_qcaps(bp);
10548 }
10549
10550 static int bnxt_fw_init_one(struct bnxt *bp)
10551 {
10552         int rc;
10553
10554         rc = bnxt_fw_init_one_p1(bp);
10555         if (rc) {
10556                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
10557                 return rc;
10558         }
10559         rc = bnxt_fw_init_one_p2(bp);
10560         if (rc) {
10561                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
10562                 return rc;
10563         }
10564         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
10565         if (rc)
10566                 return rc;
10567         bnxt_fw_init_one_p3(bp);
10568         return 0;
10569 }
10570
10571 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
10572 {
10573         struct bnxt_fw_health *fw_health = bp->fw_health;
10574         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
10575         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
10576         u32 reg_type, reg_off, delay_msecs;
10577
10578         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
10579         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
10580         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
10581         switch (reg_type) {
10582         case BNXT_FW_HEALTH_REG_TYPE_CFG:
10583                 pci_write_config_dword(bp->pdev, reg_off, val);
10584                 break;
10585         case BNXT_FW_HEALTH_REG_TYPE_GRC:
10586                 writel(reg_off & BNXT_GRC_BASE_MASK,
10587                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
10588                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
10589                 /* fall through */
10590         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
10591                 writel(val, bp->bar0 + reg_off);
10592                 break;
10593         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
10594                 writel(val, bp->bar1 + reg_off);
10595                 break;
10596         }
10597         if (delay_msecs) {
10598                 pci_read_config_dword(bp->pdev, 0, &val);
10599                 msleep(delay_msecs);
10600         }
10601 }
10602
10603 static void bnxt_reset_all(struct bnxt *bp)
10604 {
10605         struct bnxt_fw_health *fw_health = bp->fw_health;
10606         int i, rc;
10607
10608         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10609 #ifdef CONFIG_TEE_BNXT_FW
10610                 rc = tee_bnxt_fw_load();
10611                 if (rc)
10612                         netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc);
10613                 bp->fw_reset_timestamp = jiffies;
10614 #endif
10615                 return;
10616         }
10617
10618         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
10619                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
10620                         bnxt_fw_reset_writel(bp, i);
10621         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
10622                 struct hwrm_fw_reset_input req = {0};
10623
10624                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
10625                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
10626                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
10627                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
10628                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
10629                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10630                 if (rc)
10631                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
10632         }
10633         bp->fw_reset_timestamp = jiffies;
10634 }
10635
10636 static void bnxt_fw_reset_task(struct work_struct *work)
10637 {
10638         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
10639         int rc;
10640
10641         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10642                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
10643                 return;
10644         }
10645
10646         switch (bp->fw_reset_state) {
10647         case BNXT_FW_RESET_STATE_POLL_VF: {
10648                 int n = bnxt_get_registered_vfs(bp);
10649                 int tmo;
10650
10651                 if (n < 0) {
10652                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
10653                                    n, jiffies_to_msecs(jiffies -
10654                                    bp->fw_reset_timestamp));
10655                         goto fw_reset_abort;
10656                 } else if (n > 0) {
10657                         if (time_after(jiffies, bp->fw_reset_timestamp +
10658                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
10659                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10660                                 bp->fw_reset_state = 0;
10661                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
10662                                            n);
10663                                 return;
10664                         }
10665                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10666                         return;
10667                 }
10668                 bp->fw_reset_timestamp = jiffies;
10669                 rtnl_lock();
10670                 bnxt_fw_reset_close(bp);
10671                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10672                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10673                         tmo = HZ / 10;
10674                 } else {
10675                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10676                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10677                 }
10678                 rtnl_unlock();
10679                 bnxt_queue_fw_reset_work(bp, tmo);
10680                 return;
10681         }
10682         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
10683                 u32 val;
10684
10685                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
10686                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
10687                     !time_after(jiffies, bp->fw_reset_timestamp +
10688                     (bp->fw_reset_max_dsecs * HZ / 10))) {
10689                         bnxt_queue_fw_reset_work(bp, HZ / 5);
10690                         return;
10691                 }
10692
10693                 if (!bp->fw_health->master) {
10694                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
10695
10696                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10697                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10698                         return;
10699                 }
10700                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10701         }
10702         /* fall through */
10703         case BNXT_FW_RESET_STATE_RESET_FW:
10704                 bnxt_reset_all(bp);
10705                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10706                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
10707                 return;
10708         case BNXT_FW_RESET_STATE_ENABLE_DEV:
10709                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
10710                     bp->fw_health) {
10711                         u32 val;
10712
10713                         val = bnxt_fw_health_readl(bp,
10714                                                    BNXT_FW_RESET_INPROG_REG);
10715                         if (val)
10716                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
10717                                             val);
10718                 }
10719                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10720                 if (pci_enable_device(bp->pdev)) {
10721                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
10722                         goto fw_reset_abort;
10723                 }
10724                 pci_set_master(bp->pdev);
10725                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
10726                 /* fall through */
10727         case BNXT_FW_RESET_STATE_POLL_FW:
10728                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
10729                 rc = __bnxt_hwrm_ver_get(bp, true);
10730                 if (rc) {
10731                         if (time_after(jiffies, bp->fw_reset_timestamp +
10732                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
10733                                 netdev_err(bp->dev, "Firmware reset aborted\n");
10734                                 goto fw_reset_abort;
10735                         }
10736                         bnxt_queue_fw_reset_work(bp, HZ / 5);
10737                         return;
10738                 }
10739                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
10740                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
10741                 /* fall through */
10742         case BNXT_FW_RESET_STATE_OPENING:
10743                 while (!rtnl_trylock()) {
10744                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10745                         return;
10746                 }
10747                 rc = bnxt_open(bp->dev);
10748                 if (rc) {
10749                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
10750                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10751                         dev_close(bp->dev);
10752                 }
10753
10754                 bp->fw_reset_state = 0;
10755                 /* Make sure fw_reset_state is 0 before clearing the flag */
10756                 smp_mb__before_atomic();
10757                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10758                 bnxt_ulp_start(bp, rc);
10759                 bnxt_dl_health_status_update(bp, true);
10760                 rtnl_unlock();
10761                 break;
10762         }
10763         return;
10764
10765 fw_reset_abort:
10766         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10767         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
10768                 bnxt_dl_health_status_update(bp, false);
10769         bp->fw_reset_state = 0;
10770         rtnl_lock();
10771         dev_close(bp->dev);
10772         rtnl_unlock();
10773 }
10774
10775 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
10776 {
10777         int rc;
10778         struct bnxt *bp = netdev_priv(dev);
10779
10780         SET_NETDEV_DEV(dev, &pdev->dev);
10781
10782         /* enable device (incl. PCI PM wakeup), and bus-mastering */
10783         rc = pci_enable_device(pdev);
10784         if (rc) {
10785                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
10786                 goto init_err;
10787         }
10788
10789         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
10790                 dev_err(&pdev->dev,
10791                         "Cannot find PCI device base address, aborting\n");
10792                 rc = -ENODEV;
10793                 goto init_err_disable;
10794         }
10795
10796         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
10797         if (rc) {
10798                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
10799                 goto init_err_disable;
10800         }
10801
10802         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
10803             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
10804                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
10805                 goto init_err_disable;
10806         }
10807
10808         pci_set_master(pdev);
10809
10810         bp->dev = dev;
10811         bp->pdev = pdev;
10812
10813         bp->bar0 = pci_ioremap_bar(pdev, 0);
10814         if (!bp->bar0) {
10815                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
10816                 rc = -ENOMEM;
10817                 goto init_err_release;
10818         }
10819
10820         bp->bar1 = pci_ioremap_bar(pdev, 2);
10821         if (!bp->bar1) {
10822                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
10823                 rc = -ENOMEM;
10824                 goto init_err_release;
10825         }
10826
10827         bp->bar2 = pci_ioremap_bar(pdev, 4);
10828         if (!bp->bar2) {
10829                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
10830                 rc = -ENOMEM;
10831                 goto init_err_release;
10832         }
10833
10834         pci_enable_pcie_error_reporting(pdev);
10835
10836         INIT_WORK(&bp->sp_task, bnxt_sp_task);
10837         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
10838
10839         spin_lock_init(&bp->ntp_fltr_lock);
10840 #if BITS_PER_LONG == 32
10841         spin_lock_init(&bp->db_lock);
10842 #endif
10843
10844         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
10845         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
10846
10847         bnxt_init_dflt_coal(bp);
10848
10849         timer_setup(&bp->timer, bnxt_timer, 0);
10850         bp->current_interval = BNXT_TIMER_INTERVAL;
10851
10852         clear_bit(BNXT_STATE_OPEN, &bp->state);
10853         return 0;
10854
10855 init_err_release:
10856         bnxt_unmap_bars(bp, pdev);
10857         pci_release_regions(pdev);
10858
10859 init_err_disable:
10860         pci_disable_device(pdev);
10861
10862 init_err:
10863         return rc;
10864 }
10865
10866 /* rtnl_lock held */
10867 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
10868 {
10869         struct sockaddr *addr = p;
10870         struct bnxt *bp = netdev_priv(dev);
10871         int rc = 0;
10872
10873         if (!is_valid_ether_addr(addr->sa_data))
10874                 return -EADDRNOTAVAIL;
10875
10876         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
10877                 return 0;
10878
10879         rc = bnxt_approve_mac(bp, addr->sa_data, true);
10880         if (rc)
10881                 return rc;
10882
10883         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
10884         if (netif_running(dev)) {
10885                 bnxt_close_nic(bp, false, false);
10886                 rc = bnxt_open_nic(bp, false, false);
10887         }
10888
10889         return rc;
10890 }
10891
10892 /* rtnl_lock held */
10893 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
10894 {
10895         struct bnxt *bp = netdev_priv(dev);
10896
10897         if (netif_running(dev))
10898                 bnxt_close_nic(bp, false, false);
10899
10900         dev->mtu = new_mtu;
10901         bnxt_set_ring_params(bp);
10902
10903         if (netif_running(dev))
10904                 return bnxt_open_nic(bp, false, false);
10905
10906         return 0;
10907 }
10908
10909 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
10910 {
10911         struct bnxt *bp = netdev_priv(dev);
10912         bool sh = false;
10913         int rc;
10914
10915         if (tc > bp->max_tc) {
10916                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
10917                            tc, bp->max_tc);
10918                 return -EINVAL;
10919         }
10920
10921         if (netdev_get_num_tc(dev) == tc)
10922                 return 0;
10923
10924         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10925                 sh = true;
10926
10927         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
10928                               sh, tc, bp->tx_nr_rings_xdp);
10929         if (rc)
10930                 return rc;
10931
10932         /* Needs to close the device and do hw resource re-allocations */
10933         if (netif_running(bp->dev))
10934                 bnxt_close_nic(bp, true, false);
10935
10936         if (tc) {
10937                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
10938                 netdev_set_num_tc(dev, tc);
10939         } else {
10940                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
10941                 netdev_reset_tc(dev);
10942         }
10943         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
10944         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
10945                                bp->tx_nr_rings + bp->rx_nr_rings;
10946
10947         if (netif_running(bp->dev))
10948                 return bnxt_open_nic(bp, true, false);
10949
10950         return 0;
10951 }
10952
10953 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10954                                   void *cb_priv)
10955 {
10956         struct bnxt *bp = cb_priv;
10957
10958         if (!bnxt_tc_flower_enabled(bp) ||
10959             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
10960                 return -EOPNOTSUPP;
10961
10962         switch (type) {
10963         case TC_SETUP_CLSFLOWER:
10964                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
10965         default:
10966                 return -EOPNOTSUPP;
10967         }
10968 }
10969
10970 LIST_HEAD(bnxt_block_cb_list);
10971
10972 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
10973                          void *type_data)
10974 {
10975         struct bnxt *bp = netdev_priv(dev);
10976
10977         switch (type) {
10978         case TC_SETUP_BLOCK:
10979                 return flow_block_cb_setup_simple(type_data,
10980                                                   &bnxt_block_cb_list,
10981                                                   bnxt_setup_tc_block_cb,
10982                                                   bp, bp, true);
10983         case TC_SETUP_QDISC_MQPRIO: {
10984                 struct tc_mqprio_qopt *mqprio = type_data;
10985
10986                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
10987
10988                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
10989         }
10990         default:
10991                 return -EOPNOTSUPP;
10992         }
10993 }
10994
10995 #ifdef CONFIG_RFS_ACCEL
10996 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
10997                             struct bnxt_ntuple_filter *f2)
10998 {
10999         struct flow_keys *keys1 = &f1->fkeys;
11000         struct flow_keys *keys2 = &f2->fkeys;
11001
11002         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
11003             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
11004             keys1->ports.ports == keys2->ports.ports &&
11005             keys1->basic.ip_proto == keys2->basic.ip_proto &&
11006             keys1->basic.n_proto == keys2->basic.n_proto &&
11007             keys1->control.flags == keys2->control.flags &&
11008             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11009             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11010                 return true;
11011
11012         return false;
11013 }
11014
11015 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11016                               u16 rxq_index, u32 flow_id)
11017 {
11018         struct bnxt *bp = netdev_priv(dev);
11019         struct bnxt_ntuple_filter *fltr, *new_fltr;
11020         struct flow_keys *fkeys;
11021         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11022         int rc = 0, idx, bit_id, l2_idx = 0;
11023         struct hlist_head *head;
11024
11025         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11026                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11027                 int off = 0, j;
11028
11029                 netif_addr_lock_bh(dev);
11030                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11031                         if (ether_addr_equal(eth->h_dest,
11032                                              vnic->uc_list + off)) {
11033                                 l2_idx = j + 1;
11034                                 break;
11035                         }
11036                 }
11037                 netif_addr_unlock_bh(dev);
11038                 if (!l2_idx)
11039                         return -EINVAL;
11040         }
11041         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11042         if (!new_fltr)
11043                 return -ENOMEM;
11044
11045         fkeys = &new_fltr->fkeys;
11046         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11047                 rc = -EPROTONOSUPPORT;
11048                 goto err_free;
11049         }
11050
11051         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11052              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11053             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11054              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11055                 rc = -EPROTONOSUPPORT;
11056                 goto err_free;
11057         }
11058         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11059             bp->hwrm_spec_code < 0x10601) {
11060                 rc = -EPROTONOSUPPORT;
11061                 goto err_free;
11062         }
11063         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
11064             bp->hwrm_spec_code < 0x10601) {
11065                 rc = -EPROTONOSUPPORT;
11066                 goto err_free;
11067         }
11068
11069         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11070         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11071
11072         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11073         head = &bp->ntp_fltr_hash_tbl[idx];
11074         rcu_read_lock();
11075         hlist_for_each_entry_rcu(fltr, head, hash) {
11076                 if (bnxt_fltr_match(fltr, new_fltr)) {
11077                         rcu_read_unlock();
11078                         rc = 0;
11079                         goto err_free;
11080                 }
11081         }
11082         rcu_read_unlock();
11083
11084         spin_lock_bh(&bp->ntp_fltr_lock);
11085         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11086                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
11087         if (bit_id < 0) {
11088                 spin_unlock_bh(&bp->ntp_fltr_lock);
11089                 rc = -ENOMEM;
11090                 goto err_free;
11091         }
11092
11093         new_fltr->sw_id = (u16)bit_id;
11094         new_fltr->flow_id = flow_id;
11095         new_fltr->l2_fltr_idx = l2_idx;
11096         new_fltr->rxq = rxq_index;
11097         hlist_add_head_rcu(&new_fltr->hash, head);
11098         bp->ntp_fltr_count++;
11099         spin_unlock_bh(&bp->ntp_fltr_lock);
11100
11101         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11102         bnxt_queue_sp_work(bp);
11103
11104         return new_fltr->sw_id;
11105
11106 err_free:
11107         kfree(new_fltr);
11108         return rc;
11109 }
11110
11111 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11112 {
11113         int i;
11114
11115         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11116                 struct hlist_head *head;
11117                 struct hlist_node *tmp;
11118                 struct bnxt_ntuple_filter *fltr;
11119                 int rc;
11120
11121                 head = &bp->ntp_fltr_hash_tbl[i];
11122                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11123                         bool del = false;
11124
11125                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11126                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11127                                                         fltr->flow_id,
11128                                                         fltr->sw_id)) {
11129                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
11130                                                                          fltr);
11131                                         del = true;
11132                                 }
11133                         } else {
11134                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11135                                                                        fltr);
11136                                 if (rc)
11137                                         del = true;
11138                                 else
11139                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
11140                         }
11141
11142                         if (del) {
11143                                 spin_lock_bh(&bp->ntp_fltr_lock);
11144                                 hlist_del_rcu(&fltr->hash);
11145                                 bp->ntp_fltr_count--;
11146                                 spin_unlock_bh(&bp->ntp_fltr_lock);
11147                                 synchronize_rcu();
11148                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11149                                 kfree(fltr);
11150                         }
11151                 }
11152         }
11153         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11154                 netdev_info(bp->dev, "Receive PF driver unload event!");
11155 }
11156
11157 #else
11158
11159 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11160 {
11161 }
11162
11163 #endif /* CONFIG_RFS_ACCEL */
11164
11165 static void bnxt_udp_tunnel_add(struct net_device *dev,
11166                                 struct udp_tunnel_info *ti)
11167 {
11168         struct bnxt *bp = netdev_priv(dev);
11169
11170         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
11171                 return;
11172
11173         if (!netif_running(dev))
11174                 return;
11175
11176         switch (ti->type) {
11177         case UDP_TUNNEL_TYPE_VXLAN:
11178                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
11179                         return;
11180
11181                 bp->vxlan_port_cnt++;
11182                 if (bp->vxlan_port_cnt == 1) {
11183                         bp->vxlan_port = ti->port;
11184                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
11185                         bnxt_queue_sp_work(bp);
11186                 }
11187                 break;
11188         case UDP_TUNNEL_TYPE_GENEVE:
11189                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
11190                         return;
11191
11192                 bp->nge_port_cnt++;
11193                 if (bp->nge_port_cnt == 1) {
11194                         bp->nge_port = ti->port;
11195                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
11196                 }
11197                 break;
11198         default:
11199                 return;
11200         }
11201
11202         bnxt_queue_sp_work(bp);
11203 }
11204
11205 static void bnxt_udp_tunnel_del(struct net_device *dev,
11206                                 struct udp_tunnel_info *ti)
11207 {
11208         struct bnxt *bp = netdev_priv(dev);
11209
11210         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
11211                 return;
11212
11213         if (!netif_running(dev))
11214                 return;
11215
11216         switch (ti->type) {
11217         case UDP_TUNNEL_TYPE_VXLAN:
11218                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
11219                         return;
11220                 bp->vxlan_port_cnt--;
11221
11222                 if (bp->vxlan_port_cnt != 0)
11223                         return;
11224
11225                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
11226                 break;
11227         case UDP_TUNNEL_TYPE_GENEVE:
11228                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
11229                         return;
11230                 bp->nge_port_cnt--;
11231
11232                 if (bp->nge_port_cnt != 0)
11233                         return;
11234
11235                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
11236                 break;
11237         default:
11238                 return;
11239         }
11240
11241         bnxt_queue_sp_work(bp);
11242 }
11243
11244 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11245                                struct net_device *dev, u32 filter_mask,
11246                                int nlflags)
11247 {
11248         struct bnxt *bp = netdev_priv(dev);
11249
11250         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11251                                        nlflags, filter_mask, NULL);
11252 }
11253
11254 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
11255                                u16 flags, struct netlink_ext_ack *extack)
11256 {
11257         struct bnxt *bp = netdev_priv(dev);
11258         struct nlattr *attr, *br_spec;
11259         int rem, rc = 0;
11260
11261         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
11262                 return -EOPNOTSUPP;
11263
11264         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11265         if (!br_spec)
11266                 return -EINVAL;
11267
11268         nla_for_each_nested(attr, br_spec, rem) {
11269                 u16 mode;
11270
11271                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11272                         continue;
11273
11274                 if (nla_len(attr) < sizeof(mode))
11275                         return -EINVAL;
11276
11277                 mode = nla_get_u16(attr);
11278                 if (mode == bp->br_mode)
11279                         break;
11280
11281                 rc = bnxt_hwrm_set_br_mode(bp, mode);
11282                 if (!rc)
11283                         bp->br_mode = mode;
11284                 break;
11285         }
11286         return rc;
11287 }
11288
11289 int bnxt_get_port_parent_id(struct net_device *dev,
11290                             struct netdev_phys_item_id *ppid)
11291 {
11292         struct bnxt *bp = netdev_priv(dev);
11293
11294         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
11295                 return -EOPNOTSUPP;
11296
11297         /* The PF and it's VF-reps only support the switchdev framework */
11298         if (!BNXT_PF(bp))
11299                 return -EOPNOTSUPP;
11300
11301         ppid->id_len = sizeof(bp->switch_id);
11302         memcpy(ppid->id, bp->switch_id, ppid->id_len);
11303
11304         return 0;
11305 }
11306
11307 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
11308 {
11309         struct bnxt *bp = netdev_priv(dev);
11310
11311         return &bp->dl_port;
11312 }
11313
11314 static const struct net_device_ops bnxt_netdev_ops = {
11315         .ndo_open               = bnxt_open,
11316         .ndo_start_xmit         = bnxt_start_xmit,
11317         .ndo_stop               = bnxt_close,
11318         .ndo_get_stats64        = bnxt_get_stats64,
11319         .ndo_set_rx_mode        = bnxt_set_rx_mode,
11320         .ndo_do_ioctl           = bnxt_ioctl,
11321         .ndo_validate_addr      = eth_validate_addr,
11322         .ndo_set_mac_address    = bnxt_change_mac_addr,
11323         .ndo_change_mtu         = bnxt_change_mtu,
11324         .ndo_fix_features       = bnxt_fix_features,
11325         .ndo_set_features       = bnxt_set_features,
11326         .ndo_tx_timeout         = bnxt_tx_timeout,
11327 #ifdef CONFIG_BNXT_SRIOV
11328         .ndo_get_vf_config      = bnxt_get_vf_config,
11329         .ndo_set_vf_mac         = bnxt_set_vf_mac,
11330         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
11331         .ndo_set_vf_rate        = bnxt_set_vf_bw,
11332         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
11333         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
11334         .ndo_set_vf_trust       = bnxt_set_vf_trust,
11335 #endif
11336         .ndo_setup_tc           = bnxt_setup_tc,
11337 #ifdef CONFIG_RFS_ACCEL
11338         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
11339 #endif
11340         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
11341         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
11342         .ndo_bpf                = bnxt_xdp,
11343         .ndo_xdp_xmit           = bnxt_xdp_xmit,
11344         .ndo_bridge_getlink     = bnxt_bridge_getlink,
11345         .ndo_bridge_setlink     = bnxt_bridge_setlink,
11346         .ndo_get_devlink_port   = bnxt_get_devlink_port,
11347 };
11348
11349 static void bnxt_remove_one(struct pci_dev *pdev)
11350 {
11351         struct net_device *dev = pci_get_drvdata(pdev);
11352         struct bnxt *bp = netdev_priv(dev);
11353
11354         if (BNXT_PF(bp)) {
11355                 bnxt_sriov_disable(bp);
11356                 bnxt_dl_unregister(bp);
11357         }
11358
11359         pci_disable_pcie_error_reporting(pdev);
11360         unregister_netdev(dev);
11361         bnxt_shutdown_tc(bp);
11362         bnxt_cancel_sp_work(bp);
11363         bp->sp_event = 0;
11364
11365         bnxt_clear_int_mode(bp);
11366         bnxt_hwrm_func_drv_unrgtr(bp);
11367         bnxt_free_hwrm_resources(bp);
11368         bnxt_free_hwrm_short_cmd_req(bp);
11369         bnxt_ethtool_free(bp);
11370         bnxt_dcb_free(bp);
11371         kfree(bp->edev);
11372         bp->edev = NULL;
11373         bnxt_cleanup_pci(bp);
11374         bnxt_free_ctx_mem(bp);
11375         kfree(bp->ctx);
11376         bp->ctx = NULL;
11377         bnxt_free_port_stats(bp);
11378         free_netdev(dev);
11379 }
11380
11381 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
11382 {
11383         int rc = 0;
11384         struct bnxt_link_info *link_info = &bp->link_info;
11385
11386         rc = bnxt_hwrm_phy_qcaps(bp);
11387         if (rc) {
11388                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
11389                            rc);
11390                 return rc;
11391         }
11392         rc = bnxt_update_link(bp, false);
11393         if (rc) {
11394                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
11395                            rc);
11396                 return rc;
11397         }
11398
11399         /* Older firmware does not have supported_auto_speeds, so assume
11400          * that all supported speeds can be autonegotiated.
11401          */
11402         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
11403                 link_info->support_auto_speeds = link_info->support_speeds;
11404
11405         if (!fw_dflt)
11406                 return 0;
11407
11408         /*initialize the ethool setting copy with NVM settings */
11409         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11410                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11411                 if (bp->hwrm_spec_code >= 0x10201) {
11412                         if (link_info->auto_pause_setting &
11413                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11414                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11415                 } else {
11416                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11417                 }
11418                 link_info->advertising = link_info->auto_link_speeds;
11419         } else {
11420                 link_info->req_link_speed = link_info->force_link_speed;
11421                 link_info->req_duplex = link_info->duplex_setting;
11422         }
11423         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
11424                 link_info->req_flow_ctrl =
11425                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
11426         else
11427                 link_info->req_flow_ctrl = link_info->force_pause_setting;
11428         return 0;
11429 }
11430
11431 static int bnxt_get_max_irq(struct pci_dev *pdev)
11432 {
11433         u16 ctrl;
11434
11435         if (!pdev->msix_cap)
11436                 return 1;
11437
11438         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
11439         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
11440 }
11441
11442 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11443                                 int *max_cp)
11444 {
11445         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11446         int max_ring_grps = 0, max_irq;
11447
11448         *max_tx = hw_resc->max_tx_rings;
11449         *max_rx = hw_resc->max_rx_rings;
11450         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
11451         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
11452                         bnxt_get_ulp_msix_num(bp),
11453                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
11454         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11455                 *max_cp = min_t(int, *max_cp, max_irq);
11456         max_ring_grps = hw_resc->max_hw_ring_grps;
11457         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
11458                 *max_cp -= 1;
11459                 *max_rx -= 2;
11460         }
11461         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11462                 *max_rx >>= 1;
11463         if (bp->flags & BNXT_FLAG_CHIP_P5) {
11464                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
11465                 /* On P5 chips, max_cp output param should be available NQs */
11466                 *max_cp = max_irq;
11467         }
11468         *max_rx = min_t(int, *max_rx, max_ring_grps);
11469 }
11470
11471 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
11472 {
11473         int rx, tx, cp;
11474
11475         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
11476         *max_rx = rx;
11477         *max_tx = tx;
11478         if (!rx || !tx || !cp)
11479                 return -ENOMEM;
11480
11481         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
11482 }
11483
11484 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11485                                bool shared)
11486 {
11487         int rc;
11488
11489         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11490         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
11491                 /* Not enough rings, try disabling agg rings. */
11492                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
11493                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11494                 if (rc) {
11495                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
11496                         bp->flags |= BNXT_FLAG_AGG_RINGS;
11497                         return rc;
11498                 }
11499                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
11500                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11501                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11502                 bnxt_set_ring_params(bp);
11503         }
11504
11505         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
11506                 int max_cp, max_stat, max_irq;
11507
11508                 /* Reserve minimum resources for RoCE */
11509                 max_cp = bnxt_get_max_func_cp_rings(bp);
11510                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
11511                 max_irq = bnxt_get_max_func_irqs(bp);
11512                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
11513                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
11514                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
11515                         return 0;
11516
11517                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
11518                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
11519                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
11520                 max_cp = min_t(int, max_cp, max_irq);
11521                 max_cp = min_t(int, max_cp, max_stat);
11522                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
11523                 if (rc)
11524                         rc = 0;
11525         }
11526         return rc;
11527 }
11528
11529 /* In initial default shared ring setting, each shared ring must have a
11530  * RX/TX ring pair.
11531  */
11532 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
11533 {
11534         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
11535         bp->rx_nr_rings = bp->cp_nr_rings;
11536         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
11537         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11538 }
11539
11540 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
11541 {
11542         int dflt_rings, max_rx_rings, max_tx_rings, rc;
11543
11544         if (!bnxt_can_reserve_rings(bp))
11545                 return 0;
11546
11547         if (sh)
11548                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
11549         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
11550         /* Reduce default rings on multi-port cards so that total default
11551          * rings do not exceed CPU count.
11552          */
11553         if (bp->port_count > 1) {
11554                 int max_rings =
11555                         max_t(int, num_online_cpus() / bp->port_count, 1);
11556
11557                 dflt_rings = min_t(int, dflt_rings, max_rings);
11558         }
11559         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
11560         if (rc)
11561                 return rc;
11562         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
11563         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
11564         if (sh)
11565                 bnxt_trim_dflt_sh_rings(bp);
11566         else
11567                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
11568         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11569
11570         rc = __bnxt_reserve_rings(bp);
11571         if (rc)
11572                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
11573         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11574         if (sh)
11575                 bnxt_trim_dflt_sh_rings(bp);
11576
11577         /* Rings may have been trimmed, re-reserve the trimmed rings. */
11578         if (bnxt_need_reserve_rings(bp)) {
11579                 rc = __bnxt_reserve_rings(bp);
11580                 if (rc)
11581                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
11582                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11583         }
11584         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11585                 bp->rx_nr_rings++;
11586                 bp->cp_nr_rings++;
11587         }
11588         return rc;
11589 }
11590
11591 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
11592 {
11593         int rc;
11594
11595         if (bp->tx_nr_rings)
11596                 return 0;
11597
11598         bnxt_ulp_irq_stop(bp);
11599         bnxt_clear_int_mode(bp);
11600         rc = bnxt_set_dflt_rings(bp, true);
11601         if (rc) {
11602                 netdev_err(bp->dev, "Not enough rings available.\n");
11603                 goto init_dflt_ring_err;
11604         }
11605         rc = bnxt_init_int_mode(bp);
11606         if (rc)
11607                 goto init_dflt_ring_err;
11608
11609         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11610         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
11611                 bp->flags |= BNXT_FLAG_RFS;
11612                 bp->dev->features |= NETIF_F_NTUPLE;
11613         }
11614 init_dflt_ring_err:
11615         bnxt_ulp_irq_restart(bp, rc);
11616         return rc;
11617 }
11618
11619 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
11620 {
11621         int rc;
11622
11623         ASSERT_RTNL();
11624         bnxt_hwrm_func_qcaps(bp);
11625
11626         if (netif_running(bp->dev))
11627                 __bnxt_close_nic(bp, true, false);
11628
11629         bnxt_ulp_irq_stop(bp);
11630         bnxt_clear_int_mode(bp);
11631         rc = bnxt_init_int_mode(bp);
11632         bnxt_ulp_irq_restart(bp, rc);
11633
11634         if (netif_running(bp->dev)) {
11635                 if (rc)
11636                         dev_close(bp->dev);
11637                 else
11638                         rc = bnxt_open_nic(bp, true, false);
11639         }
11640
11641         return rc;
11642 }
11643
11644 static int bnxt_init_mac_addr(struct bnxt *bp)
11645 {
11646         int rc = 0;
11647
11648         if (BNXT_PF(bp)) {
11649                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
11650         } else {
11651 #ifdef CONFIG_BNXT_SRIOV
11652                 struct bnxt_vf_info *vf = &bp->vf;
11653                 bool strict_approval = true;
11654
11655                 if (is_valid_ether_addr(vf->mac_addr)) {
11656                         /* overwrite netdev dev_addr with admin VF MAC */
11657                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
11658                         /* Older PF driver or firmware may not approve this
11659                          * correctly.
11660                          */
11661                         strict_approval = false;
11662                 } else {
11663                         eth_hw_addr_random(bp->dev);
11664                 }
11665                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
11666 #endif
11667         }
11668         return rc;
11669 }
11670
11671 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
11672 {
11673         struct pci_dev *pdev = bp->pdev;
11674         int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
11675         u32 dw;
11676
11677         if (!pos) {
11678                 netdev_info(bp->dev, "Unable do read adapter's DSN");
11679                 return -EOPNOTSUPP;
11680         }
11681
11682         /* DSN (two dw) is at an offset of 4 from the cap pos */
11683         pos += 4;
11684         pci_read_config_dword(pdev, pos, &dw);
11685         put_unaligned_le32(dw, &dsn[0]);
11686         pci_read_config_dword(pdev, pos + 4, &dw);
11687         put_unaligned_le32(dw, &dsn[4]);
11688         return 0;
11689 }
11690
11691 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
11692 {
11693         static int version_printed;
11694         struct net_device *dev;
11695         struct bnxt *bp;
11696         int rc, max_irqs;
11697
11698         if (pci_is_bridge(pdev))
11699                 return -ENODEV;
11700
11701         if (version_printed++ == 0)
11702                 pr_info("%s", version);
11703
11704         max_irqs = bnxt_get_max_irq(pdev);
11705         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
11706         if (!dev)
11707                 return -ENOMEM;
11708
11709         bp = netdev_priv(dev);
11710         bnxt_set_max_func_irqs(bp, max_irqs);
11711
11712         if (bnxt_vf_pciid(ent->driver_data))
11713                 bp->flags |= BNXT_FLAG_VF;
11714
11715         if (pdev->msix_cap)
11716                 bp->flags |= BNXT_FLAG_MSIX_CAP;
11717
11718         rc = bnxt_init_board(pdev, dev);
11719         if (rc < 0)
11720                 goto init_err_free;
11721
11722         dev->netdev_ops = &bnxt_netdev_ops;
11723         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
11724         dev->ethtool_ops = &bnxt_ethtool_ops;
11725         pci_set_drvdata(pdev, dev);
11726
11727         rc = bnxt_alloc_hwrm_resources(bp);
11728         if (rc)
11729                 goto init_err_pci_clean;
11730
11731         mutex_init(&bp->hwrm_cmd_lock);
11732         mutex_init(&bp->link_lock);
11733
11734         rc = bnxt_fw_init_one_p1(bp);
11735         if (rc)
11736                 goto init_err_pci_clean;
11737
11738         if (BNXT_CHIP_P5(bp))
11739                 bp->flags |= BNXT_FLAG_CHIP_P5;
11740
11741         rc = bnxt_fw_init_one_p2(bp);
11742         if (rc)
11743                 goto init_err_pci_clean;
11744
11745         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11746                            NETIF_F_TSO | NETIF_F_TSO6 |
11747                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
11748                            NETIF_F_GSO_IPXIP4 |
11749                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
11750                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
11751                            NETIF_F_RXCSUM | NETIF_F_GRO;
11752
11753         if (BNXT_SUPPORTS_TPA(bp))
11754                 dev->hw_features |= NETIF_F_LRO;
11755
11756         dev->hw_enc_features =
11757                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11758                         NETIF_F_TSO | NETIF_F_TSO6 |
11759                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
11760                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
11761                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
11762         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
11763                                     NETIF_F_GSO_GRE_CSUM;
11764         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
11765         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
11766                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
11767         if (BNXT_SUPPORTS_TPA(bp))
11768                 dev->hw_features |= NETIF_F_GRO_HW;
11769         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
11770         if (dev->features & NETIF_F_GRO_HW)
11771                 dev->features &= ~NETIF_F_LRO;
11772         dev->priv_flags |= IFF_UNICAST_FLT;
11773
11774 #ifdef CONFIG_BNXT_SRIOV
11775         init_waitqueue_head(&bp->sriov_cfg_wait);
11776         mutex_init(&bp->sriov_lock);
11777 #endif
11778         if (BNXT_SUPPORTS_TPA(bp)) {
11779                 bp->gro_func = bnxt_gro_func_5730x;
11780                 if (BNXT_CHIP_P4(bp))
11781                         bp->gro_func = bnxt_gro_func_5731x;
11782                 else if (BNXT_CHIP_P5(bp))
11783                         bp->gro_func = bnxt_gro_func_5750x;
11784         }
11785         if (!BNXT_CHIP_P4_PLUS(bp))
11786                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
11787
11788         bp->ulp_probe = bnxt_ulp_probe;
11789
11790         rc = bnxt_init_mac_addr(bp);
11791         if (rc) {
11792                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
11793                 rc = -EADDRNOTAVAIL;
11794                 goto init_err_pci_clean;
11795         }
11796
11797         if (BNXT_PF(bp)) {
11798                 /* Read the adapter's DSN to use as the eswitch switch_id */
11799                 rc = bnxt_pcie_dsn_get(bp, bp->switch_id);
11800                 if (rc)
11801                         goto init_err_pci_clean;
11802         }
11803
11804         /* MTU range: 60 - FW defined max */
11805         dev->min_mtu = ETH_ZLEN;
11806         dev->max_mtu = bp->max_mtu;
11807
11808         rc = bnxt_probe_phy(bp, true);
11809         if (rc)
11810                 goto init_err_pci_clean;
11811
11812         bnxt_set_rx_skb_mode(bp, false);
11813         bnxt_set_tpa_flags(bp);
11814         bnxt_set_ring_params(bp);
11815         rc = bnxt_set_dflt_rings(bp, true);
11816         if (rc) {
11817                 netdev_err(bp->dev, "Not enough rings available.\n");
11818                 rc = -ENOMEM;
11819                 goto init_err_pci_clean;
11820         }
11821
11822         bnxt_fw_init_one_p3(bp);
11823
11824         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
11825                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
11826
11827         rc = bnxt_init_int_mode(bp);
11828         if (rc)
11829                 goto init_err_pci_clean;
11830
11831         /* No TC has been set yet and rings may have been trimmed due to
11832          * limited MSIX, so we re-initialize the TX rings per TC.
11833          */
11834         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11835
11836         if (BNXT_PF(bp)) {
11837                 if (!bnxt_pf_wq) {
11838                         bnxt_pf_wq =
11839                                 create_singlethread_workqueue("bnxt_pf_wq");
11840                         if (!bnxt_pf_wq) {
11841                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
11842                                 goto init_err_pci_clean;
11843                         }
11844                 }
11845                 bnxt_init_tc(bp);
11846         }
11847
11848         rc = register_netdev(dev);
11849         if (rc)
11850                 goto init_err_cleanup_tc;
11851
11852         if (BNXT_PF(bp))
11853                 bnxt_dl_register(bp);
11854
11855         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
11856                     board_info[ent->driver_data].name,
11857                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
11858         pcie_print_link_status(pdev);
11859
11860         return 0;
11861
11862 init_err_cleanup_tc:
11863         bnxt_shutdown_tc(bp);
11864         bnxt_clear_int_mode(bp);
11865
11866 init_err_pci_clean:
11867         bnxt_free_hwrm_short_cmd_req(bp);
11868         bnxt_free_hwrm_resources(bp);
11869         bnxt_free_ctx_mem(bp);
11870         kfree(bp->ctx);
11871         bp->ctx = NULL;
11872         kfree(bp->fw_health);
11873         bp->fw_health = NULL;
11874         bnxt_cleanup_pci(bp);
11875
11876 init_err_free:
11877         free_netdev(dev);
11878         return rc;
11879 }
11880
11881 static void bnxt_shutdown(struct pci_dev *pdev)
11882 {
11883         struct net_device *dev = pci_get_drvdata(pdev);
11884         struct bnxt *bp;
11885
11886         if (!dev)
11887                 return;
11888
11889         rtnl_lock();
11890         bp = netdev_priv(dev);
11891         if (!bp)
11892                 goto shutdown_exit;
11893
11894         if (netif_running(dev))
11895                 dev_close(dev);
11896
11897         bnxt_ulp_shutdown(bp);
11898
11899         if (system_state == SYSTEM_POWER_OFF) {
11900                 bnxt_clear_int_mode(bp);
11901                 pci_disable_device(pdev);
11902                 pci_wake_from_d3(pdev, bp->wol);
11903                 pci_set_power_state(pdev, PCI_D3hot);
11904         }
11905
11906 shutdown_exit:
11907         rtnl_unlock();
11908 }
11909
11910 #ifdef CONFIG_PM_SLEEP
11911 static int bnxt_suspend(struct device *device)
11912 {
11913         struct net_device *dev = dev_get_drvdata(device);
11914         struct bnxt *bp = netdev_priv(dev);
11915         int rc = 0;
11916
11917         rtnl_lock();
11918         bnxt_ulp_stop(bp);
11919         if (netif_running(dev)) {
11920                 netif_device_detach(dev);
11921                 rc = bnxt_close(dev);
11922         }
11923         bnxt_hwrm_func_drv_unrgtr(bp);
11924         pci_disable_device(bp->pdev);
11925         rtnl_unlock();
11926         return rc;
11927 }
11928
11929 static int bnxt_resume(struct device *device)
11930 {
11931         struct net_device *dev = dev_get_drvdata(device);
11932         struct bnxt *bp = netdev_priv(dev);
11933         int rc = 0;
11934
11935         rtnl_lock();
11936         rc = pci_enable_device(bp->pdev);
11937         if (rc) {
11938                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
11939                            rc);
11940                 goto resume_exit;
11941         }
11942         pci_set_master(bp->pdev);
11943         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
11944                 rc = -ENODEV;
11945                 goto resume_exit;
11946         }
11947         rc = bnxt_hwrm_func_reset(bp);
11948         if (rc) {
11949                 rc = -EBUSY;
11950                 goto resume_exit;
11951         }
11952         bnxt_get_wol_settings(bp);
11953         if (netif_running(dev)) {
11954                 rc = bnxt_open(dev);
11955                 if (!rc)
11956                         netif_device_attach(dev);
11957         }
11958
11959 resume_exit:
11960         bnxt_ulp_start(bp, rc);
11961         rtnl_unlock();
11962         return rc;
11963 }
11964
11965 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
11966 #define BNXT_PM_OPS (&bnxt_pm_ops)
11967
11968 #else
11969
11970 #define BNXT_PM_OPS NULL
11971
11972 #endif /* CONFIG_PM_SLEEP */
11973
11974 /**
11975  * bnxt_io_error_detected - called when PCI error is detected
11976  * @pdev: Pointer to PCI device
11977  * @state: The current pci connection state
11978  *
11979  * This function is called after a PCI bus error affecting
11980  * this device has been detected.
11981  */
11982 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
11983                                                pci_channel_state_t state)
11984 {
11985         struct net_device *netdev = pci_get_drvdata(pdev);
11986         struct bnxt *bp = netdev_priv(netdev);
11987
11988         netdev_info(netdev, "PCI I/O error detected\n");
11989
11990         rtnl_lock();
11991         netif_device_detach(netdev);
11992
11993         bnxt_ulp_stop(bp);
11994
11995         if (state == pci_channel_io_perm_failure) {
11996                 rtnl_unlock();
11997                 return PCI_ERS_RESULT_DISCONNECT;
11998         }
11999
12000         if (netif_running(netdev))
12001                 bnxt_close(netdev);
12002
12003         pci_disable_device(pdev);
12004         rtnl_unlock();
12005
12006         /* Request a slot slot reset. */
12007         return PCI_ERS_RESULT_NEED_RESET;
12008 }
12009
12010 /**
12011  * bnxt_io_slot_reset - called after the pci bus has been reset.
12012  * @pdev: Pointer to PCI device
12013  *
12014  * Restart the card from scratch, as if from a cold-boot.
12015  * At this point, the card has exprienced a hard reset,
12016  * followed by fixups by BIOS, and has its config space
12017  * set up identically to what it was at cold boot.
12018  */
12019 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12020 {
12021         struct net_device *netdev = pci_get_drvdata(pdev);
12022         struct bnxt *bp = netdev_priv(netdev);
12023         int err = 0;
12024         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12025
12026         netdev_info(bp->dev, "PCI Slot Reset\n");
12027
12028         rtnl_lock();
12029
12030         if (pci_enable_device(pdev)) {
12031                 dev_err(&pdev->dev,
12032                         "Cannot re-enable PCI device after reset.\n");
12033         } else {
12034                 pci_set_master(pdev);
12035
12036                 err = bnxt_hwrm_func_reset(bp);
12037                 if (!err && netif_running(netdev))
12038                         err = bnxt_open(netdev);
12039
12040                 if (!err)
12041                         result = PCI_ERS_RESULT_RECOVERED;
12042                 bnxt_ulp_start(bp, err);
12043         }
12044
12045         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
12046                 dev_close(netdev);
12047
12048         rtnl_unlock();
12049
12050         return PCI_ERS_RESULT_RECOVERED;
12051 }
12052
12053 /**
12054  * bnxt_io_resume - called when traffic can start flowing again.
12055  * @pdev: Pointer to PCI device
12056  *
12057  * This callback is called when the error recovery driver tells
12058  * us that its OK to resume normal operation.
12059  */
12060 static void bnxt_io_resume(struct pci_dev *pdev)
12061 {
12062         struct net_device *netdev = pci_get_drvdata(pdev);
12063
12064         rtnl_lock();
12065
12066         netif_device_attach(netdev);
12067
12068         rtnl_unlock();
12069 }
12070
12071 static const struct pci_error_handlers bnxt_err_handler = {
12072         .error_detected = bnxt_io_error_detected,
12073         .slot_reset     = bnxt_io_slot_reset,
12074         .resume         = bnxt_io_resume
12075 };
12076
12077 static struct pci_driver bnxt_pci_driver = {
12078         .name           = DRV_MODULE_NAME,
12079         .id_table       = bnxt_pci_tbl,
12080         .probe          = bnxt_init_one,
12081         .remove         = bnxt_remove_one,
12082         .shutdown       = bnxt_shutdown,
12083         .driver.pm      = BNXT_PM_OPS,
12084         .err_handler    = &bnxt_err_handler,
12085 #if defined(CONFIG_BNXT_SRIOV)
12086         .sriov_configure = bnxt_sriov_configure,
12087 #endif
12088 };
12089
12090 static int __init bnxt_init(void)
12091 {
12092         bnxt_debug_init();
12093         return pci_register_driver(&bnxt_pci_driver);
12094 }
12095
12096 static void __exit bnxt_exit(void)
12097 {
12098         pci_unregister_driver(&bnxt_pci_driver);
12099         if (bnxt_pf_wq)
12100                 destroy_workqueue(bnxt_pf_wq);
12101         bnxt_debug_exit();
12102 }
12103
12104 module_init(bnxt_init);
12105 module_exit(bnxt_exit);