drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/ll_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56
  57         /* Copy the NAPI object as it has been already initialized */
  58         from_fp->napi = to_fp->napi;
  59
  60         /* Move bnx2x_fastpath contents */
  61         memcpy(to_fp, from_fp, sizeof(*to_fp));
  62         to_fp->index = to;
  63
  64         /* move sp_objs contents as well, as their indices match fp ones */
  65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  66
  67         /* move fp_stats contents as well, as their indices match fp ones */
  68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  69
  70         /* Update txdata pointers in fp and move txdata content accordingly:
  71          * Each fp consumes 'max_cos' txdata structures, so the index should be
  72          * decremented by max_cos x delta.
  73          */
  74
  75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  77                                 (bp)->max_cos;
  78         if (from == FCOE_IDX(bp)) {
  79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  81         }
  82
  83         memcpy(&bp->bnx2x_txq[new_txdata_index],
  84                &bp->bnx2x_txq[old_txdata_index],
  85                sizeof(struct bnx2x_fp_txdata));
  86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  87 }
  88
  89 /**
  90  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  91  *
  92  * @bp:        driver handle
  93  * @buf:       character buffer to fill with the fw name
  94  * @buf_len:   length of the above buffer
  95  *
  96  */
  97 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
  98 {
  99         if (IS_PF(bp)) {
 100                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 101
 102                 phy_fw_ver[0] = '\0';
 103                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 104                                              phy_fw_ver, PHY_FW_VER_LEN);
 105                 strlcpy(buf, bp->fw_ver, buf_len);
 106                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 107                          "bc %d.%d.%d%s%s",
 108                          (bp->common.bc_ver & 0xff0000) >> 16,
 109                          (bp->common.bc_ver & 0xff00) >> 8,
 110                          (bp->common.bc_ver & 0xff),
 111                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 112         } else {
 113                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 114         }
 115 }
 116
 117 /**
 118  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 119  *
 120  * @bp: driver handle
 121  * @delta:      number of eth queues which were not allocated
 122  */
 123 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 124 {
 125         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 126
 127         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 128          * backward along the array could cause memory to be overridden
 129          */
 130         for (cos = 1; cos < bp->max_cos; cos++) {
 131                 for (i = 0; i < old_eth_num - delta; i++) {
 132                         struct bnx2x_fastpath *fp = &bp->fp[i];
 133                         int new_idx = cos * (old_eth_num - delta) + i;
 134
 135                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 136                                sizeof(struct bnx2x_fp_txdata));
 137                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 138                 }
 139         }
 140 }
 141
 142 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 143
 144 /* free skb in the packet ring at pos idx
 145  * return idx of last bd freed
 146  */
 147 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 148                              u16 idx, unsigned int *pkts_compl,
 149                              unsigned int *bytes_compl)
 150 {
 151         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 152         struct eth_tx_start_bd *tx_start_bd;
 153         struct eth_tx_bd *tx_data_bd;
 154         struct sk_buff *skb = tx_buf->skb;
 155         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 156         int nbd;
 157
 158         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 159         prefetch(&skb->end);
 160
 161         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 162            txdata->txq_index, idx, tx_buf, skb);
 163
 164         /* unmap first bd */
 165         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 166         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 167                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 168
 169         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 170 #ifdef BNX2X_STOP_ON_ERROR
 171         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 172                 BNX2X_ERR("BAD nbd!\n");
 173                 bnx2x_panic();
 174         }
 175 #endif
 176         new_cons = nbd + tx_buf->first_bd;
 177
 178         /* Get the next bd */
 179         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 180
 181         /* Skip a parse bd... */
 182         --nbd;
 183         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 184
 185         /* ...and the TSO split header bd since they have no mapping */
 186         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 187                 --nbd;
 188                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 189         }
 190
 191         /* now free frags */
 192         while (nbd > 0) {
 193
 194                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 195                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 196                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 197                 if (--nbd)
 198                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 199         }
 200
 201         /* release skb */
 202         WARN_ON(!skb);
 203         if (likely(skb)) {
 204                 (*pkts_compl)++;
 205                 (*bytes_compl) += skb->len;
 206         }
 207
 208         dev_kfree_skb_any(skb);
 209         tx_buf->first_bd = 0;
 210         tx_buf->skb = NULL;
 211
 212         return new_cons;
 213 }
 214
 215 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 216 {
 217         struct netdev_queue *txq;
 218         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 219         unsigned int pkts_compl = 0, bytes_compl = 0;
 220
 221 #ifdef BNX2X_STOP_ON_ERROR
 222         if (unlikely(bp->panic))
 223                 return -1;
 224 #endif
 225
 226         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 227         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 228         sw_cons = txdata->tx_pkt_cons;
 229
 230         while (sw_cons != hw_cons) {
 231                 u16 pkt_cons;
 232
 233                 pkt_cons = TX_BD(sw_cons);
 234
 235                 DP(NETIF_MSG_TX_DONE,
 236                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 237                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 238
 239                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 240                                             &pkts_compl, &bytes_compl);
 241
 242                 sw_cons++;
 243         }
 244
 245         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 246
 247         txdata->tx_pkt_cons = sw_cons;
 248         txdata->tx_bd_cons = bd_cons;
 249
 250         /* Need to make the tx_bd_cons update visible to start_xmit()
 251          * before checking for netif_tx_queue_stopped().  Without the
 252          * memory barrier, there is a small possibility that
 253          * start_xmit() will miss it and cause the queue to be stopped
 254          * forever.
 255          * On the other hand we need an rmb() here to ensure the proper
 256          * ordering of bit testing in the following
 257          * netif_tx_queue_stopped(txq) call.
 258          */
 259         smp_mb();
 260
 261         if (unlikely(netif_tx_queue_stopped(txq))) {
 262                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 263                  * while it's empty. This could have happen if rx_action() gets
 264                  * suspended in bnx2x_tx_int() after the condition before
 265                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 266                  *
 267                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 268                  * sends some packets consuming the whole queue again->
 269                  * stops the queue
 270                  */
 271
 272                 __netif_tx_lock(txq, smp_processor_id());
 273
 274                 if ((netif_tx_queue_stopped(txq)) &&
 275                     (bp->state == BNX2X_STATE_OPEN) &&
 276                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 277                         netif_tx_wake_queue(txq);
 278
 279                 __netif_tx_unlock(txq);
 280         }
 281         return 0;
 282 }
 283
 284 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 285                                              u16 idx)
 286 {
 287         u16 last_max = fp->last_max_sge;
 288
 289         if (SUB_S16(idx, last_max) > 0)
 290                 fp->last_max_sge = idx;
 291 }
 292
 293 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 294                                          u16 sge_len,
 295                                          struct eth_end_agg_rx_cqe *cqe)
 296 {
 297         struct bnx2x *bp = fp->bp;
 298         u16 last_max, last_elem, first_elem;
 299         u16 delta = 0;
 300         u16 i;
 301
 302         if (!sge_len)
 303                 return;
 304
 305         /* First mark all used pages */
 306         for (i = 0; i < sge_len; i++)
 307                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 308                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 309
 310         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 311            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 312
 313         /* Here we assume that the last SGE index is the biggest */
 314         prefetch((void *)(fp->sge_mask));
 315         bnx2x_update_last_max_sge(fp,
 316                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 317
 318         last_max = RX_SGE(fp->last_max_sge);
 319         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 320         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 321
 322         /* If ring is not full */
 323         if (last_elem + 1 != first_elem)
 324                 last_elem++;
 325
 326         /* Now update the prod */
 327         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 328                 if (likely(fp->sge_mask[i]))
 329                         break;
 330
 331                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 332                 delta += BIT_VEC64_ELEM_SZ;
 333         }
 334
 335         if (delta > 0) {
 336                 fp->rx_sge_prod += delta;
 337                 /* clear page-end entries */
 338                 bnx2x_clear_sge_mask_next_elems(fp);
 339         }
 340
 341         DP(NETIF_MSG_RX_STATUS,
 342            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 343            fp->last_max_sge, fp->rx_sge_prod);
 344 }
 345
 346 /* Get Toeplitz hash value in the skb using the value from the
 347  * CQE (calculated by HW).
 348  */
 349 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 350                             const struct eth_fast_path_rx_cqe *cqe,
 351                             bool *l4_rxhash)
 352 {
 353         /* Get Toeplitz hash from CQE */
 354         if ((bp->dev->features & NETIF_F_RXHASH) &&
 355             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 356                 enum eth_rss_hash_type htype;
 357
 358                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 359                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 360                              (htype == TCP_IPV6_HASH_TYPE);
 361                 return le32_to_cpu(cqe->rss_hash_result);
 362         }
 363         *l4_rxhash = false;
 364         return 0;
 365 }
 366
 367 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 368                             u16 cons, u16 prod,
 369                             struct eth_fast_path_rx_cqe *cqe)
 370 {
 371         struct bnx2x *bp = fp->bp;
 372         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 373         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 374         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 375         dma_addr_t mapping;
 376         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 377         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 378
 379         /* print error if current state != stop */
 380         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 381                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 382
 383         /* Try to map an empty data buffer from the aggregation info  */
 384         mapping = dma_map_single(&bp->pdev->dev,
 385                                  first_buf->data + NET_SKB_PAD,
 386                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 387         /*
 388          *  ...if it fails - move the skb from the consumer to the producer
 389          *  and set the current aggregation state as ERROR to drop it
 390          *  when TPA_STOP arrives.
 391          */
 392
 393         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 394                 /* Move the BD from the consumer to the producer */
 395                 bnx2x_reuse_rx_data(fp, cons, prod);
 396                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 397                 return;
 398         }
 399
 400         /* move empty data from pool to prod */
 401         prod_rx_buf->data = first_buf->data;
 402         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 403         /* point prod_bd to new data */
 404         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 405         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 406
 407         /* move partial skb from cons to pool (don't unmap yet) */
 408         *first_buf = *cons_rx_buf;
 409
 410         /* mark bin state as START */
 411         tpa_info->parsing_flags =
 412                 le16_to_cpu(cqe->pars_flags.flags);
 413         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 414         tpa_info->tpa_state = BNX2X_TPA_START;
 415         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 416         tpa_info->placement_offset = cqe->placement_offset;
 417         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 418         if (fp->mode == TPA_MODE_GRO) {
 419                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 420                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 421                 tpa_info->gro_size = gro_size;
 422         }
 423
 424 #ifdef BNX2X_STOP_ON_ERROR
 425         fp->tpa_queue_used |= (1 << queue);
 426 #ifdef _ASM_GENERIC_INT_L64_H
 427         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 428 #else
 429         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 430 #endif
 431            fp->tpa_queue_used);
 432 #endif
 433 }
 434
 435 /* Timestamp option length allowed for TPA aggregation:
 436  *
 437  *              nop nop kind length echo val
 438  */
 439 #define TPA_TSTAMP_OPT_LEN      12
 440 /**
 441  * bnx2x_set_gro_params - compute GRO values
 442  *
 443  * @skb:                packet skb
 444  * @parsing_flags:      parsing flags from the START CQE
 445  * @len_on_bd:          total length of the first packet for the
 446  *                      aggregation.
 447  * @pkt_len:            length of all segments
 448  *
 449  * Approximate value of the MSS for this aggregation calculated using
 450  * the first packet of it.
 451  * Compute number of aggregated segments, and gso_type.
 452  */
 453 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 454                                  u16 len_on_bd, unsigned int pkt_len,
 455                                  u16 num_of_coalesced_segs)
 456 {
 457         /* TPA aggregation won't have either IP options or TCP options
 458          * other than timestamp or IPv6 extension headers.
 459          */
 460         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 461
 462         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 463             PRS_FLAG_OVERETH_IPV6) {
 464                 hdrs_len += sizeof(struct ipv6hdr);
 465                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 466         } else {
 467                 hdrs_len += sizeof(struct iphdr);
 468                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 469         }
 470
 471         /* Check if there was a TCP timestamp, if there is it's will
 472          * always be 12 bytes length: nop nop kind length echo val.
 473          *
 474          * Otherwise FW would close the aggregation.
 475          */
 476         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 477                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 478
 479         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 480
 481         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 482          * to skb_shinfo(skb)->gso_segs
 483          */
 484         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 485 }
 486
 487 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 488                               struct bnx2x_fastpath *fp, u16 index)
 489 {
 490         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 491         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 492         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 493         dma_addr_t mapping;
 494
 495         if (unlikely(page == NULL)) {
 496                 BNX2X_ERR("Can't alloc sge\n");
 497                 return -ENOMEM;
 498         }
 499
 500         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 501                                SGE_PAGES, DMA_FROM_DEVICE);
 502         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 503                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 504                 BNX2X_ERR("Can't map sge\n");
 505                 return -ENOMEM;
 506         }
 507
 508         sw_buf->page = page;
 509         dma_unmap_addr_set(sw_buf, mapping, mapping);
 510
 511         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 512         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 513
 514         return 0;
 515 }
 516
 517 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 518                                struct bnx2x_agg_info *tpa_info,
 519                                u16 pages,
 520                                struct sk_buff *skb,
 521                                struct eth_end_agg_rx_cqe *cqe,
 522                                u16 cqe_idx)
 523 {
 524         struct sw_rx_page *rx_pg, old_rx_pg;
 525         u32 i, frag_len, frag_size;
 526         int err, j, frag_id = 0;
 527         u16 len_on_bd = tpa_info->len_on_bd;
 528         u16 full_page = 0, gro_size = 0;
 529
 530         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 531
 532         if (fp->mode == TPA_MODE_GRO) {
 533                 gro_size = tpa_info->gro_size;
 534                 full_page = tpa_info->full_page;
 535         }
 536
 537         /* This is needed in order to enable forwarding support */
 538         if (frag_size)
 539                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 540                                      le16_to_cpu(cqe->pkt_len),
 541                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 542
 543 #ifdef BNX2X_STOP_ON_ERROR
 544         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 545                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 546                           pages, cqe_idx);
 547                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 548                 bnx2x_panic();
 549                 return -EINVAL;
 550         }
 551 #endif
 552
 553         /* Run through the SGL and compose the fragmented skb */
 554         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 555                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 556
 557                 /* FW gives the indices of the SGE as if the ring is an array
 558                    (meaning that "next" element will consume 2 indices) */
 559                 if (fp->mode == TPA_MODE_GRO)
 560                         frag_len = min_t(u32, frag_size, (u32)full_page);
 561                 else /* LRO */
 562                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 563
 564                 rx_pg = &fp->rx_page_ring[sge_idx];
 565                 old_rx_pg = *rx_pg;
 566
 567                 /* If we fail to allocate a substitute page, we simply stop
 568                    where we are and drop the whole packet */
 569                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 570                 if (unlikely(err)) {
 571                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 572                         return err;
 573                 }
 574
 575                 /* Unmap the page as we're going to pass it to the stack */
 576                 dma_unmap_page(&bp->pdev->dev,
 577                                dma_unmap_addr(&old_rx_pg, mapping),
 578                                SGE_PAGES, DMA_FROM_DEVICE);
 579                 /* Add one frag and update the appropriate fields in the skb */
 580                 if (fp->mode == TPA_MODE_LRO)
 581                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 582                 else { /* GRO */
 583                         int rem;
 584                         int offset = 0;
 585                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 586                                 int len = rem > gro_size ? gro_size : rem;
 587                                 skb_fill_page_desc(skb, frag_id++,
 588                                                    old_rx_pg.page, offset, len);
 589                                 if (offset)
 590                                         get_page(old_rx_pg.page);
 591                                 offset += len;
 592                         }
 593                 }
 594
 595                 skb->data_len += frag_len;
 596                 skb->truesize += SGE_PAGES;
 597                 skb->len += frag_len;
 598
 599                 frag_size -= frag_len;
 600         }
 601
 602         return 0;
 603 }
 604
 605 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 606 {
 607         if (fp->rx_frag_size)
 608                 put_page(virt_to_head_page(data));
 609         else
 610                 kfree(data);
 611 }
 612
 613 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 614 {
 615         if (fp->rx_frag_size)
 616                 return netdev_alloc_frag(fp->rx_frag_size);
 617
 618         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 619 }
 620
 621 #ifdef CONFIG_INET
 622 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 623 {
 624         const struct iphdr *iph = ip_hdr(skb);
 625         struct tcphdr *th;
 626
 627         skb_set_transport_header(skb, sizeof(struct iphdr));
 628         th = tcp_hdr(skb);
 629
 630         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 631                                   iph->saddr, iph->daddr, 0);
 632 }
 633
 634 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 635 {
 636         struct ipv6hdr *iph = ipv6_hdr(skb);
 637         struct tcphdr *th;
 638
 639         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 640         th = tcp_hdr(skb);
 641
 642         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 643                                   &iph->saddr, &iph->daddr, 0);
 644 }
 645
 646 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 647                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 648 {
 649         skb_set_network_header(skb, 0);
 650         gro_func(bp, skb);
 651         tcp_gro_complete(skb);
 652 }
 653 #endif
 654
 655 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 656                                struct sk_buff *skb)
 657 {
 658 #ifdef CONFIG_INET
 659         if (skb_shinfo(skb)->gso_size) {
 660                 switch (be16_to_cpu(skb->protocol)) {
 661                 case ETH_P_IP:
 662                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 663                         break;
 664                 case ETH_P_IPV6:
 665                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 666                         break;
 667                 default:
 668                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 669                                   be16_to_cpu(skb->protocol));
 670                 }
 671         }
 672 #endif
 673         napi_gro_receive(&fp->napi, skb);
 674 }
 675
 676 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 677                            struct bnx2x_agg_info *tpa_info,
 678                            u16 pages,
 679                            struct eth_end_agg_rx_cqe *cqe,
 680                            u16 cqe_idx)
 681 {
 682         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 683         u8 pad = tpa_info->placement_offset;
 684         u16 len = tpa_info->len_on_bd;
 685         struct sk_buff *skb = NULL;
 686         u8 *new_data, *data = rx_buf->data;
 687         u8 old_tpa_state = tpa_info->tpa_state;
 688
 689         tpa_info->tpa_state = BNX2X_TPA_STOP;
 690
 691         /* If we there was an error during the handling of the TPA_START -
 692          * drop this aggregation.
 693          */
 694         if (old_tpa_state == BNX2X_TPA_ERROR)
 695                 goto drop;
 696
 697         /* Try to allocate the new data */
 698         new_data = bnx2x_frag_alloc(fp);
 699         /* Unmap skb in the pool anyway, as we are going to change
 700            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 701            fails. */
 702         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 703                          fp->rx_buf_size, DMA_FROM_DEVICE);
 704         if (likely(new_data))
 705                 skb = build_skb(data, fp->rx_frag_size);
 706
 707         if (likely(skb)) {
 708 #ifdef BNX2X_STOP_ON_ERROR
 709                 if (pad + len > fp->rx_buf_size) {
 710                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 711                                   pad, len, fp->rx_buf_size);
 712                         bnx2x_panic();
 713                         return;
 714                 }
 715 #endif
 716
 717                 skb_reserve(skb, pad + NET_SKB_PAD);
 718                 skb_put(skb, len);
 719                 skb->rxhash = tpa_info->rxhash;
 720                 skb->l4_rxhash = tpa_info->l4_rxhash;
 721
 722                 skb->protocol = eth_type_trans(skb, bp->dev);
 723                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 724
 725                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 726                                          skb, cqe, cqe_idx)) {
 727                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 728                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 729                         bnx2x_gro_receive(bp, fp, skb);
 730                 } else {
 731                         DP(NETIF_MSG_RX_STATUS,
 732                            "Failed to allocate new pages - dropping packet!\n");
 733                         dev_kfree_skb_any(skb);
 734                 }
 735
 736                 /* put new data in bin */
 737                 rx_buf->data = new_data;
 738
 739                 return;
 740         }
 741         bnx2x_frag_free(fp, new_data);
 742 drop:
 743         /* drop the packet and keep the buffer in the bin */
 744         DP(NETIF_MSG_RX_STATUS,
 745            "Failed to allocate or map a new skb - dropping packet!\n");
 746         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 747 }
 748
 749 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 750                                struct bnx2x_fastpath *fp, u16 index)
 751 {
 752         u8 *data;
 753         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 754         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 755         dma_addr_t mapping;
 756
 757         data = bnx2x_frag_alloc(fp);
 758         if (unlikely(data == NULL))
 759                 return -ENOMEM;
 760
 761         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 762                                  fp->rx_buf_size,
 763                                  DMA_FROM_DEVICE);
 764         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 765                 bnx2x_frag_free(fp, data);
 766                 BNX2X_ERR("Can't map rx data\n");
 767                 return -ENOMEM;
 768         }
 769
 770         rx_buf->data = data;
 771         dma_unmap_addr_set(rx_buf, mapping, mapping);
 772
 773         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 774         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 775
 776         return 0;
 777 }
 778
 779 static
 780 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 781                                  struct bnx2x_fastpath *fp,
 782                                  struct bnx2x_eth_q_stats *qstats)
 783 {
 784         /* Do nothing if no L4 csum validation was done.
 785          * We do not check whether IP csum was validated. For IPv4 we assume
 786          * that if the card got as far as validating the L4 csum, it also
 787          * validated the IP csum. IPv6 has no IP csum.
 788          */
 789         if (cqe->fast_path_cqe.status_flags &
 790             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 791                 return;
 792
 793         /* If L4 validation was done, check if an error was found. */
 794
 795         if (cqe->fast_path_cqe.type_error_flags &
 796             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 797              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 798                 qstats->hw_csum_err++;
 799         else
 800                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 801 }
 802
 803 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 804 {
 805         struct bnx2x *bp = fp->bp;
 806         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 807         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 808         int rx_pkt = 0;
 809
 810 #ifdef BNX2X_STOP_ON_ERROR
 811         if (unlikely(bp->panic))
 812                 return 0;
 813 #endif
 814
 815         /* CQ "next element" is of the size of the regular element,
 816            that's why it's ok here */
 817         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 818         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 819                 hw_comp_cons++;
 820
 821         bd_cons = fp->rx_bd_cons;
 822         bd_prod = fp->rx_bd_prod;
 823         bd_prod_fw = bd_prod;
 824         sw_comp_cons = fp->rx_comp_cons;
 825         sw_comp_prod = fp->rx_comp_prod;
 826
 827         /* Memory barrier necessary as speculative reads of the rx
 828          * buffer can be ahead of the index in the status block
 829          */
 830         rmb();
 831
 832         DP(NETIF_MSG_RX_STATUS,
 833            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 834            fp->index, hw_comp_cons, sw_comp_cons);
 835
 836         while (sw_comp_cons != hw_comp_cons) {
 837                 struct sw_rx_bd *rx_buf = NULL;
 838                 struct sk_buff *skb;
 839                 union eth_rx_cqe *cqe;
 840                 struct eth_fast_path_rx_cqe *cqe_fp;
 841                 u8 cqe_fp_flags;
 842                 enum eth_rx_cqe_type cqe_fp_type;
 843                 u16 len, pad, queue;
 844                 u8 *data;
 845                 bool l4_rxhash;
 846
 847 #ifdef BNX2X_STOP_ON_ERROR
 848                 if (unlikely(bp->panic))
 849                         return 0;
 850 #endif
 851
 852                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 853                 bd_prod = RX_BD(bd_prod);
 854                 bd_cons = RX_BD(bd_cons);
 855
 856                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 857                 cqe_fp = &cqe->fast_path_cqe;
 858                 cqe_fp_flags = cqe_fp->type_error_flags;
 859                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 860
 861                 DP(NETIF_MSG_RX_STATUS,
 862                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 863                    CQE_TYPE(cqe_fp_flags),
 864                    cqe_fp_flags, cqe_fp->status_flags,
 865                    le32_to_cpu(cqe_fp->rss_hash_result),
 866                    le16_to_cpu(cqe_fp->vlan_tag),
 867                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 868
 869                 /* is this a slowpath msg? */
 870                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 871                         bnx2x_sp_event(fp, cqe);
 872                         goto next_cqe;
 873                 }
 874
 875                 rx_buf = &fp->rx_buf_ring[bd_cons];
 876                 data = rx_buf->data;
 877
 878                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 879                         struct bnx2x_agg_info *tpa_info;
 880                         u16 frag_size, pages;
 881 #ifdef BNX2X_STOP_ON_ERROR
 882                         /* sanity check */
 883                         if (fp->disable_tpa &&
 884                             (CQE_TYPE_START(cqe_fp_type) ||
 885                              CQE_TYPE_STOP(cqe_fp_type)))
 886                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 887                                           CQE_TYPE(cqe_fp_type));
 888 #endif
 889
 890                         if (CQE_TYPE_START(cqe_fp_type)) {
 891                                 u16 queue = cqe_fp->queue_index;
 892                                 DP(NETIF_MSG_RX_STATUS,
 893                                    "calling tpa_start on queue %d\n",
 894                                    queue);
 895
 896                                 bnx2x_tpa_start(fp, queue,
 897                                                 bd_cons, bd_prod,
 898                                                 cqe_fp);
 899
 900                                 goto next_rx;
 901                         }
 902                         queue = cqe->end_agg_cqe.queue_index;
 903                         tpa_info = &fp->tpa_info[queue];
 904                         DP(NETIF_MSG_RX_STATUS,
 905                            "calling tpa_stop on queue %d\n",
 906                            queue);
 907
 908                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 909                                     tpa_info->len_on_bd;
 910
 911                         if (fp->mode == TPA_MODE_GRO)
 912                                 pages = (frag_size + tpa_info->full_page - 1) /
 913                                          tpa_info->full_page;
 914                         else
 915                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 916                                         SGE_PAGE_SHIFT;
 917
 918                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 919                                        &cqe->end_agg_cqe, comp_ring_cons);
 920 #ifdef BNX2X_STOP_ON_ERROR
 921                         if (bp->panic)
 922                                 return 0;
 923 #endif
 924
 925                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 926                         goto next_cqe;
 927                 }
 928                 /* non TPA */
 929                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 930                 pad = cqe_fp->placement_offset;
 931                 dma_sync_single_for_cpu(&bp->pdev->dev,
 932                                         dma_unmap_addr(rx_buf, mapping),
 933                                         pad + RX_COPY_THRESH,
 934                                         DMA_FROM_DEVICE);
 935                 pad += NET_SKB_PAD;
 936                 prefetch(data + pad); /* speedup eth_type_trans() */
 937                 /* is this an error packet? */
 938                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 939                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 940                            "ERROR  flags %x  rx packet %u\n",
 941                            cqe_fp_flags, sw_comp_cons);
 942                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 943                         goto reuse_rx;
 944                 }
 945
 946                 /* Since we don't have a jumbo ring
 947                  * copy small packets if mtu > 1500
 948                  */
 949                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 950                     (len <= RX_COPY_THRESH)) {
 951                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 952                         if (skb == NULL) {
 953                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 954                                    "ERROR  packet dropped because of alloc failure\n");
 955                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 956                                 goto reuse_rx;
 957                         }
 958                         memcpy(skb->data, data + pad, len);
 959                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 960                 } else {
 961                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 962                                 dma_unmap_single(&bp->pdev->dev,
 963                                                  dma_unmap_addr(rx_buf, mapping),
 964                                                  fp->rx_buf_size,
 965                                                  DMA_FROM_DEVICE);
 966                                 skb = build_skb(data, fp->rx_frag_size);
 967                                 if (unlikely(!skb)) {
 968                                         bnx2x_frag_free(fp, data);
 969                                         bnx2x_fp_qstats(bp, fp)->
 970                                                         rx_skb_alloc_failed++;
 971                                         goto next_rx;
 972                                 }
 973                                 skb_reserve(skb, pad);
 974                         } else {
 975                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 976                                    "ERROR  packet dropped because of alloc failure\n");
 977                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 978 reuse_rx:
 979                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 980                                 goto next_rx;
 981                         }
 982                 }
 983
 984                 skb_put(skb, len);
 985                 skb->protocol = eth_type_trans(skb, bp->dev);
 986
 987                 /* Set Toeplitz hash for a none-LRO skb */
 988                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 989                 skb->l4_rxhash = l4_rxhash;
 990
 991                 skb_checksum_none_assert(skb);
 992
 993                 if (bp->dev->features & NETIF_F_RXCSUM)
 994                         bnx2x_csum_validate(skb, cqe, fp,
 995                                             bnx2x_fp_qstats(bp, fp));
 996
 997                 skb_record_rx_queue(skb, fp->rx_queue);
 998
 999                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1000                     PARSING_FLAGS_VLAN)
1001                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1002                                                le16_to_cpu(cqe_fp->vlan_tag));
1003
1004                 skb_mark_ll(skb, &fp->napi);
1005
1006                 if (bnx2x_fp_ll_polling(fp))
1007                         netif_receive_skb(skb);
1008                 else
1009                         napi_gro_receive(&fp->napi, skb);
1010 next_rx:
1011                 rx_buf->data = NULL;
1012
1013                 bd_cons = NEXT_RX_IDX(bd_cons);
1014                 bd_prod = NEXT_RX_IDX(bd_prod);
1015                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1016                 rx_pkt++;
1017 next_cqe:
1018                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1019                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1020
1021                 if (rx_pkt == budget)
1022                         break;
1023         } /* while */
1024
1025         fp->rx_bd_cons = bd_cons;
1026         fp->rx_bd_prod = bd_prod_fw;
1027         fp->rx_comp_cons = sw_comp_cons;
1028         fp->rx_comp_prod = sw_comp_prod;
1029
1030         /* Update producers */
1031         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1032                              fp->rx_sge_prod);
1033
1034         fp->rx_pkt += rx_pkt;
1035         fp->rx_calls++;
1036
1037         return rx_pkt;
1038 }
1039
1040 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1041 {
1042         struct bnx2x_fastpath *fp = fp_cookie;
1043         struct bnx2x *bp = fp->bp;
1044         u8 cos;
1045
1046         DP(NETIF_MSG_INTR,
1047            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1048            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1049
1050         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1051
1052 #ifdef BNX2X_STOP_ON_ERROR
1053         if (unlikely(bp->panic))
1054                 return IRQ_HANDLED;
1055 #endif
1056
1057         /* Handle Rx and Tx according to MSI-X vector */
1058         prefetch(fp->rx_cons_sb);
1059
1060         for_each_cos_in_tx_queue(fp, cos)
1061                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1062
1063         prefetch(&fp->sb_running_index[SM_RX_ID]);
1064         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1065
1066         return IRQ_HANDLED;
1067 }
1068
1069 /* HW Lock for shared dual port PHYs */
1070 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1071 {
1072         mutex_lock(&bp->port.phy_mutex);
1073
1074         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1075 }
1076
1077 void bnx2x_release_phy_lock(struct bnx2x *bp)
1078 {
1079         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1080
1081         mutex_unlock(&bp->port.phy_mutex);
1082 }
1083
1084 /* calculates MF speed according to current linespeed and MF configuration */
1085 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1086 {
1087         u16 line_speed = bp->link_vars.line_speed;
1088         if (IS_MF(bp)) {
1089                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1090                                                    bp->mf_config[BP_VN(bp)]);
1091
1092                 /* Calculate the current MAX line speed limit for the MF
1093                  * devices
1094                  */
1095                 if (IS_MF_SI(bp))
1096                         line_speed = (line_speed * maxCfg) / 100;
1097                 else { /* SD mode */
1098                         u16 vn_max_rate = maxCfg * 100;
1099
1100                         if (vn_max_rate < line_speed)
1101                                 line_speed = vn_max_rate;
1102                 }
1103         }
1104
1105         return line_speed;
1106 }
1107
1108 /**
1109  * bnx2x_fill_report_data - fill link report data to report
1110  *
1111  * @bp:         driver handle
1112  * @data:       link state to update
1113  *
1114  * It uses a none-atomic bit operations because is called under the mutex.
1115  */
1116 static void bnx2x_fill_report_data(struct bnx2x *bp,
1117                                    struct bnx2x_link_report_data *data)
1118 {
1119         u16 line_speed = bnx2x_get_mf_speed(bp);
1120
1121         memset(data, 0, sizeof(*data));
1122
1123         /* Fill the report data: effective line speed */
1124         data->line_speed = line_speed;
1125
1126         /* Link is down */
1127         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1128                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1129                           &data->link_report_flags);
1130
1131         /* Full DUPLEX */
1132         if (bp->link_vars.duplex == DUPLEX_FULL)
1133                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1134
1135         /* Rx Flow Control is ON */
1136         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1137                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1138
1139         /* Tx Flow Control is ON */
1140         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1141                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1142 }
1143
1144 /**
1145  * bnx2x_link_report - report link status to OS.
1146  *
1147  * @bp:         driver handle
1148  *
1149  * Calls the __bnx2x_link_report() under the same locking scheme
1150  * as a link/PHY state managing code to ensure a consistent link
1151  * reporting.
1152  */
1153
1154 void bnx2x_link_report(struct bnx2x *bp)
1155 {
1156         bnx2x_acquire_phy_lock(bp);
1157         __bnx2x_link_report(bp);
1158         bnx2x_release_phy_lock(bp);
1159 }
1160
1161 /**
1162  * __bnx2x_link_report - report link status to OS.
1163  *
1164  * @bp:         driver handle
1165  *
1166  * None atomic implementation.
1167  * Should be called under the phy_lock.
1168  */
1169 void __bnx2x_link_report(struct bnx2x *bp)
1170 {
1171         struct bnx2x_link_report_data cur_data;
1172
1173         /* reread mf_cfg */
1174         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1175                 bnx2x_read_mf_cfg(bp);
1176
1177         /* Read the current link report info */
1178         bnx2x_fill_report_data(bp, &cur_data);
1179
1180         /* Don't report link down or exactly the same link status twice */
1181         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1182             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1183                       &bp->last_reported_link.link_report_flags) &&
1184              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1185                       &cur_data.link_report_flags)))
1186                 return;
1187
1188         bp->link_cnt++;
1189
1190         /* We are going to report a new link parameters now -
1191          * remember the current data for the next time.
1192          */
1193         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1194
1195         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1196                      &cur_data.link_report_flags)) {
1197                 netif_carrier_off(bp->dev);
1198                 netdev_err(bp->dev, "NIC Link is Down\n");
1199                 return;
1200         } else {
1201                 const char *duplex;
1202                 const char *flow;
1203
1204                 netif_carrier_on(bp->dev);
1205
1206                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1207                                        &cur_data.link_report_flags))
1208                         duplex = "full";
1209                 else
1210                         duplex = "half";
1211
1212                 /* Handle the FC at the end so that only these flags would be
1213                  * possibly set. This way we may easily check if there is no FC
1214                  * enabled.
1215                  */
1216                 if (cur_data.link_report_flags) {
1217                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1218                                      &cur_data.link_report_flags)) {
1219                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1220                                      &cur_data.link_report_flags))
1221                                         flow = "ON - receive & transmit";
1222                                 else
1223                                         flow = "ON - receive";
1224                         } else {
1225                                 flow = "ON - transmit";
1226                         }
1227                 } else {
1228                         flow = "none";
1229                 }
1230                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1231                             cur_data.line_speed, duplex, flow);
1232         }
1233 }
1234
1235 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1236 {
1237         int i;
1238
1239         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1240                 struct eth_rx_sge *sge;
1241
1242                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1243                 sge->addr_hi =
1244                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1245                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1246
1247                 sge->addr_lo =
1248                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1249                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1250         }
1251 }
1252
1253 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1254                                 struct bnx2x_fastpath *fp, int last)
1255 {
1256         int i;
1257
1258         for (i = 0; i < last; i++) {
1259                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1260                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1261                 u8 *data = first_buf->data;
1262
1263                 if (data == NULL) {
1264                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1265                         continue;
1266                 }
1267                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1268                         dma_unmap_single(&bp->pdev->dev,
1269                                          dma_unmap_addr(first_buf, mapping),
1270                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1271                 bnx2x_frag_free(fp, data);
1272                 first_buf->data = NULL;
1273         }
1274 }
1275
1276 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1277 {
1278         int j;
1279
1280         for_each_rx_queue_cnic(bp, j) {
1281                 struct bnx2x_fastpath *fp = &bp->fp[j];
1282
1283                 fp->rx_bd_cons = 0;
1284
1285                 /* Activate BD ring */
1286                 /* Warning!
1287                  * this will generate an interrupt (to the TSTORM)
1288                  * must only be done after chip is initialized
1289                  */
1290                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1291                                      fp->rx_sge_prod);
1292         }
1293 }
1294
1295 void bnx2x_init_rx_rings(struct bnx2x *bp)
1296 {
1297         int func = BP_FUNC(bp);
1298         u16 ring_prod;
1299         int i, j;
1300
1301         /* Allocate TPA resources */
1302         for_each_eth_queue(bp, j) {
1303                 struct bnx2x_fastpath *fp = &bp->fp[j];
1304
1305                 DP(NETIF_MSG_IFUP,
1306                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1307
1308                 if (!fp->disable_tpa) {
1309                         /* Fill the per-aggregation pool */
1310                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1311                                 struct bnx2x_agg_info *tpa_info =
1312                                         &fp->tpa_info[i];
1313                                 struct sw_rx_bd *first_buf =
1314                                         &tpa_info->first_buf;
1315
1316                                 first_buf->data = bnx2x_frag_alloc(fp);
1317                                 if (!first_buf->data) {
1318                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1319                                                   j);
1320                                         bnx2x_free_tpa_pool(bp, fp, i);
1321                                         fp->disable_tpa = 1;
1322                                         break;
1323                                 }
1324                                 dma_unmap_addr_set(first_buf, mapping, 0);
1325                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1326                         }
1327
1328                         /* "next page" elements initialization */
1329                         bnx2x_set_next_page_sgl(fp);
1330
1331                         /* set SGEs bit mask */
1332                         bnx2x_init_sge_ring_bit_mask(fp);
1333
1334                         /* Allocate SGEs and initialize the ring elements */
1335                         for (i = 0, ring_prod = 0;
1336                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1337
1338                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1339                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1340                                                   i);
1341                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1342                                                   j);
1343                                         /* Cleanup already allocated elements */
1344                                         bnx2x_free_rx_sge_range(bp, fp,
1345                                                                 ring_prod);
1346                                         bnx2x_free_tpa_pool(bp, fp,
1347                                                             MAX_AGG_QS(bp));
1348                                         fp->disable_tpa = 1;
1349                                         ring_prod = 0;
1350                                         break;
1351                                 }
1352                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1353                         }
1354
1355                         fp->rx_sge_prod = ring_prod;
1356                 }
1357         }
1358
1359         for_each_eth_queue(bp, j) {
1360                 struct bnx2x_fastpath *fp = &bp->fp[j];
1361
1362                 fp->rx_bd_cons = 0;
1363
1364                 /* Activate BD ring */
1365                 /* Warning!
1366                  * this will generate an interrupt (to the TSTORM)
1367                  * must only be done after chip is initialized
1368                  */
1369                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1370                                      fp->rx_sge_prod);
1371
1372                 if (j != 0)
1373                         continue;
1374
1375                 if (CHIP_IS_E1(bp)) {
1376                         REG_WR(bp, BAR_USTRORM_INTMEM +
1377                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1378                                U64_LO(fp->rx_comp_mapping));
1379                         REG_WR(bp, BAR_USTRORM_INTMEM +
1380                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1381                                U64_HI(fp->rx_comp_mapping));
1382                 }
1383         }
1384 }
1385
1386 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1387 {
1388         u8 cos;
1389         struct bnx2x *bp = fp->bp;
1390
1391         for_each_cos_in_tx_queue(fp, cos) {
1392                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1393                 unsigned pkts_compl = 0, bytes_compl = 0;
1394
1395                 u16 sw_prod = txdata->tx_pkt_prod;
1396                 u16 sw_cons = txdata->tx_pkt_cons;
1397
1398                 while (sw_cons != sw_prod) {
1399                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1400                                           &pkts_compl, &bytes_compl);
1401                         sw_cons++;
1402                 }
1403
1404                 netdev_tx_reset_queue(
1405                         netdev_get_tx_queue(bp->dev,
1406                                             txdata->txq_index));
1407         }
1408 }
1409
1410 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1411 {
1412         int i;
1413
1414         for_each_tx_queue_cnic(bp, i) {
1415                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1416         }
1417 }
1418
1419 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1420 {
1421         int i;
1422
1423         for_each_eth_queue(bp, i) {
1424                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1425         }
1426 }
1427
1428 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1429 {
1430         struct bnx2x *bp = fp->bp;
1431         int i;
1432
1433         /* ring wasn't allocated */
1434         if (fp->rx_buf_ring == NULL)
1435                 return;
1436
1437         for (i = 0; i < NUM_RX_BD; i++) {
1438                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1439                 u8 *data = rx_buf->data;
1440
1441                 if (data == NULL)
1442                         continue;
1443                 dma_unmap_single(&bp->pdev->dev,
1444                                  dma_unmap_addr(rx_buf, mapping),
1445                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1446
1447                 rx_buf->data = NULL;
1448                 bnx2x_frag_free(fp, data);
1449         }
1450 }
1451
1452 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1453 {
1454         int j;
1455
1456         for_each_rx_queue_cnic(bp, j) {
1457                 bnx2x_free_rx_bds(&bp->fp[j]);
1458         }
1459 }
1460
1461 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1462 {
1463         int j;
1464
1465         for_each_eth_queue(bp, j) {
1466                 struct bnx2x_fastpath *fp = &bp->fp[j];
1467
1468                 bnx2x_free_rx_bds(fp);
1469
1470                 if (!fp->disable_tpa)
1471                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1472         }
1473 }
1474
1475 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1476 {
1477         bnx2x_free_tx_skbs_cnic(bp);
1478         bnx2x_free_rx_skbs_cnic(bp);
1479 }
1480
1481 void bnx2x_free_skbs(struct bnx2x *bp)
1482 {
1483         bnx2x_free_tx_skbs(bp);
1484         bnx2x_free_rx_skbs(bp);
1485 }
1486
1487 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1488 {
1489         /* load old values */
1490         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1491
1492         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1493                 /* leave all but MAX value */
1494                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1495
1496                 /* set new MAX value */
1497                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1498                                 & FUNC_MF_CFG_MAX_BW_MASK;
1499
1500                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1501         }
1502 }
1503
1504 /**
1505  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1506  *
1507  * @bp:         driver handle
1508  * @nvecs:      number of vectors to be released
1509  */
1510 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1511 {
1512         int i, offset = 0;
1513
1514         if (nvecs == offset)
1515                 return;
1516
1517         /* VFs don't have a default SB */
1518         if (IS_PF(bp)) {
1519                 free_irq(bp->msix_table[offset].vector, bp->dev);
1520                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1521                    bp->msix_table[offset].vector);
1522                 offset++;
1523         }
1524
1525         if (CNIC_SUPPORT(bp)) {
1526                 if (nvecs == offset)
1527                         return;
1528                 offset++;
1529         }
1530
1531         for_each_eth_queue(bp, i) {
1532                 if (nvecs == offset)
1533                         return;
1534                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1535                    i, bp->msix_table[offset].vector);
1536
1537                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1538         }
1539 }
1540
1541 void bnx2x_free_irq(struct bnx2x *bp)
1542 {
1543         if (bp->flags & USING_MSIX_FLAG &&
1544             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1545                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1546
1547                 /* vfs don't have a default status block */
1548                 if (IS_PF(bp))
1549                         nvecs++;
1550
1551                 bnx2x_free_msix_irqs(bp, nvecs);
1552         } else {
1553                 free_irq(bp->dev->irq, bp->dev);
1554         }
1555 }
1556
1557 int bnx2x_enable_msix(struct bnx2x *bp)
1558 {
1559         int msix_vec = 0, i, rc;
1560
1561         /* VFs don't have a default status block */
1562         if (IS_PF(bp)) {
1563                 bp->msix_table[msix_vec].entry = msix_vec;
1564                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1565                                bp->msix_table[0].entry);
1566                 msix_vec++;
1567         }
1568
1569         /* Cnic requires an msix vector for itself */
1570         if (CNIC_SUPPORT(bp)) {
1571                 bp->msix_table[msix_vec].entry = msix_vec;
1572                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1573                                msix_vec, bp->msix_table[msix_vec].entry);
1574                 msix_vec++;
1575         }
1576
1577         /* We need separate vectors for ETH queues only (not FCoE) */
1578         for_each_eth_queue(bp, i) {
1579                 bp->msix_table[msix_vec].entry = msix_vec;
1580                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1581                                msix_vec, msix_vec, i);
1582                 msix_vec++;
1583         }
1584
1585         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1586            msix_vec);
1587
1588         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1589
1590         /*
1591          * reconfigure number of tx/rx queues according to available
1592          * MSI-X vectors
1593          */
1594         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1595                 /* how less vectors we will have? */
1596                 int diff = msix_vec - rc;
1597
1598                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1599
1600                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1601
1602                 if (rc) {
1603                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1604                         goto no_msix;
1605                 }
1606                 /*
1607                  * decrease number of queues by number of unallocated entries
1608                  */
1609                 bp->num_ethernet_queues -= diff;
1610                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1611
1612                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1613                                bp->num_queues);
1614         } else if (rc > 0) {
1615                 /* Get by with single vector */
1616                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1617                 if (rc) {
1618                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1619                                        rc);
1620                         goto no_msix;
1621                 }
1622
1623                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1624                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1625
1626                 BNX2X_DEV_INFO("set number of queues to 1\n");
1627                 bp->num_ethernet_queues = 1;
1628                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1629         } else if (rc < 0) {
1630                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1631                 goto no_msix;
1632         }
1633
1634         bp->flags |= USING_MSIX_FLAG;
1635
1636         return 0;
1637
1638 no_msix:
1639         /* fall to INTx if not enough memory */
1640         if (rc == -ENOMEM)
1641                 bp->flags |= DISABLE_MSI_FLAG;
1642
1643         return rc;
1644 }
1645
1646 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1647 {
1648         int i, rc, offset = 0;
1649
1650         /* no default status block for vf */
1651         if (IS_PF(bp)) {
1652                 rc = request_irq(bp->msix_table[offset++].vector,
1653                                  bnx2x_msix_sp_int, 0,
1654                                  bp->dev->name, bp->dev);
1655                 if (rc) {
1656                         BNX2X_ERR("request sp irq failed\n");
1657                         return -EBUSY;
1658                 }
1659         }
1660
1661         if (CNIC_SUPPORT(bp))
1662                 offset++;
1663
1664         for_each_eth_queue(bp, i) {
1665                 struct bnx2x_fastpath *fp = &bp->fp[i];
1666                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1667                          bp->dev->name, i);
1668
1669                 rc = request_irq(bp->msix_table[offset].vector,
1670                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1671                 if (rc) {
1672                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1673                               bp->msix_table[offset].vector, rc);
1674                         bnx2x_free_msix_irqs(bp, offset);
1675                         return -EBUSY;
1676                 }
1677
1678                 offset++;
1679         }
1680
1681         i = BNX2X_NUM_ETH_QUEUES(bp);
1682         if (IS_PF(bp)) {
1683                 offset = 1 + CNIC_SUPPORT(bp);
1684                 netdev_info(bp->dev,
1685                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1686                             bp->msix_table[0].vector,
1687                             0, bp->msix_table[offset].vector,
1688                             i - 1, bp->msix_table[offset + i - 1].vector);
1689         } else {
1690                 offset = CNIC_SUPPORT(bp);
1691                 netdev_info(bp->dev,
1692                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1693                             0, bp->msix_table[offset].vector,
1694                             i - 1, bp->msix_table[offset + i - 1].vector);
1695         }
1696         return 0;
1697 }
1698
1699 int bnx2x_enable_msi(struct bnx2x *bp)
1700 {
1701         int rc;
1702
1703         rc = pci_enable_msi(bp->pdev);
1704         if (rc) {
1705                 BNX2X_DEV_INFO("MSI is not attainable\n");
1706                 return -1;
1707         }
1708         bp->flags |= USING_MSI_FLAG;
1709
1710         return 0;
1711 }
1712
1713 static int bnx2x_req_irq(struct bnx2x *bp)
1714 {
1715         unsigned long flags;
1716         unsigned int irq;
1717
1718         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1719                 flags = 0;
1720         else
1721                 flags = IRQF_SHARED;
1722
1723         if (bp->flags & USING_MSIX_FLAG)
1724                 irq = bp->msix_table[0].vector;
1725         else
1726                 irq = bp->pdev->irq;
1727
1728         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1729 }
1730
1731 int bnx2x_setup_irqs(struct bnx2x *bp)
1732 {
1733         int rc = 0;
1734         if (bp->flags & USING_MSIX_FLAG &&
1735             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1736                 rc = bnx2x_req_msix_irqs(bp);
1737                 if (rc)
1738                         return rc;
1739         } else {
1740                 rc = bnx2x_req_irq(bp);
1741                 if (rc) {
1742                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1743                         return rc;
1744                 }
1745                 if (bp->flags & USING_MSI_FLAG) {
1746                         bp->dev->irq = bp->pdev->irq;
1747                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1748                                     bp->dev->irq);
1749                 }
1750                 if (bp->flags & USING_MSIX_FLAG) {
1751                         bp->dev->irq = bp->msix_table[0].vector;
1752                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1753                                     bp->dev->irq);
1754                 }
1755         }
1756
1757         return 0;
1758 }
1759
1760 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1761 {
1762         int i;
1763
1764         for_each_rx_queue_cnic(bp, i) {
1765                 bnx2x_fp_init_lock(&bp->fp[i]);
1766                 napi_enable(&bnx2x_fp(bp, i, napi));
1767         }
1768 }
1769
1770 static void bnx2x_napi_enable(struct bnx2x *bp)
1771 {
1772         int i;
1773
1774         for_each_eth_queue(bp, i) {
1775                 bnx2x_fp_init_lock(&bp->fp[i]);
1776                 napi_enable(&bnx2x_fp(bp, i, napi));
1777         }
1778 }
1779
1780 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1781 {
1782         int i;
1783
1784         local_bh_disable();
1785         for_each_rx_queue_cnic(bp, i) {
1786                 napi_disable(&bnx2x_fp(bp, i, napi));
1787                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1788                         mdelay(1);
1789         }
1790         local_bh_enable();
1791 }
1792
1793 static void bnx2x_napi_disable(struct bnx2x *bp)
1794 {
1795         int i;
1796
1797         local_bh_disable();
1798         for_each_eth_queue(bp, i) {
1799                 napi_disable(&bnx2x_fp(bp, i, napi));
1800                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1801                         mdelay(1);
1802         }
1803         local_bh_enable();
1804 }
1805
1806 void bnx2x_netif_start(struct bnx2x *bp)
1807 {
1808         if (netif_running(bp->dev)) {
1809                 bnx2x_napi_enable(bp);
1810                 if (CNIC_LOADED(bp))
1811                         bnx2x_napi_enable_cnic(bp);
1812                 bnx2x_int_enable(bp);
1813                 if (bp->state == BNX2X_STATE_OPEN)
1814                         netif_tx_wake_all_queues(bp->dev);
1815         }
1816 }
1817
1818 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1819 {
1820         bnx2x_int_disable_sync(bp, disable_hw);
1821         bnx2x_napi_disable(bp);
1822         if (CNIC_LOADED(bp))
1823                 bnx2x_napi_disable_cnic(bp);
1824 }
1825
1826 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1827 {
1828         struct bnx2x *bp = netdev_priv(dev);
1829
1830         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1831                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1832                 u16 ether_type = ntohs(hdr->h_proto);
1833
1834                 /* Skip VLAN tag if present */
1835                 if (ether_type == ETH_P_8021Q) {
1836                         struct vlan_ethhdr *vhdr =
1837                                 (struct vlan_ethhdr *)skb->data;
1838
1839                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1840                 }
1841
1842                 /* If ethertype is FCoE or FIP - use FCoE ring */
1843                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1844                         return bnx2x_fcoe_tx(bp, txq_index);
1845         }
1846
1847         /* select a non-FCoE queue */
1848         return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1849 }
1850
1851 void bnx2x_set_num_queues(struct bnx2x *bp)
1852 {
1853         /* RSS queues */
1854         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1855
1856         /* override in STORAGE SD modes */
1857         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1858                 bp->num_ethernet_queues = 1;
1859
1860         /* Add special queues */
1861         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1862         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1863
1864         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1865 }
1866
1867 /**
1868  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1869  *
1870  * @bp:         Driver handle
1871  *
1872  * We currently support for at most 16 Tx queues for each CoS thus we will
1873  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1874  * bp->max_cos.
1875  *
1876  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1877  * index after all ETH L2 indices.
1878  *
1879  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1880  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1881  * 16..31,...) with indices that are not coupled with any real Tx queue.
1882  *
1883  * The proper configuration of skb->queue_mapping is handled by
1884  * bnx2x_select_queue() and __skb_tx_hash().
1885  *
1886  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1887  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1888  */
1889 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1890 {
1891         int rc, tx, rx;
1892
1893         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1894         rx = BNX2X_NUM_ETH_QUEUES(bp);
1895
1896 /* account for fcoe queue */
1897         if (include_cnic && !NO_FCOE(bp)) {
1898                 rx++;
1899                 tx++;
1900         }
1901
1902         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1903         if (rc) {
1904                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1905                 return rc;
1906         }
1907         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1908         if (rc) {
1909                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1910                 return rc;
1911         }
1912
1913         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1914                           tx, rx);
1915
1916         return rc;
1917 }
1918
1919 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1920 {
1921         int i;
1922
1923         for_each_queue(bp, i) {
1924                 struct bnx2x_fastpath *fp = &bp->fp[i];
1925                 u32 mtu;
1926
1927                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1928                 if (IS_FCOE_IDX(i))
1929                         /*
1930                          * Although there are no IP frames expected to arrive to
1931                          * this ring we still want to add an
1932                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1933                          * overrun attack.
1934                          */
1935                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1936                 else
1937                         mtu = bp->dev->mtu;
1938                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1939                                   IP_HEADER_ALIGNMENT_PADDING +
1940                                   ETH_OVREHEAD +
1941                                   mtu +
1942                                   BNX2X_FW_RX_ALIGN_END;
1943                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1944                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1945                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1946                 else
1947                         fp->rx_frag_size = 0;
1948         }
1949 }
1950
1951 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1952 {
1953         int i;
1954         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1955
1956         /* Prepare the initial contents for the indirection table if RSS is
1957          * enabled
1958          */
1959         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1960                 bp->rss_conf_obj.ind_table[i] =
1961                         bp->fp->cl_id +
1962                         ethtool_rxfh_indir_default(i, num_eth_queues);
1963
1964         /*
1965          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1966          * per-port, so if explicit configuration is needed , do it only
1967          * for a PMF.
1968          *
1969          * For 57712 and newer on the other hand it's a per-function
1970          * configuration.
1971          */
1972         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1973 }
1974
1975 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1976                         bool config_hash)
1977 {
1978         struct bnx2x_config_rss_params params = {NULL};
1979
1980         /* Although RSS is meaningless when there is a single HW queue we
1981          * still need it enabled in order to have HW Rx hash generated.
1982          *
1983          * if (!is_eth_multi(bp))
1984          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1985          */
1986
1987         params.rss_obj = rss_obj;
1988
1989         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1990
1991         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1992
1993         /* RSS configuration */
1994         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1995         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1996         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1997         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1998         if (rss_obj->udp_rss_v4)
1999                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2000         if (rss_obj->udp_rss_v6)
2001                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2002
2003         /* Hash bits */
2004         params.rss_result_mask = MULTI_MASK;
2005
2006         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2007
2008         if (config_hash) {
2009                 /* RSS keys */
2010                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
2011                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2012         }
2013
2014         return bnx2x_config_rss(bp, &params);
2015 }
2016
2017 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2018 {
2019         struct bnx2x_func_state_params func_params = {NULL};
2020
2021         /* Prepare parameters for function state transitions */
2022         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2023
2024         func_params.f_obj = &bp->func_obj;
2025         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2026
2027         func_params.params.hw_init.load_phase = load_code;
2028
2029         return bnx2x_func_state_change(bp, &func_params);
2030 }
2031
2032 /*
2033  * Cleans the object that have internal lists without sending
2034  * ramrods. Should be run when interrupts are disabled.
2035  */
2036 void bnx2x_squeeze_objects(struct bnx2x *bp)
2037 {
2038         int rc;
2039         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2040         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2041         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2042
2043         /***************** Cleanup MACs' object first *************************/
2044
2045         /* Wait for completion of requested */
2046         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2047         /* Perform a dry cleanup */
2048         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2049
2050         /* Clean ETH primary MAC */
2051         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2052         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2053                                  &ramrod_flags);
2054         if (rc != 0)
2055                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2056
2057         /* Cleanup UC list */
2058         vlan_mac_flags = 0;
2059         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2060         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2061                                  &ramrod_flags);
2062         if (rc != 0)
2063                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2064
2065         /***************** Now clean mcast object *****************************/
2066         rparam.mcast_obj = &bp->mcast_obj;
2067         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2068
2069         /* Add a DEL command... */
2070         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2071         if (rc < 0)
2072                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2073                           rc);
2074
2075         /* ...and wait until all pending commands are cleared */
2076         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2077         while (rc != 0) {
2078                 if (rc < 0) {
2079                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2080                                   rc);
2081                         return;
2082                 }
2083
2084                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2085         }
2086 }
2087
2088 #ifndef BNX2X_STOP_ON_ERROR
2089 #define LOAD_ERROR_EXIT(bp, label) \
2090         do { \
2091                 (bp)->state = BNX2X_STATE_ERROR; \
2092                 goto label; \
2093         } while (0)
2094
2095 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2096         do { \
2097                 bp->cnic_loaded = false; \
2098                 goto label; \
2099         } while (0)
2100 #else /*BNX2X_STOP_ON_ERROR*/
2101 #define LOAD_ERROR_EXIT(bp, label) \
2102         do { \
2103                 (bp)->state = BNX2X_STATE_ERROR; \
2104                 (bp)->panic = 1; \
2105                 return -EBUSY; \
2106         } while (0)
2107 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2108         do { \
2109                 bp->cnic_loaded = false; \
2110                 (bp)->panic = 1; \
2111                 return -EBUSY; \
2112         } while (0)
2113 #endif /*BNX2X_STOP_ON_ERROR*/
2114
2115 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2116 {
2117         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2118                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2119         return;
2120 }
2121
2122 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2123 {
2124         int num_groups, vf_headroom = 0;
2125         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2126
2127         /* number of queues for statistics is number of eth queues + FCoE */
2128         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2129
2130         /* Total number of FW statistics requests =
2131          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2132          * and fcoe l2 queue) stats + num of queues (which includes another 1
2133          * for fcoe l2 queue if applicable)
2134          */
2135         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2136
2137         /* vf stats appear in the request list, but their data is allocated by
2138          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2139          * it is used to determine where to place the vf stats queries in the
2140          * request struct
2141          */
2142         if (IS_SRIOV(bp))
2143                 vf_headroom = bnx2x_vf_headroom(bp);
2144
2145         /* Request is built from stats_query_header and an array of
2146          * stats_query_cmd_group each of which contains
2147          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2148          * configured in the stats_query_header.
2149          */
2150         num_groups =
2151                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2152                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2153                  1 : 0));
2154
2155         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2156            bp->fw_stats_num, vf_headroom, num_groups);
2157         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2158                 num_groups * sizeof(struct stats_query_cmd_group);
2159
2160         /* Data for statistics requests + stats_counter
2161          * stats_counter holds per-STORM counters that are incremented
2162          * when STORM has finished with the current request.
2163          * memory for FCoE offloaded statistics are counted anyway,
2164          * even if they will not be sent.
2165          * VF stats are not accounted for here as the data of VF stats is stored
2166          * in memory allocated by the VF, not here.
2167          */
2168         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2169                 sizeof(struct per_pf_stats) +
2170                 sizeof(struct fcoe_statistics_params) +
2171                 sizeof(struct per_queue_stats) * num_queue_stats +
2172                 sizeof(struct stats_counter);
2173
2174         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2175                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2176
2177         /* Set shortcuts */
2178         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2179         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2180         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2181                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2182         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2183                 bp->fw_stats_req_sz;
2184
2185         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2186            U64_HI(bp->fw_stats_req_mapping),
2187            U64_LO(bp->fw_stats_req_mapping));
2188         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2189            U64_HI(bp->fw_stats_data_mapping),
2190            U64_LO(bp->fw_stats_data_mapping));
2191         return 0;
2192
2193 alloc_mem_err:
2194         bnx2x_free_fw_stats_mem(bp);
2195         BNX2X_ERR("Can't allocate FW stats memory\n");
2196         return -ENOMEM;
2197 }
2198
2199 /* send load request to mcp and analyze response */
2200 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2201 {
2202         u32 param;
2203
2204         /* init fw_seq */
2205         bp->fw_seq =
2206                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2207                  DRV_MSG_SEQ_NUMBER_MASK);
2208         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2209
2210         /* Get current FW pulse sequence */
2211         bp->fw_drv_pulse_wr_seq =
2212                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2213                  DRV_PULSE_SEQ_MASK);
2214         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2215
2216         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2217
2218         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2219                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2220
2221         /* load request */
2222         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2223
2224         /* if mcp fails to respond we must abort */
2225         if (!(*load_code)) {
2226                 BNX2X_ERR("MCP response failure, aborting\n");
2227                 return -EBUSY;
2228         }
2229
2230         /* If mcp refused (e.g. other port is in diagnostic mode) we
2231          * must abort
2232          */
2233         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2234                 BNX2X_ERR("MCP refused load request, aborting\n");
2235                 return -EBUSY;
2236         }
2237         return 0;
2238 }
2239
2240 /* check whether another PF has already loaded FW to chip. In
2241  * virtualized environments a pf from another VM may have already
2242  * initialized the device including loading FW
2243  */
2244 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2245 {
2246         /* is another pf loaded on this engine? */
2247         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2248             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2249                 /* build my FW version dword */
2250                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2251                         (BCM_5710_FW_MINOR_VERSION << 8) +
2252                         (BCM_5710_FW_REVISION_VERSION << 16) +
2253                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2254
2255                 /* read loaded FW from chip */
2256                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2257
2258                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2259                    loaded_fw, my_fw);
2260
2261                 /* abort nic load if version mismatch */
2262                 if (my_fw != loaded_fw) {
2263                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2264                                   loaded_fw, my_fw);
2265                         return -EBUSY;
2266                 }
2267         }
2268         return 0;
2269 }
2270
2271 /* returns the "mcp load_code" according to global load_count array */
2272 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2273 {
2274         int path = BP_PATH(bp);
2275
2276         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2277            path, load_count[path][0], load_count[path][1],
2278            load_count[path][2]);
2279         load_count[path][0]++;
2280         load_count[path][1 + port]++;
2281         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2282            path, load_count[path][0], load_count[path][1],
2283            load_count[path][2]);
2284         if (load_count[path][0] == 1)
2285                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2286         else if (load_count[path][1 + port] == 1)
2287                 return FW_MSG_CODE_DRV_LOAD_PORT;
2288         else
2289                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2290 }
2291
2292 /* mark PMF if applicable */
2293 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2294 {
2295         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2296             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2297             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2298                 bp->port.pmf = 1;
2299                 /* We need the barrier to ensure the ordering between the
2300                  * writing to bp->port.pmf here and reading it from the
2301                  * bnx2x_periodic_task().
2302                  */
2303                 smp_mb();
2304         } else {
2305                 bp->port.pmf = 0;
2306         }
2307
2308         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2309 }
2310
2311 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2312 {
2313         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2314              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2315             (bp->common.shmem2_base)) {
2316                 if (SHMEM2_HAS(bp, dcc_support))
2317                         SHMEM2_WR(bp, dcc_support,
2318                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2319                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2320                 if (SHMEM2_HAS(bp, afex_driver_support))
2321                         SHMEM2_WR(bp, afex_driver_support,
2322                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2323         }
2324
2325         /* Set AFEX default VLAN tag to an invalid value */
2326         bp->afex_def_vlan_tag = -1;
2327 }
2328
2329 /**
2330  * bnx2x_bz_fp - zero content of the fastpath structure.
2331  *
2332  * @bp:         driver handle
2333  * @index:      fastpath index to be zeroed
2334  *
2335  * Makes sure the contents of the bp->fp[index].napi is kept
2336  * intact.
2337  */
2338 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2339 {
2340         struct bnx2x_fastpath *fp = &bp->fp[index];
2341         int cos;
2342         struct napi_struct orig_napi = fp->napi;
2343         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2344
2345         /* bzero bnx2x_fastpath contents */
2346         if (fp->tpa_info)
2347                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2348                        sizeof(struct bnx2x_agg_info));
2349         memset(fp, 0, sizeof(*fp));
2350
2351         /* Restore the NAPI object as it has been already initialized */
2352         fp->napi = orig_napi;
2353         fp->tpa_info = orig_tpa_info;
2354         fp->bp = bp;
2355         fp->index = index;
2356         if (IS_ETH_FP(fp))
2357                 fp->max_cos = bp->max_cos;
2358         else
2359                 /* Special queues support only one CoS */
2360                 fp->max_cos = 1;
2361
2362         /* Init txdata pointers */
2363         if (IS_FCOE_FP(fp))
2364                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2365         if (IS_ETH_FP(fp))
2366                 for_each_cos_in_tx_queue(fp, cos)
2367                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2368                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2369
2370         /* set the tpa flag for each queue. The tpa flag determines the queue
2371          * minimal size so it must be set prior to queue memory allocation
2372          */
2373         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2374                                   (bp->flags & GRO_ENABLE_FLAG &&
2375                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2376         if (bp->flags & TPA_ENABLE_FLAG)
2377                 fp->mode = TPA_MODE_LRO;
2378         else if (bp->flags & GRO_ENABLE_FLAG)
2379                 fp->mode = TPA_MODE_GRO;
2380
2381         /* We don't want TPA on an FCoE L2 ring */
2382         if (IS_FCOE_FP(fp))
2383                 fp->disable_tpa = 1;
2384 }
2385
2386 int bnx2x_load_cnic(struct bnx2x *bp)
2387 {
2388         int i, rc, port = BP_PORT(bp);
2389
2390         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2391
2392         mutex_init(&bp->cnic_mutex);
2393
2394         if (IS_PF(bp)) {
2395                 rc = bnx2x_alloc_mem_cnic(bp);
2396                 if (rc) {
2397                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2398                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2399                 }
2400         }
2401
2402         rc = bnx2x_alloc_fp_mem_cnic(bp);
2403         if (rc) {
2404                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2405                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2406         }
2407
2408         /* Update the number of queues with the cnic queues */
2409         rc = bnx2x_set_real_num_queues(bp, 1);
2410         if (rc) {
2411                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2412                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2413         }
2414
2415         /* Add all CNIC NAPI objects */
2416         bnx2x_add_all_napi_cnic(bp);
2417         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2418         bnx2x_napi_enable_cnic(bp);
2419
2420         rc = bnx2x_init_hw_func_cnic(bp);
2421         if (rc)
2422                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2423
2424         bnx2x_nic_init_cnic(bp);
2425
2426         if (IS_PF(bp)) {
2427                 /* Enable Timer scan */
2428                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2429
2430                 /* setup cnic queues */
2431                 for_each_cnic_queue(bp, i) {
2432                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2433                         if (rc) {
2434                                 BNX2X_ERR("Queue setup failed\n");
2435                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2436                         }
2437                 }
2438         }
2439
2440         /* Initialize Rx filter. */
2441         netif_addr_lock_bh(bp->dev);
2442         bnx2x_set_rx_mode(bp->dev);
2443         netif_addr_unlock_bh(bp->dev);
2444
2445         /* re-read iscsi info */
2446         bnx2x_get_iscsi_info(bp);
2447         bnx2x_setup_cnic_irq_info(bp);
2448         bnx2x_setup_cnic_info(bp);
2449         bp->cnic_loaded = true;
2450         if (bp->state == BNX2X_STATE_OPEN)
2451                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2452
2453         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2454
2455         return 0;
2456
2457 #ifndef BNX2X_STOP_ON_ERROR
2458 load_error_cnic2:
2459         /* Disable Timer scan */
2460         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2461
2462 load_error_cnic1:
2463         bnx2x_napi_disable_cnic(bp);
2464         /* Update the number of queues without the cnic queues */
2465         rc = bnx2x_set_real_num_queues(bp, 0);
2466         if (rc)
2467                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2468 load_error_cnic0:
2469         BNX2X_ERR("CNIC-related load failed\n");
2470         bnx2x_free_fp_mem_cnic(bp);
2471         bnx2x_free_mem_cnic(bp);
2472         return rc;
2473 #endif /* ! BNX2X_STOP_ON_ERROR */
2474 }
2475
2476 /* must be called with rtnl_lock */
2477 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2478 {
2479         int port = BP_PORT(bp);
2480         int i, rc = 0, load_code = 0;
2481
2482         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2483         DP(NETIF_MSG_IFUP,
2484            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2485
2486 #ifdef BNX2X_STOP_ON_ERROR
2487         if (unlikely(bp->panic)) {
2488                 BNX2X_ERR("Can't load NIC when there is panic\n");
2489                 return -EPERM;
2490         }
2491 #endif
2492
2493         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2494
2495         /* zero the structure w/o any lock, before SP handler is initialized */
2496         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2497         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2498                 &bp->last_reported_link.link_report_flags);
2499
2500         if (IS_PF(bp))
2501                 /* must be called before memory allocation and HW init */
2502                 bnx2x_ilt_set_info(bp);
2503
2504         /*
2505          * Zero fastpath structures preserving invariants like napi, which are
2506          * allocated only once, fp index, max_cos, bp pointer.
2507          * Also set fp->disable_tpa and txdata_ptr.
2508          */
2509         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2510         for_each_queue(bp, i)
2511                 bnx2x_bz_fp(bp, i);
2512         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2513                                   bp->num_cnic_queues) *
2514                                   sizeof(struct bnx2x_fp_txdata));
2515
2516         bp->fcoe_init = false;
2517
2518         /* Set the receive queues buffer size */
2519         bnx2x_set_rx_buf_size(bp);
2520
2521         if (IS_PF(bp)) {
2522                 rc = bnx2x_alloc_mem(bp);
2523                 if (rc) {
2524                         BNX2X_ERR("Unable to allocate bp memory\n");
2525                         return rc;
2526                 }
2527         }
2528
2529         /* Allocated memory for FW statistics  */
2530         if (bnx2x_alloc_fw_stats_mem(bp))
2531                 LOAD_ERROR_EXIT(bp, load_error0);
2532
2533         /* need to be done after alloc mem, since it's self adjusting to amount
2534          * of memory available for RSS queues
2535          */
2536         rc = bnx2x_alloc_fp_mem(bp);
2537         if (rc) {
2538                 BNX2X_ERR("Unable to allocate memory for fps\n");
2539                 LOAD_ERROR_EXIT(bp, load_error0);
2540         }
2541
2542         /* request pf to initialize status blocks */
2543         if (IS_VF(bp)) {
2544                 rc = bnx2x_vfpf_init(bp);
2545                 if (rc)
2546                         LOAD_ERROR_EXIT(bp, load_error0);
2547         }
2548
2549         /* As long as bnx2x_alloc_mem() may possibly update
2550          * bp->num_queues, bnx2x_set_real_num_queues() should always
2551          * come after it. At this stage cnic queues are not counted.
2552          */
2553         rc = bnx2x_set_real_num_queues(bp, 0);
2554         if (rc) {
2555                 BNX2X_ERR("Unable to set real_num_queues\n");
2556                 LOAD_ERROR_EXIT(bp, load_error0);
2557         }
2558
2559         /* configure multi cos mappings in kernel.
2560          * this configuration may be overridden by a multi class queue
2561          * discipline or by a dcbx negotiation result.
2562          */
2563         bnx2x_setup_tc(bp->dev, bp->max_cos);
2564
2565         /* Add all NAPI objects */
2566         bnx2x_add_all_napi(bp);
2567         DP(NETIF_MSG_IFUP, "napi added\n");
2568         bnx2x_napi_enable(bp);
2569
2570         if (IS_PF(bp)) {
2571                 /* set pf load just before approaching the MCP */
2572                 bnx2x_set_pf_load(bp);
2573
2574                 /* if mcp exists send load request and analyze response */
2575                 if (!BP_NOMCP(bp)) {
2576                         /* attempt to load pf */
2577                         rc = bnx2x_nic_load_request(bp, &load_code);
2578                         if (rc)
2579                                 LOAD_ERROR_EXIT(bp, load_error1);
2580
2581                         /* what did mcp say? */
2582                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2583                         if (rc) {
2584                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2585                                 LOAD_ERROR_EXIT(bp, load_error2);
2586                         }
2587                 } else {
2588                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2589                 }
2590
2591                 /* mark pmf if applicable */
2592                 bnx2x_nic_load_pmf(bp, load_code);
2593
2594                 /* Init Function state controlling object */
2595                 bnx2x__init_func_obj(bp);
2596
2597                 /* Initialize HW */
2598                 rc = bnx2x_init_hw(bp, load_code);
2599                 if (rc) {
2600                         BNX2X_ERR("HW init failed, aborting\n");
2601                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2602                         LOAD_ERROR_EXIT(bp, load_error2);
2603                 }
2604         }
2605
2606         bnx2x_pre_irq_nic_init(bp);
2607
2608         /* Connect to IRQs */
2609         rc = bnx2x_setup_irqs(bp);
2610         if (rc) {
2611                 BNX2X_ERR("setup irqs failed\n");
2612                 if (IS_PF(bp))
2613                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2614                 LOAD_ERROR_EXIT(bp, load_error2);
2615         }
2616
2617         /* Init per-function objects */
2618         if (IS_PF(bp)) {
2619                 /* Setup NIC internals and enable interrupts */
2620                 bnx2x_post_irq_nic_init(bp, load_code);
2621
2622                 bnx2x_init_bp_objs(bp);
2623                 bnx2x_iov_nic_init(bp);
2624
2625                 /* Set AFEX default VLAN tag to an invalid value */
2626                 bp->afex_def_vlan_tag = -1;
2627                 bnx2x_nic_load_afex_dcc(bp, load_code);
2628                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2629                 rc = bnx2x_func_start(bp);
2630                 if (rc) {
2631                         BNX2X_ERR("Function start failed!\n");
2632                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2633
2634                         LOAD_ERROR_EXIT(bp, load_error3);
2635                 }
2636
2637                 /* Send LOAD_DONE command to MCP */
2638                 if (!BP_NOMCP(bp)) {
2639                         load_code = bnx2x_fw_command(bp,
2640                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2641                         if (!load_code) {
2642                                 BNX2X_ERR("MCP response failure, aborting\n");
2643                                 rc = -EBUSY;
2644                                 LOAD_ERROR_EXIT(bp, load_error3);
2645                         }
2646                 }
2647
2648                 /* initialize FW coalescing state machines in RAM */
2649                 bnx2x_update_coalesce(bp);
2650
2651                 /* setup the leading queue */
2652                 rc = bnx2x_setup_leading(bp);
2653                 if (rc) {
2654                         BNX2X_ERR("Setup leading failed!\n");
2655                         LOAD_ERROR_EXIT(bp, load_error3);
2656                 }
2657
2658                 /* set up the rest of the queues */
2659                 for_each_nondefault_eth_queue(bp, i) {
2660                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2661                         if (rc) {
2662                                 BNX2X_ERR("Queue setup failed\n");
2663                                 LOAD_ERROR_EXIT(bp, load_error3);
2664                         }
2665                 }
2666
2667                 /* setup rss */
2668                 rc = bnx2x_init_rss_pf(bp);
2669                 if (rc) {
2670                         BNX2X_ERR("PF RSS init failed\n");
2671                         LOAD_ERROR_EXIT(bp, load_error3);
2672                 }
2673
2674         } else { /* vf */
2675                 for_each_eth_queue(bp, i) {
2676                         rc = bnx2x_vfpf_setup_q(bp, i);
2677                         if (rc) {
2678                                 BNX2X_ERR("Queue setup failed\n");
2679                                 LOAD_ERROR_EXIT(bp, load_error3);
2680                         }
2681                 }
2682         }
2683
2684         /* Now when Clients are configured we are ready to work */
2685         bp->state = BNX2X_STATE_OPEN;
2686
2687         /* Configure a ucast MAC */
2688         if (IS_PF(bp))
2689                 rc = bnx2x_set_eth_mac(bp, true);
2690         else /* vf */
2691                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2692                                            true);
2693         if (rc) {
2694                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2695                 LOAD_ERROR_EXIT(bp, load_error3);
2696         }
2697
2698         if (IS_PF(bp) && bp->pending_max) {
2699                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2700                 bp->pending_max = 0;
2701         }
2702
2703         if (bp->port.pmf) {
2704                 rc = bnx2x_initial_phy_init(bp, load_mode);
2705                 if (rc)
2706                         LOAD_ERROR_EXIT(bp, load_error3);
2707         }
2708         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2709
2710         /* Start fast path */
2711
2712         /* Initialize Rx filter. */
2713         netif_addr_lock_bh(bp->dev);
2714         bnx2x_set_rx_mode(bp->dev);
2715         netif_addr_unlock_bh(bp->dev);
2716
2717         /* Start the Tx */
2718         switch (load_mode) {
2719         case LOAD_NORMAL:
2720                 /* Tx queue should be only re-enabled */
2721                 netif_tx_wake_all_queues(bp->dev);
2722                 break;
2723
2724         case LOAD_OPEN:
2725                 netif_tx_start_all_queues(bp->dev);
2726                 smp_mb__after_clear_bit();
2727                 break;
2728
2729         case LOAD_DIAG:
2730         case LOAD_LOOPBACK_EXT:
2731                 bp->state = BNX2X_STATE_DIAG;
2732                 break;
2733
2734         default:
2735                 break;
2736         }
2737
2738         if (bp->port.pmf)
2739                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2740         else
2741                 bnx2x__link_status_update(bp);
2742
2743         /* start the timer */
2744         mod_timer(&bp->timer, jiffies + bp->current_interval);
2745
2746         if (CNIC_ENABLED(bp))
2747                 bnx2x_load_cnic(bp);
2748
2749         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2750                 /* mark driver is loaded in shmem2 */
2751                 u32 val;
2752                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2753                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2754                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2755                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2756         }
2757
2758         /* Wait for all pending SP commands to complete */
2759         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2760                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2761                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2762                 return -EBUSY;
2763         }
2764
2765         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2766         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2767                 bnx2x_dcbx_init(bp, false);
2768
2769         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2770
2771         return 0;
2772
2773 #ifndef BNX2X_STOP_ON_ERROR
2774 load_error3:
2775         if (IS_PF(bp)) {
2776                 bnx2x_int_disable_sync(bp, 1);
2777
2778                 /* Clean queueable objects */
2779                 bnx2x_squeeze_objects(bp);
2780         }
2781
2782         /* Free SKBs, SGEs, TPA pool and driver internals */
2783         bnx2x_free_skbs(bp);
2784         for_each_rx_queue(bp, i)
2785                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2786
2787         /* Release IRQs */
2788         bnx2x_free_irq(bp);
2789 load_error2:
2790         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2791                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2792                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2793         }
2794
2795         bp->port.pmf = 0;
2796 load_error1:
2797         bnx2x_napi_disable(bp);
2798         bnx2x_del_all_napi(bp);
2799
2800         /* clear pf_load status, as it was already set */
2801         if (IS_PF(bp))
2802                 bnx2x_clear_pf_load(bp);
2803 load_error0:
2804         bnx2x_free_fp_mem(bp);
2805         bnx2x_free_fw_stats_mem(bp);
2806         bnx2x_free_mem(bp);
2807
2808         return rc;
2809 #endif /* ! BNX2X_STOP_ON_ERROR */
2810 }
2811
2812 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2813 {
2814         u8 rc = 0, cos, i;
2815
2816         /* Wait until tx fastpath tasks complete */
2817         for_each_tx_queue(bp, i) {
2818                 struct bnx2x_fastpath *fp = &bp->fp[i];
2819
2820                 for_each_cos_in_tx_queue(fp, cos)
2821                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2822                 if (rc)
2823                         return rc;
2824         }
2825         return 0;
2826 }
2827
2828 /* must be called with rtnl_lock */
2829 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2830 {
2831         int i;
2832         bool global = false;
2833
2834         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2835
2836         /* mark driver is unloaded in shmem2 */
2837         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2838                 u32 val;
2839                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2840                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2841                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2842         }
2843
2844         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2845             (bp->state == BNX2X_STATE_CLOSED ||
2846              bp->state == BNX2X_STATE_ERROR)) {
2847                 /* We can get here if the driver has been unloaded
2848                  * during parity error recovery and is either waiting for a
2849                  * leader to complete or for other functions to unload and
2850                  * then ifdown has been issued. In this case we want to
2851                  * unload and let other functions to complete a recovery
2852                  * process.
2853                  */
2854                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2855                 bp->is_leader = 0;
2856                 bnx2x_release_leader_lock(bp);
2857                 smp_mb();
2858
2859                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2860                 BNX2X_ERR("Can't unload in closed or error state\n");
2861                 return -EINVAL;
2862         }
2863
2864         /* Nothing to do during unload if previous bnx2x_nic_load()
2865          * have not completed successfully - all resources are released.
2866          *
2867          * we can get here only after unsuccessful ndo_* callback, during which
2868          * dev->IFF_UP flag is still on.
2869          */
2870         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2871                 return 0;
2872
2873         /* It's important to set the bp->state to the value different from
2874          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2875          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2876          */
2877         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2878         smp_mb();
2879
2880         if (CNIC_LOADED(bp))
2881                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2882
2883         /* Stop Tx */
2884         bnx2x_tx_disable(bp);
2885         netdev_reset_tc(bp->dev);
2886
2887         bp->rx_mode = BNX2X_RX_MODE_NONE;
2888
2889         del_timer_sync(&bp->timer);
2890
2891         if (IS_PF(bp)) {
2892                 /* Set ALWAYS_ALIVE bit in shmem */
2893                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2894                 bnx2x_drv_pulse(bp);
2895                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2896                 bnx2x_save_statistics(bp);
2897         }
2898
2899         /* wait till consumers catch up with producers in all queues */
2900         bnx2x_drain_tx_queues(bp);
2901
2902         /* if VF indicate to PF this function is going down (PF will delete sp
2903          * elements and clear initializations
2904          */
2905         if (IS_VF(bp))
2906                 bnx2x_vfpf_close_vf(bp);
2907         else if (unload_mode != UNLOAD_RECOVERY)
2908                 /* if this is a normal/close unload need to clean up chip*/
2909                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2910         else {
2911                 /* Send the UNLOAD_REQUEST to the MCP */
2912                 bnx2x_send_unload_req(bp, unload_mode);
2913
2914                 /* Prevent transactions to host from the functions on the
2915                  * engine that doesn't reset global blocks in case of global
2916                  * attention once global blocks are reset and gates are opened
2917                  * (the engine which leader will perform the recovery
2918                  * last).
2919                  */
2920                 if (!CHIP_IS_E1x(bp))
2921                         bnx2x_pf_disable(bp);
2922
2923                 /* Disable HW interrupts, NAPI */
2924                 bnx2x_netif_stop(bp, 1);
2925                 /* Delete all NAPI objects */
2926                 bnx2x_del_all_napi(bp);
2927                 if (CNIC_LOADED(bp))
2928                         bnx2x_del_all_napi_cnic(bp);
2929                 /* Release IRQs */
2930                 bnx2x_free_irq(bp);
2931
2932                 /* Report UNLOAD_DONE to MCP */
2933                 bnx2x_send_unload_done(bp, false);
2934         }
2935
2936         /*
2937          * At this stage no more interrupts will arrive so we may safely clean
2938          * the queueable objects here in case they failed to get cleaned so far.
2939          */
2940         if (IS_PF(bp))
2941                 bnx2x_squeeze_objects(bp);
2942
2943         /* There should be no more pending SP commands at this stage */
2944         bp->sp_state = 0;
2945
2946         bp->port.pmf = 0;
2947
2948         /* Free SKBs, SGEs, TPA pool and driver internals */
2949         bnx2x_free_skbs(bp);
2950         if (CNIC_LOADED(bp))
2951                 bnx2x_free_skbs_cnic(bp);
2952         for_each_rx_queue(bp, i)
2953                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2954
2955         bnx2x_free_fp_mem(bp);
2956         if (CNIC_LOADED(bp))
2957                 bnx2x_free_fp_mem_cnic(bp);
2958
2959         if (IS_PF(bp)) {
2960                 if (CNIC_LOADED(bp))
2961                         bnx2x_free_mem_cnic(bp);
2962                 bnx2x_free_mem(bp);
2963         }
2964         bp->state = BNX2X_STATE_CLOSED;
2965         bp->cnic_loaded = false;
2966
2967         /* Check if there are pending parity attentions. If there are - set
2968          * RECOVERY_IN_PROGRESS.
2969          */
2970         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2971                 bnx2x_set_reset_in_progress(bp);
2972
2973                 /* Set RESET_IS_GLOBAL if needed */
2974                 if (global)
2975                         bnx2x_set_reset_global(bp);
2976         }
2977
2978         /* The last driver must disable a "close the gate" if there is no
2979          * parity attention or "process kill" pending.
2980          */
2981         if (IS_PF(bp) &&
2982             !bnx2x_clear_pf_load(bp) &&
2983             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2984                 bnx2x_disable_close_the_gate(bp);
2985
2986         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2987
2988         return 0;
2989 }
2990
2991 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2992 {
2993         u16 pmcsr;
2994
2995         /* If there is no power capability, silently succeed */
2996         if (!bp->pm_cap) {
2997                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2998                 return 0;
2999         }
3000
3001         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
3002
3003         switch (state) {
3004         case PCI_D0:
3005                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3006                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3007                                        PCI_PM_CTRL_PME_STATUS));
3008
3009                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3010                         /* delay required during transition out of D3hot */
3011                         msleep(20);
3012                 break;
3013
3014         case PCI_D3hot:
3015                 /* If there are other clients above don't
3016                    shut down the power */
3017                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3018                         return 0;
3019                 /* Don't shut down the power for emulation and FPGA */
3020                 if (CHIP_REV_IS_SLOW(bp))
3021                         return 0;
3022
3023                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3024                 pmcsr |= 3;
3025
3026                 if (bp->wol)
3027                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3028
3029                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3030                                       pmcsr);
3031
3032                 /* No more memory access after this point until
3033                 * device is brought back to D0.
3034                 */
3035                 break;
3036
3037         default:
3038                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3039                 return -EINVAL;
3040         }
3041         return 0;
3042 }
3043
3044 /*
3045  * net_device service functions
3046  */
3047 int bnx2x_poll(struct napi_struct *napi, int budget)
3048 {
3049         int work_done = 0;
3050         u8 cos;
3051         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3052                                                  napi);
3053         struct bnx2x *bp = fp->bp;
3054
3055         while (1) {
3056 #ifdef BNX2X_STOP_ON_ERROR
3057                 if (unlikely(bp->panic)) {
3058                         napi_complete(napi);
3059                         return 0;
3060                 }
3061 #endif
3062                 if (!bnx2x_fp_lock_napi(fp))
3063                         return work_done;
3064
3065                 for_each_cos_in_tx_queue(fp, cos)
3066                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3067                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3068
3069                 if (bnx2x_has_rx_work(fp)) {
3070                         work_done += bnx2x_rx_int(fp, budget - work_done);
3071
3072                         /* must not complete if we consumed full budget */
3073                         if (work_done >= budget) {
3074                                 bnx2x_fp_unlock_napi(fp);
3075                                 break;
3076                         }
3077                 }
3078
3079                 /* Fall out from the NAPI loop if needed */
3080                 if (!bnx2x_fp_unlock_napi(fp) &&
3081                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3082
3083                         /* No need to update SB for FCoE L2 ring as long as
3084                          * it's connected to the default SB and the SB
3085                          * has been updated when NAPI was scheduled.
3086                          */
3087                         if (IS_FCOE_FP(fp)) {
3088                                 napi_complete(napi);
3089                                 break;
3090                         }
3091                         bnx2x_update_fpsb_idx(fp);
3092                         /* bnx2x_has_rx_work() reads the status block,
3093                          * thus we need to ensure that status block indices
3094                          * have been actually read (bnx2x_update_fpsb_idx)
3095                          * prior to this check (bnx2x_has_rx_work) so that
3096                          * we won't write the "newer" value of the status block
3097                          * to IGU (if there was a DMA right after
3098                          * bnx2x_has_rx_work and if there is no rmb, the memory
3099                          * reading (bnx2x_update_fpsb_idx) may be postponed
3100                          * to right before bnx2x_ack_sb). In this case there
3101                          * will never be another interrupt until there is
3102                          * another update of the status block, while there
3103                          * is still unhandled work.
3104                          */
3105                         rmb();
3106
3107                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3108                                 napi_complete(napi);
3109                                 /* Re-enable interrupts */
3110                                 DP(NETIF_MSG_RX_STATUS,
3111                                    "Update index to %d\n", fp->fp_hc_idx);
3112                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3113                                              le16_to_cpu(fp->fp_hc_idx),
3114                                              IGU_INT_ENABLE, 1);
3115                                 break;
3116                         }
3117                 }
3118         }
3119
3120         return work_done;
3121 }
3122
3123 #ifdef CONFIG_NET_LL_RX_POLL
3124 /* must be called with local_bh_disable()d */
3125 int bnx2x_low_latency_recv(struct napi_struct *napi)
3126 {
3127         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3128                                                  napi);
3129         struct bnx2x *bp = fp->bp;
3130         int found = 0;
3131
3132         if ((bp->state == BNX2X_STATE_CLOSED) ||
3133             (bp->state == BNX2X_STATE_ERROR) ||
3134             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3135                 return LL_FLUSH_FAILED;
3136
3137         if (!bnx2x_fp_lock_poll(fp))
3138                 return LL_FLUSH_BUSY;
3139
3140         if (bnx2x_has_rx_work(fp)) {
3141                 bnx2x_update_fpsb_idx(fp);
3142                 found = bnx2x_rx_int(fp, 4);
3143         }
3144
3145         bnx2x_fp_unlock_poll(fp);
3146
3147         return found;
3148 }
3149 #endif
3150
3151 /* we split the first BD into headers and data BDs
3152  * to ease the pain of our fellow microcode engineers
3153  * we use one mapping for both BDs
3154  */
3155 static u16 bnx2x_tx_split(struct bnx2x *bp,
3156                           struct bnx2x_fp_txdata *txdata,
3157                           struct sw_tx_bd *tx_buf,
3158                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3159                           u16 bd_prod)
3160 {
3161         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3162         struct eth_tx_bd *d_tx_bd;
3163         dma_addr_t mapping;
3164         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3165
3166         /* first fix first BD */
3167         h_tx_bd->nbytes = cpu_to_le16(hlen);
3168
3169         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3170            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3171
3172         /* now get a new data BD
3173          * (after the pbd) and fill it */
3174         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3175         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3176
3177         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3178                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3179
3180         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3181         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3182         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3183
3184         /* this marks the BD as one that has no individual mapping */
3185         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3186
3187         DP(NETIF_MSG_TX_QUEUED,
3188            "TSO split data size is %d (%x:%x)\n",
3189            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3190
3191         /* update tx_bd */
3192         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3193
3194         return bd_prod;
3195 }
3196
3197 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3198 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3199 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3200 {
3201         __sum16 tsum = (__force __sum16) csum;
3202
3203         if (fix > 0)
3204                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3205                                   csum_partial(t_header - fix, fix, 0)));
3206
3207         else if (fix < 0)
3208                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3209                                   csum_partial(t_header, -fix, 0)));
3210
3211         return bswab16(tsum);
3212 }
3213
3214 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3215 {
3216         u32 rc;
3217         __u8 prot = 0;
3218         __be16 protocol;
3219
3220         if (skb->ip_summed != CHECKSUM_PARTIAL)
3221                 return XMIT_PLAIN;
3222
3223         protocol = vlan_get_protocol(skb);
3224         if (protocol == htons(ETH_P_IPV6)) {
3225                 rc = XMIT_CSUM_V6;
3226                 prot = ipv6_hdr(skb)->nexthdr;
3227         } else {
3228                 rc = XMIT_CSUM_V4;
3229                 prot = ip_hdr(skb)->protocol;
3230         }
3231
3232         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3233                 if (inner_ip_hdr(skb)->version == 6) {
3234                         rc |= XMIT_CSUM_ENC_V6;
3235                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3236                                 rc |= XMIT_CSUM_TCP;
3237                 } else {
3238                         rc |= XMIT_CSUM_ENC_V4;
3239                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3240                                 rc |= XMIT_CSUM_TCP;
3241                 }
3242         }
3243         if (prot == IPPROTO_TCP)
3244                 rc |= XMIT_CSUM_TCP;
3245
3246         if (skb_is_gso_v6(skb)) {
3247                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3248                 if (rc & XMIT_CSUM_ENC)
3249                         rc |= XMIT_GSO_ENC_V6;
3250         } else if (skb_is_gso(skb)) {
3251                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3252                 if (rc & XMIT_CSUM_ENC)
3253                         rc |= XMIT_GSO_ENC_V4;
3254         }
3255
3256         return rc;
3257 }
3258
3259 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3260 /* check if packet requires linearization (packet is too fragmented)
3261    no need to check fragmentation if page size > 8K (there will be no
3262    violation to FW restrictions) */
3263 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3264                              u32 xmit_type)
3265 {
3266         int to_copy = 0;
3267         int hlen = 0;
3268         int first_bd_sz = 0;
3269
3270         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3271         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3272
3273                 if (xmit_type & XMIT_GSO) {
3274                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3275                         /* Check if LSO packet needs to be copied:
3276                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3277                         int wnd_size = MAX_FETCH_BD - 3;
3278                         /* Number of windows to check */
3279                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3280                         int wnd_idx = 0;
3281                         int frag_idx = 0;
3282                         u32 wnd_sum = 0;
3283
3284                         /* Headers length */
3285                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3286                                 tcp_hdrlen(skb);
3287
3288                         /* Amount of data (w/o headers) on linear part of SKB*/
3289                         first_bd_sz = skb_headlen(skb) - hlen;
3290
3291                         wnd_sum  = first_bd_sz;
3292
3293                         /* Calculate the first sum - it's special */
3294                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3295                                 wnd_sum +=
3296                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3297
3298                         /* If there was data on linear skb data - check it */
3299                         if (first_bd_sz > 0) {
3300                                 if (unlikely(wnd_sum < lso_mss)) {
3301                                         to_copy = 1;
3302                                         goto exit_lbl;
3303                                 }
3304
3305                                 wnd_sum -= first_bd_sz;
3306                         }
3307
3308                         /* Others are easier: run through the frag list and
3309                            check all windows */
3310                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3311                                 wnd_sum +=
3312                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3313
3314                                 if (unlikely(wnd_sum < lso_mss)) {
3315                                         to_copy = 1;
3316                                         break;
3317                                 }
3318                                 wnd_sum -=
3319                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3320                         }
3321                 } else {
3322                         /* in non-LSO too fragmented packet should always
3323                            be linearized */
3324                         to_copy = 1;
3325                 }
3326         }
3327
3328 exit_lbl:
3329         if (unlikely(to_copy))
3330                 DP(NETIF_MSG_TX_QUEUED,
3331                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3332                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3333                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3334
3335         return to_copy;
3336 }
3337 #endif
3338
3339 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3340                                  u32 xmit_type)
3341 {
3342         struct ipv6hdr *ipv6;
3343
3344         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3345                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3346                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3347
3348         if (xmit_type & XMIT_GSO_ENC_V6)
3349                 ipv6 = inner_ipv6_hdr(skb);
3350         else if (xmit_type & XMIT_GSO_V6)
3351                 ipv6 = ipv6_hdr(skb);
3352         else
3353                 ipv6 = NULL;
3354
3355         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3356                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3357 }
3358
3359 /**
3360  * bnx2x_set_pbd_gso - update PBD in GSO case.
3361  *
3362  * @skb:        packet skb
3363  * @pbd:        parse BD
3364  * @xmit_type:  xmit flags
3365  */
3366 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3367                               struct eth_tx_parse_bd_e1x *pbd,
3368                               struct eth_tx_start_bd *tx_start_bd,
3369                               u32 xmit_type)
3370 {
3371         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3372         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3373         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3374
3375         if (xmit_type & XMIT_GSO_V4) {
3376                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3377                 pbd->tcp_pseudo_csum =
3378                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3379                                                    ip_hdr(skb)->daddr,
3380                                                    0, IPPROTO_TCP, 0));
3381
3382                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3383                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3384         } else {
3385                 pbd->tcp_pseudo_csum =
3386                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3387                                                  &ipv6_hdr(skb)->daddr,
3388                                                  0, IPPROTO_TCP, 0));
3389         }
3390
3391         pbd->global_data |=
3392                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3393 }
3394
3395 /**
3396  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3397  *
3398  * @bp:                 driver handle
3399  * @skb:                packet skb
3400  * @parsing_data:       data to be updated
3401  * @xmit_type:          xmit flags
3402  *
3403  * 57712/578xx related, when skb has encapsulation
3404  */
3405 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3406                                  u32 *parsing_data, u32 xmit_type)
3407 {
3408         *parsing_data |=
3409                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3410                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3411                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3412
3413         if (xmit_type & XMIT_CSUM_TCP) {
3414                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3415                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3416                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3417
3418                 return skb_inner_transport_header(skb) +
3419                         inner_tcp_hdrlen(skb) - skb->data;
3420         }
3421
3422         /* We support checksum offload for TCP and UDP only.
3423          * No need to pass the UDP header length - it's a constant.
3424          */
3425         return skb_inner_transport_header(skb) +
3426                 sizeof(struct udphdr) - skb->data;
3427 }
3428
3429 /**
3430  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3431  *
3432  * @bp:                 driver handle
3433  * @skb:                packet skb
3434  * @parsing_data:       data to be updated
3435  * @xmit_type:          xmit flags
3436  *
3437  * 57712/578xx related
3438  */
3439 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3440                                 u32 *parsing_data, u32 xmit_type)
3441 {
3442         *parsing_data |=
3443                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3444                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3445                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3446
3447         if (xmit_type & XMIT_CSUM_TCP) {
3448                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3449                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3450                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3451
3452                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3453         }
3454         /* We support checksum offload for TCP and UDP only.
3455          * No need to pass the UDP header length - it's a constant.
3456          */
3457         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3458 }
3459
3460 /* set FW indication according to inner or outer protocols if tunneled */
3461 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3462                                struct eth_tx_start_bd *tx_start_bd,
3463                                u32 xmit_type)
3464 {
3465         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3466
3467         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3468                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3469
3470         if (!(xmit_type & XMIT_CSUM_TCP))
3471                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3472 }
3473
3474 /**
3475  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3476  *
3477  * @bp:         driver handle
3478  * @skb:        packet skb
3479  * @pbd:        parse BD to be updated
3480  * @xmit_type:  xmit flags
3481  */
3482 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3483                              struct eth_tx_parse_bd_e1x *pbd,
3484                              u32 xmit_type)
3485 {
3486         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3487
3488         /* for now NS flag is not used in Linux */
3489         pbd->global_data =
3490                 cpu_to_le16(hlen |
3491                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3492                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3493
3494         pbd->ip_hlen_w = (skb_transport_header(skb) -
3495                         skb_network_header(skb)) >> 1;
3496
3497         hlen += pbd->ip_hlen_w;
3498
3499         /* We support checksum offload for TCP and UDP only */
3500         if (xmit_type & XMIT_CSUM_TCP)
3501                 hlen += tcp_hdrlen(skb) / 2;
3502         else
3503                 hlen += sizeof(struct udphdr) / 2;
3504
3505         pbd->total_hlen_w = cpu_to_le16(hlen);
3506         hlen = hlen*2;
3507
3508         if (xmit_type & XMIT_CSUM_TCP) {
3509                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3510
3511         } else {
3512                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3513
3514                 DP(NETIF_MSG_TX_QUEUED,
3515                    "hlen %d  fix %d  csum before fix %x\n",
3516                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3517
3518                 /* HW bug: fixup the CSUM */
3519                 pbd->tcp_pseudo_csum =
3520                         bnx2x_csum_fix(skb_transport_header(skb),
3521                                        SKB_CS(skb), fix);
3522
3523                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3524                    pbd->tcp_pseudo_csum);
3525         }
3526
3527         return hlen;
3528 }
3529
3530 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3531                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3532                                       struct eth_tx_parse_2nd_bd *pbd2,
3533                                       u16 *global_data,
3534                                       u32 xmit_type)
3535 {
3536         u16 hlen_w = 0;
3537         u8 outerip_off, outerip_len = 0;
3538
3539         /* from outer IP to transport */
3540         hlen_w = (skb_inner_transport_header(skb) -
3541                   skb_network_header(skb)) >> 1;
3542
3543         /* transport len */
3544         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3545
3546         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3547
3548         /* outer IP header info */
3549         if (xmit_type & XMIT_CSUM_V4) {
3550                 struct iphdr *iph = ip_hdr(skb);
3551                 pbd2->fw_ip_csum_wo_len_flags_frag =
3552                         bswab16(csum_fold((~iph->check) -
3553                                           iph->tot_len - iph->frag_off));
3554         } else {
3555                 pbd2->fw_ip_hdr_to_payload_w =
3556                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3557         }
3558
3559         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3560
3561         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3562
3563         if (xmit_type & XMIT_GSO_V4) {
3564                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3565
3566                 pbd_e2->data.tunnel_data.pseudo_csum =
3567                         bswab16(~csum_tcpudp_magic(
3568                                         inner_ip_hdr(skb)->saddr,
3569                                         inner_ip_hdr(skb)->daddr,
3570                                         0, IPPROTO_TCP, 0));
3571
3572                 outerip_len = ip_hdr(skb)->ihl << 1;
3573         } else {
3574                 pbd_e2->data.tunnel_data.pseudo_csum =
3575                         bswab16(~csum_ipv6_magic(
3576                                         &inner_ipv6_hdr(skb)->saddr,
3577                                         &inner_ipv6_hdr(skb)->daddr,
3578                                         0, IPPROTO_TCP, 0));
3579         }
3580
3581         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3582
3583         *global_data |=
3584                 outerip_off |
3585                 (!!(xmit_type & XMIT_CSUM_V6) <<
3586                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3587                 (outerip_len <<
3588                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3589                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3590                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3591
3592         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3593                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3594                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3595         }
3596 }
3597
3598 /* called with netif_tx_lock
3599  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3600  * netif_wake_queue()
3601  */
3602 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3603 {
3604         struct bnx2x *bp = netdev_priv(dev);
3605
3606         struct netdev_queue *txq;
3607         struct bnx2x_fp_txdata *txdata;
3608         struct sw_tx_bd *tx_buf;
3609         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3610         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3611         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3612         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3613         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3614         u32 pbd_e2_parsing_data = 0;
3615         u16 pkt_prod, bd_prod;
3616         int nbd, txq_index;
3617         dma_addr_t mapping;
3618         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3619         int i;
3620         u8 hlen = 0;
3621         __le16 pkt_size = 0;
3622         struct ethhdr *eth;
3623         u8 mac_type = UNICAST_ADDRESS;
3624
3625 #ifdef BNX2X_STOP_ON_ERROR
3626         if (unlikely(bp->panic))
3627                 return NETDEV_TX_BUSY;
3628 #endif
3629
3630         txq_index = skb_get_queue_mapping(skb);
3631         txq = netdev_get_tx_queue(dev, txq_index);
3632
3633         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3634
3635         txdata = &bp->bnx2x_txq[txq_index];
3636
3637         /* enable this debug print to view the transmission queue being used
3638         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3639            txq_index, fp_index, txdata_index); */
3640
3641         /* enable this debug print to view the transmission details
3642         DP(NETIF_MSG_TX_QUEUED,
3643            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3644            txdata->cid, fp_index, txdata_index, txdata, fp); */
3645
3646         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3647                         skb_shinfo(skb)->nr_frags +
3648                         BDS_PER_TX_PKT +
3649                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3650                 /* Handle special storage cases separately */
3651                 if (txdata->tx_ring_size == 0) {
3652                         struct bnx2x_eth_q_stats *q_stats =
3653                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3654                         q_stats->driver_filtered_tx_pkt++;
3655                         dev_kfree_skb(skb);
3656                         return NETDEV_TX_OK;
3657                 }
3658                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3659                 netif_tx_stop_queue(txq);
3660                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3661
3662                 return NETDEV_TX_BUSY;
3663         }
3664
3665         DP(NETIF_MSG_TX_QUEUED,
3666            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3667            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3668            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3669            skb->len);
3670
3671         eth = (struct ethhdr *)skb->data;
3672
3673         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3674         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3675                 if (is_broadcast_ether_addr(eth->h_dest))
3676                         mac_type = BROADCAST_ADDRESS;
3677                 else
3678                         mac_type = MULTICAST_ADDRESS;
3679         }
3680
3681 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3682         /* First, check if we need to linearize the skb (due to FW
3683            restrictions). No need to check fragmentation if page size > 8K
3684            (there will be no violation to FW restrictions) */
3685         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3686                 /* Statistics of linearization */
3687                 bp->lin_cnt++;
3688                 if (skb_linearize(skb) != 0) {
3689                         DP(NETIF_MSG_TX_QUEUED,
3690                            "SKB linearization failed - silently dropping this SKB\n");
3691                         dev_kfree_skb_any(skb);
3692                         return NETDEV_TX_OK;
3693                 }
3694         }
3695 #endif
3696         /* Map skb linear data for DMA */
3697         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3698                                  skb_headlen(skb), DMA_TO_DEVICE);
3699         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3700                 DP(NETIF_MSG_TX_QUEUED,
3701                    "SKB mapping failed - silently dropping this SKB\n");
3702                 dev_kfree_skb_any(skb);
3703                 return NETDEV_TX_OK;
3704         }
3705         /*
3706         Please read carefully. First we use one BD which we mark as start,
3707         then we have a parsing info BD (used for TSO or xsum),
3708         and only then we have the rest of the TSO BDs.
3709         (don't forget to mark the last one as last,
3710         and to unmap only AFTER you write to the BD ...)
3711         And above all, all pdb sizes are in words - NOT DWORDS!
3712         */
3713
3714         /* get current pkt produced now - advance it just before sending packet
3715          * since mapping of pages may fail and cause packet to be dropped
3716          */
3717         pkt_prod = txdata->tx_pkt_prod;
3718         bd_prod = TX_BD(txdata->tx_bd_prod);
3719
3720         /* get a tx_buf and first BD
3721          * tx_start_bd may be changed during SPLIT,
3722          * but first_bd will always stay first
3723          */
3724         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3725         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3726         first_bd = tx_start_bd;
3727
3728         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3729
3730         /* header nbd: indirectly zero other flags! */
3731         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3732
3733         /* remember the first BD of the packet */
3734         tx_buf->first_bd = txdata->tx_bd_prod;
3735         tx_buf->skb = skb;
3736         tx_buf->flags = 0;
3737
3738         DP(NETIF_MSG_TX_QUEUED,
3739            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3740            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3741
3742         if (vlan_tx_tag_present(skb)) {
3743                 tx_start_bd->vlan_or_ethertype =
3744                     cpu_to_le16(vlan_tx_tag_get(skb));
3745                 tx_start_bd->bd_flags.as_bitfield |=
3746                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3747         } else {
3748                 /* when transmitting in a vf, start bd must hold the ethertype
3749                  * for fw to enforce it
3750                  */
3751                 if (IS_VF(bp))
3752                         tx_start_bd->vlan_or_ethertype =
3753                                 cpu_to_le16(ntohs(eth->h_proto));
3754                 else
3755                         /* used by FW for packet accounting */
3756                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3757         }
3758
3759         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3760
3761         /* turn on parsing and get a BD */
3762         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3763
3764         if (xmit_type & XMIT_CSUM)
3765                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3766
3767         if (!CHIP_IS_E1x(bp)) {
3768                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3769                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3770
3771                 if (xmit_type & XMIT_CSUM_ENC) {
3772                         u16 global_data = 0;
3773
3774                         /* Set PBD in enc checksum offload case */
3775                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3776                                                       &pbd_e2_parsing_data,
3777                                                       xmit_type);
3778
3779                         /* turn on 2nd parsing and get a BD */
3780                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3781
3782                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3783
3784                         memset(pbd2, 0, sizeof(*pbd2));
3785
3786                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3787                                 (skb_inner_network_header(skb) -
3788                                  skb->data) >> 1;
3789
3790                         if (xmit_type & XMIT_GSO_ENC)
3791                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3792                                                           &global_data,
3793                                                           xmit_type);
3794
3795                         pbd2->global_data = cpu_to_le16(global_data);
3796
3797                         /* add addition parse BD indication to start BD */
3798                         SET_FLAG(tx_start_bd->general_data,
3799                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3800                         /* set encapsulation flag in start BD */
3801                         SET_FLAG(tx_start_bd->general_data,
3802                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3803                         nbd++;
3804                 } else if (xmit_type & XMIT_CSUM) {
3805                         /* Set PBD in checksum offload case w/o encapsulation */
3806                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3807                                                      &pbd_e2_parsing_data,
3808                                                      xmit_type);
3809                 }
3810
3811                 /* Add the macs to the parsing BD this is a vf */
3812                 if (IS_VF(bp)) {
3813                         /* override GRE parameters in BD */
3814                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3815                                               &pbd_e2->data.mac_addr.src_mid,
3816                                               &pbd_e2->data.mac_addr.src_lo,
3817                                               eth->h_source);
3818
3819                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3820                                               &pbd_e2->data.mac_addr.dst_mid,
3821                                               &pbd_e2->data.mac_addr.dst_lo,
3822                                               eth->h_dest);
3823                 }
3824
3825                 SET_FLAG(pbd_e2_parsing_data,
3826                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3827         } else {
3828                 u16 global_data = 0;
3829                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3830                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3831                 /* Set PBD in checksum offload case */
3832                 if (xmit_type & XMIT_CSUM)
3833                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3834
3835                 SET_FLAG(global_data,
3836                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3837                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3838         }
3839
3840         /* Setup the data pointer of the first BD of the packet */
3841         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3842         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3843         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3844         pkt_size = tx_start_bd->nbytes;
3845
3846         DP(NETIF_MSG_TX_QUEUED,
3847            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3848            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3849            le16_to_cpu(tx_start_bd->nbytes),
3850            tx_start_bd->bd_flags.as_bitfield,
3851            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3852
3853         if (xmit_type & XMIT_GSO) {
3854
3855                 DP(NETIF_MSG_TX_QUEUED,
3856                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3857                    skb->len, hlen, skb_headlen(skb),
3858                    skb_shinfo(skb)->gso_size);
3859
3860                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3861
3862                 if (unlikely(skb_headlen(skb) > hlen)) {
3863                         nbd++;
3864                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3865                                                  &tx_start_bd, hlen,
3866                                                  bd_prod);
3867                 }
3868                 if (!CHIP_IS_E1x(bp))
3869                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3870                                              xmit_type);
3871                 else
3872                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3873         }
3874
3875         /* Set the PBD's parsing_data field if not zero
3876          * (for the chips newer than 57711).
3877          */
3878         if (pbd_e2_parsing_data)
3879                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3880
3881         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3882
3883         /* Handle fragmented skb */
3884         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3885                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3886
3887                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3888                                            skb_frag_size(frag), DMA_TO_DEVICE);
3889                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3890                         unsigned int pkts_compl = 0, bytes_compl = 0;
3891
3892                         DP(NETIF_MSG_TX_QUEUED,
3893                            "Unable to map page - dropping packet...\n");
3894
3895                         /* we need unmap all buffers already mapped
3896                          * for this SKB;
3897                          * first_bd->nbd need to be properly updated
3898                          * before call to bnx2x_free_tx_pkt
3899                          */
3900                         first_bd->nbd = cpu_to_le16(nbd);
3901                         bnx2x_free_tx_pkt(bp, txdata,
3902                                           TX_BD(txdata->tx_pkt_prod),
3903                                           &pkts_compl, &bytes_compl);
3904                         return NETDEV_TX_OK;
3905                 }
3906
3907                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3908                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3909                 if (total_pkt_bd == NULL)
3910                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3911
3912                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3913                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3914                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3915                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3916                 nbd++;
3917
3918                 DP(NETIF_MSG_TX_QUEUED,
3919                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3920                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3921                    le16_to_cpu(tx_data_bd->nbytes));
3922         }
3923
3924         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3925
3926         /* update with actual num BDs */
3927         first_bd->nbd = cpu_to_le16(nbd);
3928
3929         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3930
3931         /* now send a tx doorbell, counting the next BD
3932          * if the packet contains or ends with it
3933          */
3934         if (TX_BD_POFF(bd_prod) < nbd)
3935                 nbd++;
3936
3937         /* total_pkt_bytes should be set on the first data BD if
3938          * it's not an LSO packet and there is more than one
3939          * data BD. In this case pkt_size is limited by an MTU value.
3940          * However we prefer to set it for an LSO packet (while we don't
3941          * have to) in order to save some CPU cycles in a none-LSO
3942          * case, when we much more care about them.
3943          */
3944         if (total_pkt_bd != NULL)
3945                 total_pkt_bd->total_pkt_bytes = pkt_size;
3946
3947         if (pbd_e1x)
3948                 DP(NETIF_MSG_TX_QUEUED,
3949                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3950                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3951                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3952                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3953                     le16_to_cpu(pbd_e1x->total_hlen_w));
3954         if (pbd_e2)
3955                 DP(NETIF_MSG_TX_QUEUED,
3956                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3957                    pbd_e2,
3958                    pbd_e2->data.mac_addr.dst_hi,
3959                    pbd_e2->data.mac_addr.dst_mid,
3960                    pbd_e2->data.mac_addr.dst_lo,
3961                    pbd_e2->data.mac_addr.src_hi,
3962                    pbd_e2->data.mac_addr.src_mid,
3963                    pbd_e2->data.mac_addr.src_lo,
3964                    pbd_e2->parsing_data);
3965         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3966
3967         netdev_tx_sent_queue(txq, skb->len);
3968
3969         skb_tx_timestamp(skb);
3970
3971         txdata->tx_pkt_prod++;
3972         /*
3973          * Make sure that the BD data is updated before updating the producer
3974          * since FW might read the BD right after the producer is updated.
3975          * This is only applicable for weak-ordered memory model archs such
3976          * as IA-64. The following barrier is also mandatory since FW will
3977          * assumes packets must have BDs.
3978          */
3979         wmb();
3980
3981         txdata->tx_db.data.prod += nbd;
3982         barrier();
3983
3984         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3985
3986         mmiowb();
3987
3988         txdata->tx_bd_prod += nbd;
3989
3990         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3991                 netif_tx_stop_queue(txq);
3992
3993                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3994                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3995                  * fp->bd_tx_cons */
3996                 smp_mb();
3997
3998                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3999                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4000                         netif_tx_wake_queue(txq);
4001         }
4002         txdata->tx_pkt++;
4003
4004         return NETDEV_TX_OK;
4005 }
4006
4007 /**
4008  * bnx2x_setup_tc - routine to configure net_device for multi tc
4009  *
4010  * @netdev: net device to configure
4011  * @tc: number of traffic classes to enable
4012  *
4013  * callback connected to the ndo_setup_tc function pointer
4014  */
4015 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4016 {
4017         int cos, prio, count, offset;
4018         struct bnx2x *bp = netdev_priv(dev);
4019
4020         /* setup tc must be called under rtnl lock */
4021         ASSERT_RTNL();
4022
4023         /* no traffic classes requested. Aborting */
4024         if (!num_tc) {
4025                 netdev_reset_tc(dev);
4026                 return 0;
4027         }
4028
4029         /* requested to support too many traffic classes */
4030         if (num_tc > bp->max_cos) {
4031                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4032                           num_tc, bp->max_cos);
4033                 return -EINVAL;
4034         }
4035
4036         /* declare amount of supported traffic classes */
4037         if (netdev_set_num_tc(dev, num_tc)) {
4038                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4039                 return -EINVAL;
4040         }
4041
4042         /* configure priority to traffic class mapping */
4043         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4044                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4045                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4046                    "mapping priority %d to tc %d\n",
4047                    prio, bp->prio_to_cos[prio]);
4048         }
4049
4050         /* Use this configuration to differentiate tc0 from other COSes
4051            This can be used for ets or pfc, and save the effort of setting
4052            up a multio class queue disc or negotiating DCBX with a switch
4053         netdev_set_prio_tc_map(dev, 0, 0);
4054         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4055         for (prio = 1; prio < 16; prio++) {
4056                 netdev_set_prio_tc_map(dev, prio, 1);
4057                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4058         } */
4059
4060         /* configure traffic class to transmission queue mapping */
4061         for (cos = 0; cos < bp->max_cos; cos++) {
4062                 count = BNX2X_NUM_ETH_QUEUES(bp);
4063                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4064                 netdev_set_tc_queue(dev, cos, count, offset);
4065                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4066                    "mapping tc %d to offset %d count %d\n",
4067                    cos, offset, count);
4068         }
4069
4070         return 0;
4071 }
4072
4073 /* called with rtnl_lock */
4074 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4075 {
4076         struct sockaddr *addr = p;
4077         struct bnx2x *bp = netdev_priv(dev);
4078         int rc = 0;
4079
4080         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4081                 BNX2X_ERR("Requested MAC address is not valid\n");
4082                 return -EINVAL;
4083         }
4084
4085         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4086             !is_zero_ether_addr(addr->sa_data)) {
4087                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4088                 return -EINVAL;
4089         }
4090
4091         if (netif_running(dev))  {
4092                 rc = bnx2x_set_eth_mac(bp, false);
4093                 if (rc)
4094                         return rc;
4095         }
4096
4097         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4098
4099         if (netif_running(dev))
4100                 rc = bnx2x_set_eth_mac(bp, true);
4101
4102         return rc;
4103 }
4104
4105 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4106 {
4107         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4108         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4109         u8 cos;
4110
4111         /* Common */
4112
4113         if (IS_FCOE_IDX(fp_index)) {
4114                 memset(sb, 0, sizeof(union host_hc_status_block));
4115                 fp->status_blk_mapping = 0;
4116         } else {
4117                 /* status blocks */
4118                 if (!CHIP_IS_E1x(bp))
4119                         BNX2X_PCI_FREE(sb->e2_sb,
4120                                        bnx2x_fp(bp, fp_index,
4121                                                 status_blk_mapping),
4122                                        sizeof(struct host_hc_status_block_e2));
4123                 else
4124                         BNX2X_PCI_FREE(sb->e1x_sb,
4125                                        bnx2x_fp(bp, fp_index,
4126                                                 status_blk_mapping),
4127                                        sizeof(struct host_hc_status_block_e1x));
4128         }
4129
4130         /* Rx */
4131         if (!skip_rx_queue(bp, fp_index)) {
4132                 bnx2x_free_rx_bds(fp);
4133
4134                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4135                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4136                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4137                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4138                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4139
4140                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4141                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4142                                sizeof(struct eth_fast_path_rx_cqe) *
4143                                NUM_RCQ_BD);
4144
4145                 /* SGE ring */
4146                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4147                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4148                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4149                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4150         }
4151
4152         /* Tx */
4153         if (!skip_tx_queue(bp, fp_index)) {
4154                 /* fastpath tx rings: tx_buf tx_desc */
4155                 for_each_cos_in_tx_queue(fp, cos) {
4156                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4157
4158                         DP(NETIF_MSG_IFDOWN,
4159                            "freeing tx memory of fp %d cos %d cid %d\n",
4160                            fp_index, cos, txdata->cid);
4161
4162                         BNX2X_FREE(txdata->tx_buf_ring);
4163                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4164                                 txdata->tx_desc_mapping,
4165                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4166                 }
4167         }
4168         /* end of fastpath */
4169 }
4170
4171 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4172 {
4173         int i;
4174         for_each_cnic_queue(bp, i)
4175                 bnx2x_free_fp_mem_at(bp, i);
4176 }
4177
4178 void bnx2x_free_fp_mem(struct bnx2x *bp)
4179 {
4180         int i;
4181         for_each_eth_queue(bp, i)
4182                 bnx2x_free_fp_mem_at(bp, i);
4183 }
4184
4185 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4186 {
4187         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4188         if (!CHIP_IS_E1x(bp)) {
4189                 bnx2x_fp(bp, index, sb_index_values) =
4190                         (__le16 *)status_blk.e2_sb->sb.index_values;
4191                 bnx2x_fp(bp, index, sb_running_index) =
4192                         (__le16 *)status_blk.e2_sb->sb.running_index;
4193         } else {
4194                 bnx2x_fp(bp, index, sb_index_values) =
4195                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4196                 bnx2x_fp(bp, index, sb_running_index) =
4197                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4198         }
4199 }
4200
4201 /* Returns the number of actually allocated BDs */
4202 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4203                               int rx_ring_size)
4204 {
4205         struct bnx2x *bp = fp->bp;
4206         u16 ring_prod, cqe_ring_prod;
4207         int i, failure_cnt = 0;
4208
4209         fp->rx_comp_cons = 0;
4210         cqe_ring_prod = ring_prod = 0;
4211
4212         /* This routine is called only during fo init so
4213          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4214          */
4215         for (i = 0; i < rx_ring_size; i++) {
4216                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4217                         failure_cnt++;
4218                         continue;
4219                 }
4220                 ring_prod = NEXT_RX_IDX(ring_prod);
4221                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4222                 WARN_ON(ring_prod <= (i - failure_cnt));
4223         }
4224
4225         if (failure_cnt)
4226                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4227                           i - failure_cnt, fp->index);
4228
4229         fp->rx_bd_prod = ring_prod;
4230         /* Limit the CQE producer by the CQE ring size */
4231         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4232                                cqe_ring_prod);
4233         fp->rx_pkt = fp->rx_calls = 0;
4234
4235         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4236
4237         return i - failure_cnt;
4238 }
4239
4240 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4241 {
4242         int i;
4243
4244         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4245                 struct eth_rx_cqe_next_page *nextpg;
4246
4247                 nextpg = (struct eth_rx_cqe_next_page *)
4248                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4249                 nextpg->addr_hi =
4250                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4251                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4252                 nextpg->addr_lo =
4253                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4254                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4255         }
4256 }
4257
4258 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4259 {
4260         union host_hc_status_block *sb;
4261         struct bnx2x_fastpath *fp = &bp->fp[index];
4262         int ring_size = 0;
4263         u8 cos;
4264         int rx_ring_size = 0;
4265
4266         if (!bp->rx_ring_size &&
4267             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4268                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4269                 bp->rx_ring_size = rx_ring_size;
4270         } else if (!bp->rx_ring_size) {
4271                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4272
4273                 if (CHIP_IS_E3(bp)) {
4274                         u32 cfg = SHMEM_RD(bp,
4275                                            dev_info.port_hw_config[BP_PORT(bp)].
4276                                            default_cfg);
4277
4278                         /* Decrease ring size for 1G functions */
4279                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4280                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4281                                 rx_ring_size /= 10;
4282                 }
4283
4284                 /* allocate at least number of buffers required by FW */
4285                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4286                                      MIN_RX_SIZE_TPA, rx_ring_size);
4287
4288                 bp->rx_ring_size = rx_ring_size;
4289         } else /* if rx_ring_size specified - use it */
4290                 rx_ring_size = bp->rx_ring_size;
4291
4292         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4293
4294         /* Common */
4295         sb = &bnx2x_fp(bp, index, status_blk);
4296
4297         if (!IS_FCOE_IDX(index)) {
4298                 /* status blocks */
4299                 if (!CHIP_IS_E1x(bp))
4300                         BNX2X_PCI_ALLOC(sb->e2_sb,
4301                                 &bnx2x_fp(bp, index, status_blk_mapping),
4302                                 sizeof(struct host_hc_status_block_e2));
4303                 else
4304                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4305                                 &bnx2x_fp(bp, index, status_blk_mapping),
4306                             sizeof(struct host_hc_status_block_e1x));
4307         }
4308
4309         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4310          * set shortcuts for it.
4311          */
4312         if (!IS_FCOE_IDX(index))
4313                 set_sb_shortcuts(bp, index);
4314
4315         /* Tx */
4316         if (!skip_tx_queue(bp, index)) {
4317                 /* fastpath tx rings: tx_buf tx_desc */
4318                 for_each_cos_in_tx_queue(fp, cos) {
4319                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4320
4321                         DP(NETIF_MSG_IFUP,
4322                            "allocating tx memory of fp %d cos %d\n",
4323                            index, cos);
4324
4325                         BNX2X_ALLOC(txdata->tx_buf_ring,
4326                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4327                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4328                                 &txdata->tx_desc_mapping,
4329                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4330                 }
4331         }
4332
4333         /* Rx */
4334         if (!skip_rx_queue(bp, index)) {
4335                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4336                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4337                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4338                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4339                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4340                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4341
4342                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4343                                 &bnx2x_fp(bp, index, rx_comp_mapping),
4344                                 sizeof(struct eth_fast_path_rx_cqe) *
4345                                 NUM_RCQ_BD);
4346
4347                 /* SGE ring */
4348                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4349                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4350                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4351                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4352                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4353                 /* RX BD ring */
4354                 bnx2x_set_next_page_rx_bd(fp);
4355
4356                 /* CQ ring */
4357                 bnx2x_set_next_page_rx_cq(fp);
4358
4359                 /* BDs */
4360                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4361                 if (ring_size < rx_ring_size)
4362                         goto alloc_mem_err;
4363         }
4364
4365         return 0;
4366
4367 /* handles low memory cases */
4368 alloc_mem_err:
4369         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4370                                                 index, ring_size);
4371         /* FW will drop all packets if queue is not big enough,
4372          * In these cases we disable the queue
4373          * Min size is different for OOO, TPA and non-TPA queues
4374          */
4375         if (ring_size < (fp->disable_tpa ?
4376                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4377                         /* release memory allocated for this queue */
4378                         bnx2x_free_fp_mem_at(bp, index);
4379                         return -ENOMEM;
4380         }
4381         return 0;
4382 }
4383
4384 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4385 {
4386         if (!NO_FCOE(bp))
4387                 /* FCoE */
4388                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4389                         /* we will fail load process instead of mark
4390                          * NO_FCOE_FLAG
4391                          */
4392                         return -ENOMEM;
4393
4394         return 0;
4395 }
4396
4397 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4398 {
4399         int i;
4400
4401         /* 1. Allocate FP for leading - fatal if error
4402          * 2. Allocate RSS - fix number of queues if error
4403          */
4404
4405         /* leading */
4406         if (bnx2x_alloc_fp_mem_at(bp, 0))
4407                 return -ENOMEM;
4408
4409         /* RSS */
4410         for_each_nondefault_eth_queue(bp, i)
4411                 if (bnx2x_alloc_fp_mem_at(bp, i))
4412                         break;
4413
4414         /* handle memory failures */
4415         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4416                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4417
4418                 WARN_ON(delta < 0);
4419                 bnx2x_shrink_eth_fp(bp, delta);
4420                 if (CNIC_SUPPORT(bp))
4421                         /* move non eth FPs next to last eth FP
4422                          * must be done in that order
4423                          * FCOE_IDX < FWD_IDX < OOO_IDX
4424                          */
4425
4426                         /* move FCoE fp even NO_FCOE_FLAG is on */
4427                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4428                 bp->num_ethernet_queues -= delta;
4429                 bp->num_queues = bp->num_ethernet_queues +
4430                                  bp->num_cnic_queues;
4431                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4432                           bp->num_queues + delta, bp->num_queues);
4433         }
4434
4435         return 0;
4436 }
4437
4438 void bnx2x_free_mem_bp(struct bnx2x *bp)
4439 {
4440         int i;
4441
4442         for (i = 0; i < bp->fp_array_size; i++)
4443                 kfree(bp->fp[i].tpa_info);
4444         kfree(bp->fp);
4445         kfree(bp->sp_objs);
4446         kfree(bp->fp_stats);
4447         kfree(bp->bnx2x_txq);
4448         kfree(bp->msix_table);
4449         kfree(bp->ilt);
4450 }
4451
4452 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4453 {
4454         struct bnx2x_fastpath *fp;
4455         struct msix_entry *tbl;
4456         struct bnx2x_ilt *ilt;
4457         int msix_table_size = 0;
4458         int fp_array_size, txq_array_size;
4459         int i;
4460
4461         /*
4462          * The biggest MSI-X table we might need is as a maximum number of fast
4463          * path IGU SBs plus default SB (for PF only).
4464          */
4465         msix_table_size = bp->igu_sb_cnt;
4466         if (IS_PF(bp))
4467                 msix_table_size++;
4468         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4469
4470         /* fp array: RSS plus CNIC related L2 queues */
4471         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4472         bp->fp_array_size = fp_array_size;
4473         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4474
4475         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4476         if (!fp)
4477                 goto alloc_err;
4478         for (i = 0; i < bp->fp_array_size; i++) {
4479                 fp[i].tpa_info =
4480                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4481                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4482                 if (!(fp[i].tpa_info))
4483                         goto alloc_err;
4484         }
4485
4486         bp->fp = fp;
4487
4488         /* allocate sp objs */
4489         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4490                               GFP_KERNEL);
4491         if (!bp->sp_objs)
4492                 goto alloc_err;
4493
4494         /* allocate fp_stats */
4495         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4496                                GFP_KERNEL);
4497         if (!bp->fp_stats)
4498                 goto alloc_err;
4499
4500         /* Allocate memory for the transmission queues array */
4501         txq_array_size =
4502                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4503         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4504
4505         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4506                                 GFP_KERNEL);
4507         if (!bp->bnx2x_txq)
4508                 goto alloc_err;
4509
4510         /* msix table */
4511         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4512         if (!tbl)
4513                 goto alloc_err;
4514         bp->msix_table = tbl;
4515
4516         /* ilt */
4517         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4518         if (!ilt)
4519                 goto alloc_err;
4520         bp->ilt = ilt;
4521
4522         return 0;
4523 alloc_err:
4524         bnx2x_free_mem_bp(bp);
4525         return -ENOMEM;
4526 }
4527
4528 int bnx2x_reload_if_running(struct net_device *dev)
4529 {
4530         struct bnx2x *bp = netdev_priv(dev);
4531
4532         if (unlikely(!netif_running(dev)))
4533                 return 0;
4534
4535         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4536         return bnx2x_nic_load(bp, LOAD_NORMAL);
4537 }
4538
4539 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4540 {
4541         u32 sel_phy_idx = 0;
4542         if (bp->link_params.num_phys <= 1)
4543                 return INT_PHY;
4544
4545         if (bp->link_vars.link_up) {
4546                 sel_phy_idx = EXT_PHY1;
4547                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4548                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4549                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4550                         sel_phy_idx = EXT_PHY2;
4551         } else {
4552
4553                 switch (bnx2x_phy_selection(&bp->link_params)) {
4554                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4555                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4556                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4557                        sel_phy_idx = EXT_PHY1;
4558                        break;
4559                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4560                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4561                        sel_phy_idx = EXT_PHY2;
4562                        break;
4563                 }
4564         }
4565
4566         return sel_phy_idx;
4567 }
4568 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4569 {
4570         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4571         /*
4572          * The selected activated PHY is always after swapping (in case PHY
4573          * swapping is enabled). So when swapping is enabled, we need to reverse
4574          * the configuration
4575          */
4576
4577         if (bp->link_params.multi_phy_config &
4578             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4579                 if (sel_phy_idx == EXT_PHY1)
4580                         sel_phy_idx = EXT_PHY2;
4581                 else if (sel_phy_idx == EXT_PHY2)
4582                         sel_phy_idx = EXT_PHY1;
4583         }
4584         return LINK_CONFIG_IDX(sel_phy_idx);
4585 }
4586
4587 #ifdef NETDEV_FCOE_WWNN
4588 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4589 {
4590         struct bnx2x *bp = netdev_priv(dev);
4591         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4592
4593         switch (type) {
4594         case NETDEV_FCOE_WWNN:
4595                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4596                                 cp->fcoe_wwn_node_name_lo);
4597                 break;
4598         case NETDEV_FCOE_WWPN:
4599                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4600                                 cp->fcoe_wwn_port_name_lo);
4601                 break;
4602         default:
4603                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4604                 return -EINVAL;
4605         }
4606
4607         return 0;
4608 }
4609 #endif
4610
4611 /* called with rtnl_lock */
4612 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4613 {
4614         struct bnx2x *bp = netdev_priv(dev);
4615
4616         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4617                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4618                 return -EAGAIN;
4619         }
4620
4621         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4622             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4623                 BNX2X_ERR("Can't support requested MTU size\n");
4624                 return -EINVAL;
4625         }
4626
4627         /* This does not race with packet allocation
4628          * because the actual alloc size is
4629          * only updated as part of load
4630          */
4631         dev->mtu = new_mtu;
4632
4633         return bnx2x_reload_if_running(dev);
4634 }
4635
4636 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4637                                      netdev_features_t features)
4638 {
4639         struct bnx2x *bp = netdev_priv(dev);
4640
4641         /* TPA requires Rx CSUM offloading */
4642         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4643                 features &= ~NETIF_F_LRO;
4644                 features &= ~NETIF_F_GRO;
4645         }
4646
4647         return features;
4648 }
4649
4650 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4651 {
4652         struct bnx2x *bp = netdev_priv(dev);
4653         u32 flags = bp->flags;
4654         u32 changes;
4655         bool bnx2x_reload = false;
4656
4657         if (features & NETIF_F_LRO)
4658                 flags |= TPA_ENABLE_FLAG;
4659         else
4660                 flags &= ~TPA_ENABLE_FLAG;
4661
4662         if (features & NETIF_F_GRO)
4663                 flags |= GRO_ENABLE_FLAG;
4664         else
4665                 flags &= ~GRO_ENABLE_FLAG;
4666
4667         if (features & NETIF_F_LOOPBACK) {
4668                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4669                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4670                         bnx2x_reload = true;
4671                 }
4672         } else {
4673                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4674                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4675                         bnx2x_reload = true;
4676                 }
4677         }
4678
4679         changes = flags ^ bp->flags;
4680
4681         /* if GRO is changed while LRO is enabled, don't force a reload */
4682         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4683                 changes &= ~GRO_ENABLE_FLAG;
4684
4685         if (changes)
4686                 bnx2x_reload = true;
4687
4688         bp->flags = flags;
4689
4690         if (bnx2x_reload) {
4691                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4692                         return bnx2x_reload_if_running(dev);
4693                 /* else: bnx2x_nic_load() will be called at end of recovery */
4694         }
4695
4696         return 0;
4697 }
4698
4699 void bnx2x_tx_timeout(struct net_device *dev)
4700 {
4701         struct bnx2x *bp = netdev_priv(dev);
4702
4703 #ifdef BNX2X_STOP_ON_ERROR
4704         if (!bp->panic)
4705                 bnx2x_panic();
4706 #endif
4707
4708         smp_mb__before_clear_bit();
4709         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4710         smp_mb__after_clear_bit();
4711
4712         /* This allows the netif to be shutdown gracefully before resetting */
4713         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4714 }
4715
4716 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4717 {
4718         struct net_device *dev = pci_get_drvdata(pdev);
4719         struct bnx2x *bp;
4720
4721         if (!dev) {
4722                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4723                 return -ENODEV;
4724         }
4725         bp = netdev_priv(dev);
4726
4727         rtnl_lock();
4728
4729         pci_save_state(pdev);
4730
4731         if (!netif_running(dev)) {
4732                 rtnl_unlock();
4733                 return 0;
4734         }
4735
4736         netif_device_detach(dev);
4737
4738         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4739
4740         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4741
4742         rtnl_unlock();
4743
4744         return 0;
4745 }
4746
4747 int bnx2x_resume(struct pci_dev *pdev)
4748 {
4749         struct net_device *dev = pci_get_drvdata(pdev);
4750         struct bnx2x *bp;
4751         int rc;
4752
4753         if (!dev) {
4754                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4755                 return -ENODEV;
4756         }
4757         bp = netdev_priv(dev);
4758
4759         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4760                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4761                 return -EAGAIN;
4762         }
4763
4764         rtnl_lock();
4765
4766         pci_restore_state(pdev);
4767
4768         if (!netif_running(dev)) {
4769                 rtnl_unlock();
4770                 return 0;
4771         }
4772
4773         bnx2x_set_power_state(bp, PCI_D0);
4774         netif_device_attach(dev);
4775
4776         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4777
4778         rtnl_unlock();
4779
4780         return rc;
4781 }
4782
4783 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4784                               u32 cid)
4785 {
4786         /* ustorm cxt validation */
4787         cxt->ustorm_ag_context.cdu_usage =
4788                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4789                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4790         /* xcontext validation */
4791         cxt->xstorm_ag_context.cdu_reserved =
4792                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4793                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4794 }
4795
4796 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4797                                     u8 fw_sb_id, u8 sb_index,
4798                                     u8 ticks)
4799 {
4800         u32 addr = BAR_CSTRORM_INTMEM +
4801                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4802         REG_WR8(bp, addr, ticks);
4803         DP(NETIF_MSG_IFUP,
4804            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4805            port, fw_sb_id, sb_index, ticks);
4806 }
4807
4808 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4809                                     u16 fw_sb_id, u8 sb_index,
4810                                     u8 disable)
4811 {
4812         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4813         u32 addr = BAR_CSTRORM_INTMEM +
4814                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4815         u8 flags = REG_RD8(bp, addr);
4816         /* clear and set */
4817         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4818         flags |= enable_flag;
4819         REG_WR8(bp, addr, flags);
4820         DP(NETIF_MSG_IFUP,
4821            "port %x fw_sb_id %d sb_index %d disable %d\n",
4822            port, fw_sb_id, sb_index, disable);
4823 }
4824
4825 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4826                                     u8 sb_index, u8 disable, u16 usec)
4827 {
4828         int port = BP_PORT(bp);
4829         u8 ticks = usec / BNX2X_BTR;
4830
4831         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4832
4833         disable = disable ? 1 : (usec ? 0 : 1);
4834         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4835 }