drivers/net/ethernet/cavium/liquidio/lio_main.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/module.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pci.h>
  21 #include <linux/firmware.h>
  22 #include <net/vxlan.h>
  23 #include <linux/kthread.h>
  24 #include "liquidio_common.h"
  25 #include "octeon_droq.h"
  26 #include "octeon_iq.h"
  27 #include "response_manager.h"
  28 #include "octeon_device.h"
  29 #include "octeon_nic.h"
  30 #include "octeon_main.h"
  31 #include "octeon_network.h"
  32 #include "cn66xx_regs.h"
  33 #include "cn66xx_device.h"
  34 #include "cn68xx_device.h"
  35 #include "cn23xx_pf_device.h"
  36 #include "liquidio_image.h"
  37 #include "lio_vf_rep.h"
  38
  39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
  40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
  41 MODULE_LICENSE("GPL");
  42 MODULE_VERSION(LIQUIDIO_VERSION);
  43 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
  44                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
  46                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
  48                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  49 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
  50                 "_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
  51
  52 static int ddr_timeout = 10000;
  53 module_param(ddr_timeout, int, 0644);
  54 MODULE_PARM_DESC(ddr_timeout,
  55                  "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
  56
  57 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  58
  59 static int debug = -1;
  60 module_param(debug, int, 0644);
  61 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
  62
  63 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
  64 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
  65 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
  66
  67 static u32 console_bitmask;
  68 module_param(console_bitmask, int, 0644);
  69 MODULE_PARM_DESC(console_bitmask,
  70                  "Bitmask indicating which consoles have debug output redirected to syslog.");
  71
  72 /**
  73  * \brief determines if a given console has debug enabled.
  74  * @param console console to check
  75  * @returns  1 = enabled. 0 otherwise
  76  */
  77 static int octeon_console_debug_enabled(u32 console)
  78 {
  79         return (console_bitmask >> (console)) & 0x1;
  80 }
  81
  82 /* Polling interval for determining when NIC application is alive */
  83 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
  84
  85 /* runtime link query interval */
  86 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
  87 /* update localtime to octeon firmware every 60 seconds.
  88  * make firmware to use same time reference, so that it will be easy to
  89  * correlate firmware logged events/errors with host events, for debugging.
  90  */
  91 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
  92
  93 /* time to wait for possible in-flight requests in milliseconds */
  94 #define WAIT_INFLIGHT_REQUEST   msecs_to_jiffies(1000)
  95
  96 struct lio_trusted_vf_ctx {
  97         struct completion complete;
  98         int status;
  99 };
 100
 101 struct oct_link_status_resp {
 102         u64 rh;
 103         struct oct_link_info link_info;
 104         u64 status;
 105 };
 106
 107 struct oct_timestamp_resp {
 108         u64 rh;
 109         u64 timestamp;
 110         u64 status;
 111 };
 112
 113 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
 114
 115 union tx_info {
 116         u64 u64;
 117         struct {
 118 #ifdef __BIG_ENDIAN_BITFIELD
 119                 u16 gso_size;
 120                 u16 gso_segs;
 121                 u32 reserved;
 122 #else
 123                 u32 reserved;
 124                 u16 gso_segs;
 125                 u16 gso_size;
 126 #endif
 127         } s;
 128 };
 129
 130 /** Octeon device properties to be used by the NIC module.
 131  * Each octeon device in the system will be represented
 132  * by this structure in the NIC module.
 133  */
 134
 135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
 136 #define OCTNIC_GSO_MAX_SIZE                                                    \
 137         (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
 138
 139 struct handshake {
 140         struct completion init;
 141         struct completion started;
 142         struct pci_dev *pci_dev;
 143         int init_ok;
 144         int started_ok;
 145 };
 146
 147 #ifdef CONFIG_PCI_IOV
 148 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
 149 #endif
 150
 151 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
 152                                     char *prefix, char *suffix);
 153
 154 static int octeon_device_init(struct octeon_device *);
 155 static int liquidio_stop(struct net_device *netdev);
 156 static void liquidio_remove(struct pci_dev *pdev);
 157 static int liquidio_probe(struct pci_dev *pdev,
 158                           const struct pci_device_id *ent);
 159 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
 160                                       int linkstate);
 161
 162 static struct handshake handshake[MAX_OCTEON_DEVICES];
 163 static struct completion first_stage;
 164
 165 static void octeon_droq_bh(unsigned long pdev)
 166 {
 167         int q_no;
 168         int reschedule = 0;
 169         struct octeon_device *oct = (struct octeon_device *)pdev;
 170         struct octeon_device_priv *oct_priv =
 171                 (struct octeon_device_priv *)oct->priv;
 172
 173         for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
 174                 if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
 175                         continue;
 176                 reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
 177                                                           MAX_PACKET_BUDGET);
 178                 lio_enable_irq(oct->droq[q_no], NULL);
 179
 180                 if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
 181                         /* set time and cnt interrupt thresholds for this DROQ
 182                          * for NAPI
 183                          */
 184                         int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
 185
 186                         octeon_write_csr64(
 187                             oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
 188                             0x5700000040ULL);
 189                         octeon_write_csr64(
 190                             oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
 191                 }
 192         }
 193
 194         if (reschedule)
 195                 tasklet_schedule(&oct_priv->droq_tasklet);
 196 }
 197
 198 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
 199 {
 200         struct octeon_device_priv *oct_priv =
 201                 (struct octeon_device_priv *)oct->priv;
 202         int retry = 100, pkt_cnt = 0, pending_pkts = 0;
 203         int i;
 204
 205         do {
 206                 pending_pkts = 0;
 207
 208                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
 209                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
 210                                 continue;
 211                         pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
 212                 }
 213                 if (pkt_cnt > 0) {
 214                         pending_pkts += pkt_cnt;
 215                         tasklet_schedule(&oct_priv->droq_tasklet);
 216                 }
 217                 pkt_cnt = 0;
 218                 schedule_timeout_uninterruptible(1);
 219
 220         } while (retry-- && pending_pkts);
 221
 222         return pkt_cnt;
 223 }
 224
 225 /**
 226  * \brief Forces all IO queues off on a given device
 227  * @param oct Pointer to Octeon device
 228  */
 229 static void force_io_queues_off(struct octeon_device *oct)
 230 {
 231         if ((oct->chip_id == OCTEON_CN66XX) ||
 232             (oct->chip_id == OCTEON_CN68XX)) {
 233                 /* Reset the Enable bits for Input Queues. */
 234                 octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
 235
 236                 /* Reset the Enable bits for Output Queues. */
 237                 octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
 238         }
 239 }
 240
 241 /**
 242  * \brief Cause device to go quiet so it can be safely removed/reset/etc
 243  * @param oct Pointer to Octeon device
 244  */
 245 static inline void pcierror_quiesce_device(struct octeon_device *oct)
 246 {
 247         int i;
 248
 249         /* Disable the input and output queues now. No more packets will
 250          * arrive from Octeon, but we should wait for all packet processing
 251          * to finish.
 252          */
 253         force_io_queues_off(oct);
 254
 255         /* To allow for in-flight requests */
 256         schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
 257
 258         if (wait_for_pending_requests(oct))
 259                 dev_err(&oct->pci_dev->dev, "There were pending requests\n");
 260
 261         /* Force all requests waiting to be fetched by OCTEON to complete. */
 262         for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
 263                 struct octeon_instr_queue *iq;
 264
 265                 if (!(oct->io_qmask.iq & BIT_ULL(i)))
 266                         continue;
 267                 iq = oct->instr_queue[i];
 268
 269                 if (atomic_read(&iq->instr_pending)) {
 270                         spin_lock_bh(&iq->lock);
 271                         iq->fill_cnt = 0;
 272                         iq->octeon_read_index = iq->host_write_index;
 273                         iq->stats.instr_processed +=
 274                                 atomic_read(&iq->instr_pending);
 275                         lio_process_iq_request_list(oct, iq, 0);
 276                         spin_unlock_bh(&iq->lock);
 277                 }
 278         }
 279
 280         /* Force all pending ordered list requests to time out. */
 281         lio_process_ordered_list(oct, 1);
 282
 283         /* We do not need to wait for output queue packets to be processed. */
 284 }
 285
 286 /**
 287  * \brief Cleanup PCI AER uncorrectable error status
 288  * @param dev Pointer to PCI device
 289  */
 290 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
 291 {
 292         int pos = 0x100;
 293         u32 status, mask;
 294
 295         pr_info("%s :\n", __func__);
 296
 297         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 298         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
 299         if (dev->error_state == pci_channel_io_normal)
 300                 status &= ~mask;        /* Clear corresponding nonfatal bits */
 301         else
 302                 status &= mask;         /* Clear corresponding fatal bits */
 303         pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
 304 }
 305
 306 /**
 307  * \brief Stop all PCI IO to a given device
 308  * @param dev Pointer to Octeon device
 309  */
 310 static void stop_pci_io(struct octeon_device *oct)
 311 {
 312         /* No more instructions will be forwarded. */
 313         atomic_set(&oct->status, OCT_DEV_IN_RESET);
 314
 315         pci_disable_device(oct->pci_dev);
 316
 317         /* Disable interrupts  */
 318         oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
 319
 320         pcierror_quiesce_device(oct);
 321
 322         /* Release the interrupt line */
 323         free_irq(oct->pci_dev->irq, oct);
 324
 325         if (oct->flags & LIO_FLAG_MSI_ENABLED)
 326                 pci_disable_msi(oct->pci_dev);
 327
 328         dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 329                 lio_get_state_string(&oct->status));
 330
 331         /* making it a common function for all OCTEON models */
 332         cleanup_aer_uncorrect_error_status(oct->pci_dev);
 333 }
 334
 335 /**
 336  * \brief called when PCI error is detected
 337  * @param pdev Pointer to PCI device
 338  * @param state The current pci connection state
 339  *
 340  * This function is called after a PCI bus error affecting
 341  * this device has been detected.
 342  */
 343 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
 344                                                      pci_channel_state_t state)
 345 {
 346         struct octeon_device *oct = pci_get_drvdata(pdev);
 347
 348         /* Non-correctable Non-fatal errors */
 349         if (state == pci_channel_io_normal) {
 350                 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
 351                 cleanup_aer_uncorrect_error_status(oct->pci_dev);
 352                 return PCI_ERS_RESULT_CAN_RECOVER;
 353         }
 354
 355         /* Non-correctable Fatal errors */
 356         dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
 357         stop_pci_io(oct);
 358
 359         /* Always return a DISCONNECT. There is no support for recovery but only
 360          * for a clean shutdown.
 361          */
 362         return PCI_ERS_RESULT_DISCONNECT;
 363 }
 364
 365 /**
 366  * \brief mmio handler
 367  * @param pdev Pointer to PCI device
 368  */
 369 static pci_ers_result_t liquidio_pcie_mmio_enabled(
 370                                 struct pci_dev *pdev __attribute__((unused)))
 371 {
 372         /* We should never hit this since we never ask for a reset for a Fatal
 373          * Error. We always return DISCONNECT in io_error above.
 374          * But play safe and return RECOVERED for now.
 375          */
 376         return PCI_ERS_RESULT_RECOVERED;
 377 }
 378
 379 /**
 380  * \brief called after the pci bus has been reset.
 381  * @param pdev Pointer to PCI device
 382  *
 383  * Restart the card from scratch, as if from a cold-boot. Implementation
 384  * resembles the first-half of the octeon_resume routine.
 385  */
 386 static pci_ers_result_t liquidio_pcie_slot_reset(
 387                                 struct pci_dev *pdev __attribute__((unused)))
 388 {
 389         /* We should never hit this since we never ask for a reset for a Fatal
 390          * Error. We always return DISCONNECT in io_error above.
 391          * But play safe and return RECOVERED for now.
 392          */
 393         return PCI_ERS_RESULT_RECOVERED;
 394 }
 395
 396 /**
 397  * \brief called when traffic can start flowing again.
 398  * @param pdev Pointer to PCI device
 399  *
 400  * This callback is called when the error recovery driver tells us that
 401  * its OK to resume normal operation. Implementation resembles the
 402  * second-half of the octeon_resume routine.
 403  */
 404 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
 405 {
 406         /* Nothing to be done here. */
 407 }
 408
 409 #ifdef CONFIG_PM
 410 /**
 411  * \brief called when suspending
 412  * @param pdev Pointer to PCI device
 413  * @param state state to suspend to
 414  */
 415 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
 416                             pm_message_t state __attribute__((unused)))
 417 {
 418         return 0;
 419 }
 420
 421 /**
 422  * \brief called when resuming
 423  * @param pdev Pointer to PCI device
 424  */
 425 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
 426 {
 427         return 0;
 428 }
 429 #endif
 430
 431 /* For PCI-E Advanced Error Recovery (AER) Interface */
 432 static const struct pci_error_handlers liquidio_err_handler = {
 433         .error_detected = liquidio_pcie_error_detected,
 434         .mmio_enabled   = liquidio_pcie_mmio_enabled,
 435         .slot_reset     = liquidio_pcie_slot_reset,
 436         .resume         = liquidio_pcie_resume,
 437 };
 438
 439 static const struct pci_device_id liquidio_pci_tbl[] = {
 440         {       /* 68xx */
 441                 PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 442         },
 443         {       /* 66xx */
 444                 PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 445         },
 446         {       /* 23xx pf */
 447                 PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
 448         },
 449         {
 450                 0, 0, 0, 0, 0, 0, 0
 451         }
 452 };
 453 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 454
 455 static struct pci_driver liquidio_pci_driver = {
 456         .name           = "LiquidIO",
 457         .id_table       = liquidio_pci_tbl,
 458         .probe          = liquidio_probe,
 459         .remove         = liquidio_remove,
 460         .err_handler    = &liquidio_err_handler,    /* For AER */
 461
 462 #ifdef CONFIG_PM
 463         .suspend        = liquidio_suspend,
 464         .resume         = liquidio_resume,
 465 #endif
 466 #ifdef CONFIG_PCI_IOV
 467         .sriov_configure = liquidio_enable_sriov,
 468 #endif
 469 };
 470
 471 /**
 472  * \brief register PCI driver
 473  */
 474 static int liquidio_init_pci(void)
 475 {
 476         return pci_register_driver(&liquidio_pci_driver);
 477 }
 478
 479 /**
 480  * \brief unregister PCI driver
 481  */
 482 static void liquidio_deinit_pci(void)
 483 {
 484         pci_unregister_driver(&liquidio_pci_driver);
 485 }
 486
 487 /**
 488  * \brief Check Tx queue status, and take appropriate action
 489  * @param lio per-network private data
 490  * @returns 0 if full, number of queues woken up otherwise
 491  */
 492 static inline int check_txq_status(struct lio *lio)
 493 {
 494         int numqs = lio->netdev->real_num_tx_queues;
 495         int ret_val = 0;
 496         int q, iq;
 497
 498         /* check each sub-queue state */
 499         for (q = 0; q < numqs; q++) {
 500                 iq = lio->linfo.txpciq[q %
 501                         lio->oct_dev->num_iqs].s.q_no;
 502                 if (octnet_iq_is_full(lio->oct_dev, iq))
 503                         continue;
 504                 if (__netif_subqueue_stopped(lio->netdev, q)) {
 505                         netif_wake_subqueue(lio->netdev, q);
 506                         INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
 507                                                   tx_restart, 1);
 508                         ret_val++;
 509                 }
 510         }
 511
 512         return ret_val;
 513 }
 514
 515 /**
 516  * \brief Print link information
 517  * @param netdev network device
 518  */
 519 static void print_link_info(struct net_device *netdev)
 520 {
 521         struct lio *lio = GET_LIO(netdev);
 522
 523         if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
 524             ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
 525                 struct oct_link_info *linfo = &lio->linfo;
 526
 527                 if (linfo->link.s.link_up) {
 528                         netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
 529                                    linfo->link.s.speed,
 530                                    (linfo->link.s.duplex) ? "Full" : "Half");
 531                 } else {
 532                         netif_info(lio, link, lio->netdev, "Link Down\n");
 533                 }
 534         }
 535 }
 536
 537 /**
 538  * \brief Routine to notify MTU change
 539  * @param work work_struct data structure
 540  */
 541 static void octnet_link_status_change(struct work_struct *work)
 542 {
 543         struct cavium_wk *wk = (struct cavium_wk *)work;
 544         struct lio *lio = (struct lio *)wk->ctxptr;
 545
 546         /* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
 547          * this API is invoked only when new max-MTU of the interface is
 548          * less than current MTU.
 549          */
 550         rtnl_lock();
 551         dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
 552         rtnl_unlock();
 553 }
 554
 555 /**
 556  * \brief Sets up the mtu status change work
 557  * @param netdev network device
 558  */
 559 static inline int setup_link_status_change_wq(struct net_device *netdev)
 560 {
 561         struct lio *lio = GET_LIO(netdev);
 562         struct octeon_device *oct = lio->oct_dev;
 563
 564         lio->link_status_wq.wq = alloc_workqueue("link-status",
 565                                                  WQ_MEM_RECLAIM, 0);
 566         if (!lio->link_status_wq.wq) {
 567                 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
 568                 return -1;
 569         }
 570         INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
 571                           octnet_link_status_change);
 572         lio->link_status_wq.wk.ctxptr = lio;
 573
 574         return 0;
 575 }
 576
 577 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
 578 {
 579         struct lio *lio = GET_LIO(netdev);
 580
 581         if (lio->link_status_wq.wq) {
 582                 cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
 583                 destroy_workqueue(lio->link_status_wq.wq);
 584         }
 585 }
 586
 587 /**
 588  * \brief Update link status
 589  * @param netdev network device
 590  * @param ls link status structure
 591  *
 592  * Called on receipt of a link status response from the core application to
 593  * update each interface's link status.
 594  */
 595 static inline void update_link_status(struct net_device *netdev,
 596                                       union oct_link_status *ls)
 597 {
 598         struct lio *lio = GET_LIO(netdev);
 599         int changed = (lio->linfo.link.u64 != ls->u64);
 600         int current_max_mtu = lio->linfo.link.s.mtu;
 601         struct octeon_device *oct = lio->oct_dev;
 602
 603         dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
 604                 __func__, lio->linfo.link.u64, ls->u64);
 605         lio->linfo.link.u64 = ls->u64;
 606
 607         if ((lio->intf_open) && (changed)) {
 608                 print_link_info(netdev);
 609                 lio->link_changes++;
 610
 611                 if (lio->linfo.link.s.link_up) {
 612                         dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
 613                         netif_carrier_on(netdev);
 614                         wake_txqs(netdev);
 615                 } else {
 616                         dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
 617                         netif_carrier_off(netdev);
 618                         stop_txqs(netdev);
 619                 }
 620                 if (lio->linfo.link.s.mtu != current_max_mtu) {
 621                         netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
 622                                    current_max_mtu, lio->linfo.link.s.mtu);
 623                         netdev->max_mtu = lio->linfo.link.s.mtu;
 624                 }
 625                 if (lio->linfo.link.s.mtu < netdev->mtu) {
 626                         dev_warn(&oct->pci_dev->dev,
 627                                  "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
 628                                      netdev->mtu, lio->linfo.link.s.mtu);
 629                         queue_delayed_work(lio->link_status_wq.wq,
 630                                            &lio->link_status_wq.wk.work, 0);
 631                 }
 632         }
 633 }
 634
 635 /**
 636  * lio_sync_octeon_time - send latest localtime to octeon firmware so that
 637  * firmware will correct it's time, in case there is a time skew
 638  *
 639  * @work: work scheduled to send time update to octeon firmware
 640  **/
 641 static void lio_sync_octeon_time(struct work_struct *work)
 642 {
 643         struct cavium_wk *wk = (struct cavium_wk *)work;
 644         struct lio *lio = (struct lio *)wk->ctxptr;
 645         struct octeon_device *oct = lio->oct_dev;
 646         struct octeon_soft_command *sc;
 647         struct timespec64 ts;
 648         struct lio_time *lt;
 649         int ret;
 650
 651         sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
 652         if (!sc) {
 653                 dev_err(&oct->pci_dev->dev,
 654                         "Failed to sync time to octeon: soft command allocation failed\n");
 655                 return;
 656         }
 657
 658         lt = (struct lio_time *)sc->virtdptr;
 659
 660         /* Get time of the day */
 661         ktime_get_real_ts64(&ts);
 662         lt->sec = ts.tv_sec;
 663         lt->nsec = ts.tv_nsec;
 664         octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
 665
 666         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
 667         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
 668                                     OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
 669
 670         init_completion(&sc->complete);
 671         sc->sc_status = OCTEON_REQUEST_PENDING;
 672
 673         ret = octeon_send_soft_command(oct, sc);
 674         if (ret == IQ_SEND_FAILED) {
 675                 dev_err(&oct->pci_dev->dev,
 676                         "Failed to sync time to octeon: failed to send soft command\n");
 677                 octeon_free_soft_command(oct, sc);
 678         } else {
 679                 WRITE_ONCE(sc->caller_is_done, true);
 680         }
 681
 682         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 683                            &lio->sync_octeon_time_wq.wk.work,
 684                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 685 }
 686
 687 /**
 688  * setup_sync_octeon_time_wq - Sets up the work to periodically update
 689  * local time to octeon firmware
 690  *
 691  * @netdev - network device which should send time update to firmware
 692  **/
 693 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
 694 {
 695         struct lio *lio = GET_LIO(netdev);
 696         struct octeon_device *oct = lio->oct_dev;
 697
 698         lio->sync_octeon_time_wq.wq =
 699                 alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
 700         if (!lio->sync_octeon_time_wq.wq) {
 701                 dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
 702                 return -1;
 703         }
 704         INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
 705                           lio_sync_octeon_time);
 706         lio->sync_octeon_time_wq.wk.ctxptr = lio;
 707         queue_delayed_work(lio->sync_octeon_time_wq.wq,
 708                            &lio->sync_octeon_time_wq.wk.work,
 709                            msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
 710
 711         return 0;
 712 }
 713
 714 /**
 715  * cleanup_sync_octeon_time_wq - stop scheduling and destroy the work created
 716  * to periodically update local time to octeon firmware
 717  *
 718  * @netdev - network device which should send time update to firmware
 719  **/
 720 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
 721 {
 722         struct lio *lio = GET_LIO(netdev);
 723         struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
 724
 725         if (time_wq->wq) {
 726                 cancel_delayed_work_sync(&time_wq->wk.work);
 727                 destroy_workqueue(time_wq->wq);
 728         }
 729 }
 730
 731 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
 732 {
 733         struct octeon_device *other_oct;
 734
 735         other_oct = lio_get_device(oct->octeon_id + 1);
 736
 737         if (other_oct && other_oct->pci_dev) {
 738                 int oct_busnum, other_oct_busnum;
 739
 740                 oct_busnum = oct->pci_dev->bus->number;
 741                 other_oct_busnum = other_oct->pci_dev->bus->number;
 742
 743                 if (oct_busnum == other_oct_busnum) {
 744                         int oct_slot, other_oct_slot;
 745
 746                         oct_slot = PCI_SLOT(oct->pci_dev->devfn);
 747                         other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
 748
 749                         if (oct_slot == other_oct_slot)
 750                                 return other_oct;
 751                 }
 752         }
 753
 754         return NULL;
 755 }
 756
 757 static void disable_all_vf_links(struct octeon_device *oct)
 758 {
 759         struct net_device *netdev;
 760         int max_vfs, vf, i;
 761
 762         if (!oct)
 763                 return;
 764
 765         max_vfs = oct->sriov_info.max_vfs;
 766
 767         for (i = 0; i < oct->ifcount; i++) {
 768                 netdev = oct->props[i].netdev;
 769                 if (!netdev)
 770                         continue;
 771
 772                 for (vf = 0; vf < max_vfs; vf++)
 773                         liquidio_set_vf_link_state(netdev, vf,
 774                                                    IFLA_VF_LINK_STATE_DISABLE);
 775         }
 776 }
 777
 778 static int liquidio_watchdog(void *param)
 779 {
 780         bool err_msg_was_printed[LIO_MAX_CORES];
 781         u16 mask_of_crashed_or_stuck_cores = 0;
 782         bool all_vf_links_are_disabled = false;
 783         struct octeon_device *oct = param;
 784         struct octeon_device *other_oct;
 785 #ifdef CONFIG_MODULE_UNLOAD
 786         long refcount, vfs_referencing_pf;
 787         u64 vfs_mask1, vfs_mask2;
 788 #endif
 789         int core;
 790
 791         memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
 792
 793         while (!kthread_should_stop()) {
 794                 /* sleep for a couple of seconds so that we don't hog the CPU */
 795                 set_current_state(TASK_INTERRUPTIBLE);
 796                 schedule_timeout(msecs_to_jiffies(2000));
 797
 798                 mask_of_crashed_or_stuck_cores =
 799                     (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
 800
 801                 if (!mask_of_crashed_or_stuck_cores)
 802                         continue;
 803
 804                 WRITE_ONCE(oct->cores_crashed, true);
 805                 other_oct = get_other_octeon_device(oct);
 806                 if (other_oct)
 807                         WRITE_ONCE(other_oct->cores_crashed, true);
 808
 809                 for (core = 0; core < LIO_MAX_CORES; core++) {
 810                         bool core_crashed_or_got_stuck;
 811
 812                         core_crashed_or_got_stuck =
 813                                                 (mask_of_crashed_or_stuck_cores
 814                                                  >> core) & 1;
 815
 816                         if (core_crashed_or_got_stuck &&
 817                             !err_msg_was_printed[core]) {
 818                                 dev_err(&oct->pci_dev->dev,
 819                                         "ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
 820                                         core);
 821                                 err_msg_was_printed[core] = true;
 822                         }
 823                 }
 824
 825                 if (all_vf_links_are_disabled)
 826                         continue;
 827
 828                 disable_all_vf_links(oct);
 829                 disable_all_vf_links(other_oct);
 830                 all_vf_links_are_disabled = true;
 831
 832 #ifdef CONFIG_MODULE_UNLOAD
 833                 vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
 834                 vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
 835
 836                 vfs_referencing_pf  = hweight64(vfs_mask1);
 837                 vfs_referencing_pf += hweight64(vfs_mask2);
 838
 839                 refcount = module_refcount(THIS_MODULE);
 840                 if (refcount >= vfs_referencing_pf) {
 841                         while (vfs_referencing_pf) {
 842                                 module_put(THIS_MODULE);
 843                                 vfs_referencing_pf--;
 844                         }
 845                 }
 846 #endif
 847         }
 848
 849         return 0;
 850 }
 851
 852 /**
 853  * \brief PCI probe handler
 854  * @param pdev PCI device structure
 855  * @param ent unused
 856  */
 857 static int
 858 liquidio_probe(struct pci_dev *pdev,
 859                const struct pci_device_id *ent __attribute__((unused)))
 860 {
 861         struct octeon_device *oct_dev = NULL;
 862         struct handshake *hs;
 863
 864         oct_dev = octeon_allocate_device(pdev->device,
 865                                          sizeof(struct octeon_device_priv));
 866         if (!oct_dev) {
 867                 dev_err(&pdev->dev, "Unable to allocate device\n");
 868                 return -ENOMEM;
 869         }
 870
 871         if (pdev->device == OCTEON_CN23XX_PF_VID)
 872                 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
 873
 874         /* Enable PTP for 6XXX Device */
 875         if (((pdev->device == OCTEON_CN66XX) ||
 876              (pdev->device == OCTEON_CN68XX)))
 877                 oct_dev->ptp_enable = true;
 878         else
 879                 oct_dev->ptp_enable = false;
 880
 881         dev_info(&pdev->dev, "Initializing device %x:%x.\n",
 882                  (u32)pdev->vendor, (u32)pdev->device);
 883
 884         /* Assign octeon_device for this device to the private data area. */
 885         pci_set_drvdata(pdev, oct_dev);
 886
 887         /* set linux specific device pointer */
 888         oct_dev->pci_dev = (void *)pdev;
 889
 890         oct_dev->subsystem_id = pdev->subsystem_vendor |
 891                 (pdev->subsystem_device << 16);
 892
 893         hs = &handshake[oct_dev->octeon_id];
 894         init_completion(&hs->init);
 895         init_completion(&hs->started);
 896         hs->pci_dev = pdev;
 897
 898         if (oct_dev->octeon_id == 0)
 899                 /* first LiquidIO NIC is detected */
 900                 complete(&first_stage);
 901
 902         if (octeon_device_init(oct_dev)) {
 903                 complete(&hs->init);
 904                 liquidio_remove(pdev);
 905                 return -ENOMEM;
 906         }
 907
 908         if (OCTEON_CN23XX_PF(oct_dev)) {
 909                 u8 bus, device, function;
 910
 911                 if (atomic_read(oct_dev->adapter_refcount) == 1) {
 912                         /* Each NIC gets one watchdog kernel thread.  The first
 913                          * PF (of each NIC) that gets pci_driver->probe()'d
 914                          * creates that thread.
 915                          */
 916                         bus = pdev->bus->number;
 917                         device = PCI_SLOT(pdev->devfn);
 918                         function = PCI_FUNC(pdev->devfn);
 919                         oct_dev->watchdog_task = kthread_create(
 920                             liquidio_watchdog, oct_dev,
 921                             "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
 922                         if (!IS_ERR(oct_dev->watchdog_task)) {
 923                                 wake_up_process(oct_dev->watchdog_task);
 924                         } else {
 925                                 oct_dev->watchdog_task = NULL;
 926                                 dev_err(&oct_dev->pci_dev->dev,
 927                                         "failed to create kernel_thread\n");
 928                                 liquidio_remove(pdev);
 929                                 return -1;
 930                         }
 931                 }
 932         }
 933
 934         oct_dev->rx_pause = 1;
 935         oct_dev->tx_pause = 1;
 936
 937         dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
 938
 939         return 0;
 940 }
 941
 942 static bool fw_type_is_auto(void)
 943 {
 944         return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
 945                        sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
 946 }
 947
 948 /**
 949  * \brief PCI FLR for each Octeon device.
 950  * @param oct octeon device
 951  */
 952 static void octeon_pci_flr(struct octeon_device *oct)
 953 {
 954         int rc;
 955
 956         pci_save_state(oct->pci_dev);
 957
 958         pci_cfg_access_lock(oct->pci_dev);
 959
 960         /* Quiesce the device completely */
 961         pci_write_config_word(oct->pci_dev, PCI_COMMAND,
 962                               PCI_COMMAND_INTX_DISABLE);
 963
 964         rc = __pci_reset_function_locked(oct->pci_dev);
 965
 966         if (rc != 0)
 967                 dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
 968                         rc, oct->pf_num);
 969
 970         pci_cfg_access_unlock(oct->pci_dev);
 971
 972         pci_restore_state(oct->pci_dev);
 973 }
 974
 975 /**
 976  *\brief Destroy resources associated with octeon device
 977  * @param pdev PCI device structure
 978  * @param ent unused
 979  */
 980 static void octeon_destroy_resources(struct octeon_device *oct)
 981 {
 982         int i, refcount;
 983         struct msix_entry *msix_entries;
 984         struct octeon_device_priv *oct_priv =
 985                 (struct octeon_device_priv *)oct->priv;
 986
 987         struct handshake *hs;
 988
 989         switch (atomic_read(&oct->status)) {
 990         case OCT_DEV_RUNNING:
 991         case OCT_DEV_CORE_OK:
 992
 993                 /* No more instructions will be forwarded. */
 994                 atomic_set(&oct->status, OCT_DEV_IN_RESET);
 995
 996                 oct->app_mode = CVM_DRV_INVALID_APP;
 997                 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
 998                         lio_get_state_string(&oct->status));
 999
1000                 schedule_timeout_uninterruptible(HZ / 10);
1001
1002                 /* fallthrough */
1003         case OCT_DEV_HOST_OK:
1004
1005                 /* fallthrough */
1006         case OCT_DEV_CONSOLE_INIT_DONE:
1007                 /* Remove any consoles */
1008                 octeon_remove_consoles(oct);
1009
1010                 /* fallthrough */
1011         case OCT_DEV_IO_QUEUES_DONE:
1012                 if (lio_wait_for_instr_fetch(oct))
1013                         dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1014
1015                 if (wait_for_pending_requests(oct))
1016                         dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1017
1018                 /* Disable the input and output queues now. No more packets will
1019                  * arrive from Octeon, but we should wait for all packet
1020                  * processing to finish.
1021                  */
1022                 oct->fn_list.disable_io_queues(oct);
1023
1024                 if (lio_wait_for_oq_pkts(oct))
1025                         dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1026
1027                 /* Force all requests waiting to be fetched by OCTEON to
1028                  * complete.
1029                  */
1030                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1031                         struct octeon_instr_queue *iq;
1032
1033                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1034                                 continue;
1035                         iq = oct->instr_queue[i];
1036
1037                         if (atomic_read(&iq->instr_pending)) {
1038                                 spin_lock_bh(&iq->lock);
1039                                 iq->fill_cnt = 0;
1040                                 iq->octeon_read_index = iq->host_write_index;
1041                                 iq->stats.instr_processed +=
1042                                         atomic_read(&iq->instr_pending);
1043                                 lio_process_iq_request_list(oct, iq, 0);
1044                                 spin_unlock_bh(&iq->lock);
1045                         }
1046                 }
1047
1048                 lio_process_ordered_list(oct, 1);
1049                 octeon_free_sc_done_list(oct);
1050                 octeon_free_sc_zombie_list(oct);
1051
1052         /* fallthrough */
1053         case OCT_DEV_INTR_SET_DONE:
1054                 /* Disable interrupts  */
1055                 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1056
1057                 if (oct->msix_on) {
1058                         msix_entries = (struct msix_entry *)oct->msix_entries;
1059                         for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1060                                 if (oct->ioq_vector[i].vector) {
1061                                         /* clear the affinity_cpumask */
1062                                         irq_set_affinity_hint(
1063                                                         msix_entries[i].vector,
1064                                                         NULL);
1065                                         free_irq(msix_entries[i].vector,
1066                                                  &oct->ioq_vector[i]);
1067                                         oct->ioq_vector[i].vector = 0;
1068                                 }
1069                         }
1070                         /* non-iov vector's argument is oct struct */
1071                         free_irq(msix_entries[i].vector, oct);
1072
1073                         pci_disable_msix(oct->pci_dev);
1074                         kfree(oct->msix_entries);
1075                         oct->msix_entries = NULL;
1076                 } else {
1077                         /* Release the interrupt line */
1078                         free_irq(oct->pci_dev->irq, oct);
1079
1080                         if (oct->flags & LIO_FLAG_MSI_ENABLED)
1081                                 pci_disable_msi(oct->pci_dev);
1082                 }
1083
1084                 kfree(oct->irq_name_storage);
1085                 oct->irq_name_storage = NULL;
1086
1087         /* fallthrough */
1088         case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1089                 if (OCTEON_CN23XX_PF(oct))
1090                         octeon_free_ioq_vector(oct);
1091
1092         /* fallthrough */
1093         case OCT_DEV_MBOX_SETUP_DONE:
1094                 if (OCTEON_CN23XX_PF(oct))
1095                         oct->fn_list.free_mbox(oct);
1096
1097         /* fallthrough */
1098         case OCT_DEV_IN_RESET:
1099         case OCT_DEV_DROQ_INIT_DONE:
1100                 /* Wait for any pending operations */
1101                 mdelay(100);
1102                 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1103                         if (!(oct->io_qmask.oq & BIT_ULL(i)))
1104                                 continue;
1105                         octeon_delete_droq(oct, i);
1106                 }
1107
1108                 /* Force any pending handshakes to complete */
1109                 for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1110                         hs = &handshake[i];
1111
1112                         if (hs->pci_dev) {
1113                                 handshake[oct->octeon_id].init_ok = 0;
1114                                 complete(&handshake[oct->octeon_id].init);
1115                                 handshake[oct->octeon_id].started_ok = 0;
1116                                 complete(&handshake[oct->octeon_id].started);
1117                         }
1118                 }
1119
1120                 /* fallthrough */
1121         case OCT_DEV_RESP_LIST_INIT_DONE:
1122                 octeon_delete_response_list(oct);
1123
1124                 /* fallthrough */
1125         case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1126                 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1127                         if (!(oct->io_qmask.iq & BIT_ULL(i)))
1128                                 continue;
1129                         octeon_delete_instr_queue(oct, i);
1130                 }
1131 #ifdef CONFIG_PCI_IOV
1132                 if (oct->sriov_info.sriov_enabled)
1133                         pci_disable_sriov(oct->pci_dev);
1134 #endif
1135                 /* fallthrough */
1136         case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1137                 octeon_free_sc_buffer_pool(oct);
1138
1139                 /* fallthrough */
1140         case OCT_DEV_DISPATCH_INIT_DONE:
1141                 octeon_delete_dispatch_list(oct);
1142                 cancel_delayed_work_sync(&oct->nic_poll_work.work);
1143
1144                 /* fallthrough */
1145         case OCT_DEV_PCI_MAP_DONE:
1146                 refcount = octeon_deregister_device(oct);
1147
1148                 /* Soft reset the octeon device before exiting.
1149                  * However, if fw was loaded from card (i.e. autoboot),
1150                  * perform an FLR instead.
1151                  * Implementation note: only soft-reset the device
1152                  * if it is a CN6XXX OR the LAST CN23XX device.
1153                  */
1154                 if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1155                         octeon_pci_flr(oct);
1156                 else if (OCTEON_CN6XXX(oct) || !refcount)
1157                         oct->fn_list.soft_reset(oct);
1158
1159                 octeon_unmap_pci_barx(oct, 0);
1160                 octeon_unmap_pci_barx(oct, 1);
1161
1162                 /* fallthrough */
1163         case OCT_DEV_PCI_ENABLE_DONE:
1164                 pci_clear_master(oct->pci_dev);
1165                 /* Disable the device, releasing the PCI INT */
1166                 pci_disable_device(oct->pci_dev);
1167
1168                 /* fallthrough */
1169         case OCT_DEV_BEGIN_STATE:
1170                 /* Nothing to be done here either */
1171                 break;
1172         }                       /* end switch (oct->status) */
1173
1174         tasklet_kill(&oct_priv->droq_tasklet);
1175 }
1176
1177 /**
1178  * \brief Send Rx control command
1179  * @param lio per-network private data
1180  * @param start_stop whether to start or stop
1181  */
1182 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1183 {
1184         struct octeon_soft_command *sc;
1185         union octnet_cmd *ncmd;
1186         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1187         int retval;
1188
1189         if (oct->props[lio->ifidx].rx_on == start_stop)
1190                 return;
1191
1192         sc = (struct octeon_soft_command *)
1193                 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1194                                           16, 0);
1195
1196         ncmd = (union octnet_cmd *)sc->virtdptr;
1197
1198         ncmd->u64 = 0;
1199         ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1200         ncmd->s.param1 = start_stop;
1201
1202         octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1203
1204         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1205
1206         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1207                                     OPCODE_NIC_CMD, 0, 0, 0);
1208
1209         init_completion(&sc->complete);
1210         sc->sc_status = OCTEON_REQUEST_PENDING;
1211
1212         retval = octeon_send_soft_command(oct, sc);
1213         if (retval == IQ_SEND_FAILED) {
1214                 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1215                 octeon_free_soft_command(oct, sc);
1216                 return;
1217         } else {
1218                 /* Sleep on a wait queue till the cond flag indicates that the
1219                  * response arrived or timed-out.
1220                  */
1221                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
1222                 if (retval)
1223                         return;
1224
1225                 oct->props[lio->ifidx].rx_on = start_stop;
1226                 WRITE_ONCE(sc->caller_is_done, true);
1227         }
1228 }
1229
1230 /**
1231  * \brief Destroy NIC device interface
1232  * @param oct octeon device
1233  * @param ifidx which interface to destroy
1234  *
1235  * Cleanup associated with each interface for an Octeon device  when NIC
1236  * module is being unloaded or if initialization fails during load.
1237  */
1238 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1239 {
1240         struct net_device *netdev = oct->props[ifidx].netdev;
1241         struct octeon_device_priv *oct_priv =
1242                 (struct octeon_device_priv *)oct->priv;
1243         struct napi_struct *napi, *n;
1244         struct lio *lio;
1245
1246         if (!netdev) {
1247                 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1248                         __func__, ifidx);
1249                 return;
1250         }
1251
1252         lio = GET_LIO(netdev);
1253
1254         dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1255
1256         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1257                 liquidio_stop(netdev);
1258
1259         if (oct->props[lio->ifidx].napi_enabled == 1) {
1260                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1261                         napi_disable(napi);
1262
1263                 oct->props[lio->ifidx].napi_enabled = 0;
1264
1265                 if (OCTEON_CN23XX_PF(oct))
1266                         oct->droq[0]->ops.poll_mode = 0;
1267         }
1268
1269         /* Delete NAPI */
1270         list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1271                 netif_napi_del(napi);
1272
1273         tasklet_enable(&oct_priv->droq_tasklet);
1274
1275         if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1276                 unregister_netdev(netdev);
1277
1278         cleanup_sync_octeon_time_wq(netdev);
1279         cleanup_link_status_change_wq(netdev);
1280
1281         cleanup_rx_oom_poll_fn(netdev);
1282
1283         lio_delete_glists(lio);
1284
1285         free_netdev(netdev);
1286
1287         oct->props[ifidx].gmxport = -1;
1288
1289         oct->props[ifidx].netdev = NULL;
1290 }
1291
1292 /**
1293  * \brief Stop complete NIC functionality
1294  * @param oct octeon device
1295  */
1296 static int liquidio_stop_nic_module(struct octeon_device *oct)
1297 {
1298         int i, j;
1299         struct lio *lio;
1300
1301         dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1302         if (!oct->ifcount) {
1303                 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1304                 return 1;
1305         }
1306
1307         spin_lock_bh(&oct->cmd_resp_wqlock);
1308         oct->cmd_resp_state = OCT_DRV_OFFLINE;
1309         spin_unlock_bh(&oct->cmd_resp_wqlock);
1310
1311         lio_vf_rep_destroy(oct);
1312
1313         for (i = 0; i < oct->ifcount; i++) {
1314                 lio = GET_LIO(oct->props[i].netdev);
1315                 for (j = 0; j < oct->num_oqs; j++)
1316                         octeon_unregister_droq_ops(oct,
1317                                                    lio->linfo.rxpciq[j].s.q_no);
1318         }
1319
1320         for (i = 0; i < oct->ifcount; i++)
1321                 liquidio_destroy_nic_device(oct, i);
1322
1323         if (oct->devlink) {
1324                 devlink_unregister(oct->devlink);
1325                 devlink_free(oct->devlink);
1326                 oct->devlink = NULL;
1327         }
1328
1329         dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1330         return 0;
1331 }
1332
1333 /**
1334  * \brief Cleans up resources at unload time
1335  * @param pdev PCI device structure
1336  */
1337 static void liquidio_remove(struct pci_dev *pdev)
1338 {
1339         struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1340
1341         dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1342
1343         if (oct_dev->watchdog_task)
1344                 kthread_stop(oct_dev->watchdog_task);
1345
1346         if (!oct_dev->octeon_id &&
1347             oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1348                 lio_vf_rep_modexit();
1349
1350         if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1351                 liquidio_stop_nic_module(oct_dev);
1352
1353         /* Reset the octeon device and cleanup all memory allocated for
1354          * the octeon device by driver.
1355          */
1356         octeon_destroy_resources(oct_dev);
1357
1358         dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1359
1360         /* This octeon device has been removed. Update the global
1361          * data structure to reflect this. Free the device structure.
1362          */
1363         octeon_free_device_mem(oct_dev);
1364 }
1365
1366 /**
1367  * \brief Identify the Octeon device and to map the BAR address space
1368  * @param oct octeon device
1369  */
1370 static int octeon_chip_specific_setup(struct octeon_device *oct)
1371 {
1372         u32 dev_id, rev_id;
1373         int ret = 1;
1374         char *s;
1375
1376         pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1377         pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1378         oct->rev_id = rev_id & 0xff;
1379
1380         switch (dev_id) {
1381         case OCTEON_CN68XX_PCIID:
1382                 oct->chip_id = OCTEON_CN68XX;
1383                 ret = lio_setup_cn68xx_octeon_device(oct);
1384                 s = "CN68XX";
1385                 break;
1386
1387         case OCTEON_CN66XX_PCIID:
1388                 oct->chip_id = OCTEON_CN66XX;
1389                 ret = lio_setup_cn66xx_octeon_device(oct);
1390                 s = "CN66XX";
1391                 break;
1392
1393         case OCTEON_CN23XX_PCIID_PF:
1394                 oct->chip_id = OCTEON_CN23XX_PF_VID;
1395                 ret = setup_cn23xx_octeon_pf_device(oct);
1396                 if (ret)
1397                         break;
1398 #ifdef CONFIG_PCI_IOV
1399                 if (!ret)
1400                         pci_sriov_set_totalvfs(oct->pci_dev,
1401                                                oct->sriov_info.max_vfs);
1402 #endif
1403                 s = "CN23XX";
1404                 break;
1405
1406         default:
1407                 s = "?";
1408                 dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1409                         dev_id);
1410         }
1411
1412         if (!ret)
1413                 dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1414                          OCTEON_MAJOR_REV(oct),
1415                          OCTEON_MINOR_REV(oct),
1416                          octeon_get_conf(oct)->card_name,
1417                          LIQUIDIO_VERSION);
1418
1419         return ret;
1420 }
1421
1422 /**
1423  * \brief PCI initialization for each Octeon device.
1424  * @param oct octeon device
1425  */
1426 static int octeon_pci_os_setup(struct octeon_device *oct)
1427 {
1428         /* setup PCI stuff first */
1429         if (pci_enable_device(oct->pci_dev)) {
1430                 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1431                 return 1;
1432         }
1433
1434         if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1435                 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1436                 pci_disable_device(oct->pci_dev);
1437                 return 1;
1438         }
1439
1440         /* Enable PCI DMA Master. */
1441         pci_set_master(oct->pci_dev);
1442
1443         return 0;
1444 }
1445
1446 /**
1447  * \brief Unmap and free network buffer
1448  * @param buf buffer
1449  */
1450 static void free_netbuf(void *buf)
1451 {
1452         struct sk_buff *skb;
1453         struct octnet_buf_free_info *finfo;
1454         struct lio *lio;
1455
1456         finfo = (struct octnet_buf_free_info *)buf;
1457         skb = finfo->skb;
1458         lio = finfo->lio;
1459
1460         dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1461                          DMA_TO_DEVICE);
1462
1463         tx_buffer_free(skb);
1464 }
1465
1466 /**
1467  * \brief Unmap and free gather buffer
1468  * @param buf buffer
1469  */
1470 static void free_netsgbuf(void *buf)
1471 {
1472         struct octnet_buf_free_info *finfo;
1473         struct sk_buff *skb;
1474         struct lio *lio;
1475         struct octnic_gather *g;
1476         int i, frags, iq;
1477
1478         finfo = (struct octnet_buf_free_info *)buf;
1479         skb = finfo->skb;
1480         lio = finfo->lio;
1481         g = finfo->g;
1482         frags = skb_shinfo(skb)->nr_frags;
1483
1484         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1485                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1486                          DMA_TO_DEVICE);
1487
1488         i = 1;
1489         while (frags--) {
1490                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1491
1492                 pci_unmap_page((lio->oct_dev)->pci_dev,
1493                                g->sg[(i >> 2)].ptr[(i & 3)],
1494                                frag->size, DMA_TO_DEVICE);
1495                 i++;
1496         }
1497
1498         iq = skb_iq(lio->oct_dev, skb);
1499         spin_lock(&lio->glist_lock[iq]);
1500         list_add_tail(&g->list, &lio->glist[iq]);
1501         spin_unlock(&lio->glist_lock[iq]);
1502
1503         tx_buffer_free(skb);
1504 }
1505
1506 /**
1507  * \brief Unmap and free gather buffer with response
1508  * @param buf buffer
1509  */
1510 static void free_netsgbuf_with_resp(void *buf)
1511 {
1512         struct octeon_soft_command *sc;
1513         struct octnet_buf_free_info *finfo;
1514         struct sk_buff *skb;
1515         struct lio *lio;
1516         struct octnic_gather *g;
1517         int i, frags, iq;
1518
1519         sc = (struct octeon_soft_command *)buf;
1520         skb = (struct sk_buff *)sc->callback_arg;
1521         finfo = (struct octnet_buf_free_info *)&skb->cb;
1522
1523         lio = finfo->lio;
1524         g = finfo->g;
1525         frags = skb_shinfo(skb)->nr_frags;
1526
1527         dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1528                          g->sg[0].ptr[0], (skb->len - skb->data_len),
1529                          DMA_TO_DEVICE);
1530
1531         i = 1;
1532         while (frags--) {
1533                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1534
1535                 pci_unmap_page((lio->oct_dev)->pci_dev,
1536                                g->sg[(i >> 2)].ptr[(i & 3)],
1537                                frag->size, DMA_TO_DEVICE);
1538                 i++;
1539         }
1540
1541         iq = skb_iq(lio->oct_dev, skb);
1542
1543         spin_lock(&lio->glist_lock[iq]);
1544         list_add_tail(&g->list, &lio->glist[iq]);
1545         spin_unlock(&lio->glist_lock[iq]);
1546
1547         /* Don't free the skb yet */
1548 }
1549
1550 /**
1551  * \brief Adjust ptp frequency
1552  * @param ptp PTP clock info
1553  * @param ppb how much to adjust by, in parts-per-billion
1554  */
1555 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1556 {
1557         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1558         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1559         u64 comp, delta;
1560         unsigned long flags;
1561         bool neg_adj = false;
1562
1563         if (ppb < 0) {
1564                 neg_adj = true;
1565                 ppb = -ppb;
1566         }
1567
1568         /* The hardware adds the clock compensation value to the
1569          * PTP clock on every coprocessor clock cycle, so we
1570          * compute the delta in terms of coprocessor clocks.
1571          */
1572         delta = (u64)ppb << 32;
1573         do_div(delta, oct->coproc_clock_rate);
1574
1575         spin_lock_irqsave(&lio->ptp_lock, flags);
1576         comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1577         if (neg_adj)
1578                 comp -= delta;
1579         else
1580                 comp += delta;
1581         lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1582         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1583
1584         return 0;
1585 }
1586
1587 /**
1588  * \brief Adjust ptp time
1589  * @param ptp PTP clock info
1590  * @param delta how much to adjust by, in nanosecs
1591  */
1592 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1593 {
1594         unsigned long flags;
1595         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1596
1597         spin_lock_irqsave(&lio->ptp_lock, flags);
1598         lio->ptp_adjust += delta;
1599         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1600
1601         return 0;
1602 }
1603
1604 /**
1605  * \brief Get hardware clock time, including any adjustment
1606  * @param ptp PTP clock info
1607  * @param ts timespec
1608  */
1609 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1610                                 struct timespec64 *ts)
1611 {
1612         u64 ns;
1613         unsigned long flags;
1614         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1615         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1616
1617         spin_lock_irqsave(&lio->ptp_lock, flags);
1618         ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1619         ns += lio->ptp_adjust;
1620         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1621
1622         *ts = ns_to_timespec64(ns);
1623
1624         return 0;
1625 }
1626
1627 /**
1628  * \brief Set hardware clock time. Reset adjustment
1629  * @param ptp PTP clock info
1630  * @param ts timespec
1631  */
1632 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1633                                 const struct timespec64 *ts)
1634 {
1635         u64 ns;
1636         unsigned long flags;
1637         struct lio *lio = container_of(ptp, struct lio, ptp_info);
1638         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1639
1640         ns = timespec64_to_ns(ts);
1641
1642         spin_lock_irqsave(&lio->ptp_lock, flags);
1643         lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1644         lio->ptp_adjust = 0;
1645         spin_unlock_irqrestore(&lio->ptp_lock, flags);
1646
1647         return 0;
1648 }
1649
1650 /**
1651  * \brief Check if PTP is enabled
1652  * @param ptp PTP clock info
1653  * @param rq request
1654  * @param on is it on
1655  */
1656 static int
1657 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1658                     struct ptp_clock_request *rq __attribute__((unused)),
1659                     int on __attribute__((unused)))
1660 {
1661         return -EOPNOTSUPP;
1662 }
1663
1664 /**
1665  * \brief Open PTP clock source
1666  * @param netdev network device
1667  */
1668 static void oct_ptp_open(struct net_device *netdev)
1669 {
1670         struct lio *lio = GET_LIO(netdev);
1671         struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1672
1673         spin_lock_init(&lio->ptp_lock);
1674
1675         snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1676         lio->ptp_info.owner = THIS_MODULE;
1677         lio->ptp_info.max_adj = 250000000;
1678         lio->ptp_info.n_alarm = 0;
1679         lio->ptp_info.n_ext_ts = 0;
1680         lio->ptp_info.n_per_out = 0;
1681         lio->ptp_info.pps = 0;
1682         lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1683         lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1684         lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1685         lio->ptp_info.settime64 = liquidio_ptp_settime;
1686         lio->ptp_info.enable = liquidio_ptp_enable;
1687
1688         lio->ptp_adjust = 0;
1689
1690         lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1691                                              &oct->pci_dev->dev);
1692
1693         if (IS_ERR(lio->ptp_clock))
1694                 lio->ptp_clock = NULL;
1695 }
1696
1697 /**
1698  * \brief Init PTP clock
1699  * @param oct octeon device
1700  */
1701 static void liquidio_ptp_init(struct octeon_device *oct)
1702 {
1703         u64 clock_comp, cfg;
1704
1705         clock_comp = (u64)NSEC_PER_SEC << 32;
1706         do_div(clock_comp, oct->coproc_clock_rate);
1707         lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1708
1709         /* Enable */
1710         cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1711         lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1712 }
1713
1714 /**
1715  * \brief Load firmware to device
1716  * @param oct octeon device
1717  *
1718  * Maps device to firmware filename, requests firmware, and downloads it
1719  */
1720 static int load_firmware(struct octeon_device *oct)
1721 {
1722         int ret = 0;
1723         const struct firmware *fw;
1724         char fw_name[LIO_MAX_FW_FILENAME_LEN];
1725         char *tmp_fw_type;
1726
1727         if (fw_type_is_auto()) {
1728                 tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1729                 strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
1730         } else {
1731                 tmp_fw_type = fw_type;
1732         }
1733
1734         sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1735                 octeon_get_conf(oct)->card_name, tmp_fw_type,
1736                 LIO_FW_NAME_SUFFIX);
1737
1738         ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1739         if (ret) {
1740                 dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1741                         fw_name);
1742                 release_firmware(fw);
1743                 return ret;
1744         }
1745
1746         ret = octeon_download_firmware(oct, fw->data, fw->size);
1747
1748         release_firmware(fw);
1749
1750         return ret;
1751 }
1752
1753 /**
1754  * \brief Poll routine for checking transmit queue status
1755  * @param work work_struct data structure
1756  */
1757 static void octnet_poll_check_txq_status(struct work_struct *work)
1758 {
1759         struct cavium_wk *wk = (struct cavium_wk *)work;
1760         struct lio *lio = (struct lio *)wk->ctxptr;
1761
1762         if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1763                 return;
1764
1765         check_txq_status(lio);
1766         queue_delayed_work(lio->txq_status_wq.wq,
1767                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1768 }
1769
1770 /**
1771  * \brief Sets up the txq poll check
1772  * @param netdev network device
1773  */
1774 static inline int setup_tx_poll_fn(struct net_device *netdev)
1775 {
1776         struct lio *lio = GET_LIO(netdev);
1777         struct octeon_device *oct = lio->oct_dev;
1778
1779         lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1780                                                 WQ_MEM_RECLAIM, 0);
1781         if (!lio->txq_status_wq.wq) {
1782                 dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1783                 return -1;
1784         }
1785         INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1786                           octnet_poll_check_txq_status);
1787         lio->txq_status_wq.wk.ctxptr = lio;
1788         queue_delayed_work(lio->txq_status_wq.wq,
1789                            &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1790         return 0;
1791 }
1792
1793 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1794 {
1795         struct lio *lio = GET_LIO(netdev);
1796
1797         if (lio->txq_status_wq.wq) {
1798                 cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1799                 destroy_workqueue(lio->txq_status_wq.wq);
1800         }
1801 }
1802
1803 /**
1804  * \brief Net device open for LiquidIO
1805  * @param netdev network device
1806  */
1807 static int liquidio_open(struct net_device *netdev)
1808 {
1809         struct lio *lio = GET_LIO(netdev);
1810         struct octeon_device *oct = lio->oct_dev;
1811         struct octeon_device_priv *oct_priv =
1812                 (struct octeon_device_priv *)oct->priv;
1813         struct napi_struct *napi, *n;
1814
1815         if (oct->props[lio->ifidx].napi_enabled == 0) {
1816                 tasklet_disable(&oct_priv->droq_tasklet);
1817
1818                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1819                         napi_enable(napi);
1820
1821                 oct->props[lio->ifidx].napi_enabled = 1;
1822
1823                 if (OCTEON_CN23XX_PF(oct))
1824                         oct->droq[0]->ops.poll_mode = 1;
1825         }
1826
1827         if (oct->ptp_enable)
1828                 oct_ptp_open(netdev);
1829
1830         ifstate_set(lio, LIO_IFSTATE_RUNNING);
1831
1832         if (OCTEON_CN23XX_PF(oct)) {
1833                 if (!oct->msix_on)
1834                         if (setup_tx_poll_fn(netdev))
1835                                 return -1;
1836         } else {
1837                 if (setup_tx_poll_fn(netdev))
1838                         return -1;
1839         }
1840
1841         netif_tx_start_all_queues(netdev);
1842
1843         /* Ready for link status updates */
1844         lio->intf_open = 1;
1845
1846         netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1847
1848         /* tell Octeon to start forwarding packets to host */
1849         send_rx_ctrl_cmd(lio, 1);
1850
1851         /* start periodical statistics fetch */
1852         INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1853         lio->stats_wk.ctxptr = lio;
1854         schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1855                                         (LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1856
1857         dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1858                  netdev->name);
1859
1860         return 0;
1861 }
1862
1863 /**
1864  * \brief Net device stop for LiquidIO
1865  * @param netdev network device
1866  */
1867 static int liquidio_stop(struct net_device *netdev)
1868 {
1869         struct lio *lio = GET_LIO(netdev);
1870         struct octeon_device *oct = lio->oct_dev;
1871         struct octeon_device_priv *oct_priv =
1872                 (struct octeon_device_priv *)oct->priv;
1873         struct napi_struct *napi, *n;
1874
1875         ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1876
1877         /* Stop any link updates */
1878         lio->intf_open = 0;
1879
1880         stop_txqs(netdev);
1881
1882         /* Inform that netif carrier is down */
1883         netif_carrier_off(netdev);
1884         netif_tx_disable(netdev);
1885
1886         lio->linfo.link.s.link_up = 0;
1887         lio->link_changes++;
1888
1889         /* Tell Octeon that nic interface is down. */
1890         send_rx_ctrl_cmd(lio, 0);
1891
1892         if (OCTEON_CN23XX_PF(oct)) {
1893                 if (!oct->msix_on)
1894                         cleanup_tx_poll_fn(netdev);
1895         } else {
1896                 cleanup_tx_poll_fn(netdev);
1897         }
1898
1899         cancel_delayed_work_sync(&lio->stats_wk.work);
1900
1901         if (lio->ptp_clock) {
1902                 ptp_clock_unregister(lio->ptp_clock);
1903                 lio->ptp_clock = NULL;
1904         }
1905
1906         /* Wait for any pending Rx descriptors */
1907         if (lio_wait_for_clean_oq(oct))
1908                 netif_info(lio, rx_err, lio->netdev,
1909                            "Proceeding with stop interface after partial RX desc processing\n");
1910
1911         if (oct->props[lio->ifidx].napi_enabled == 1) {
1912                 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1913                         napi_disable(napi);
1914
1915                 oct->props[lio->ifidx].napi_enabled = 0;
1916
1917                 if (OCTEON_CN23XX_PF(oct))
1918                         oct->droq[0]->ops.poll_mode = 0;
1919
1920                 tasklet_enable(&oct_priv->droq_tasklet);
1921         }
1922
1923         dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1924
1925         return 0;
1926 }
1927
1928 /**
1929  * \brief Converts a mask based on net device flags
1930  * @param netdev network device
1931  *
1932  * This routine generates a octnet_ifflags mask from the net device flags
1933  * received from the OS.
1934  */
1935 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1936 {
1937         enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1938
1939         if (netdev->flags & IFF_PROMISC)
1940                 f |= OCTNET_IFFLAG_PROMISC;
1941
1942         if (netdev->flags & IFF_ALLMULTI)
1943                 f |= OCTNET_IFFLAG_ALLMULTI;
1944
1945         if (netdev->flags & IFF_MULTICAST) {
1946                 f |= OCTNET_IFFLAG_MULTICAST;
1947
1948                 /* Accept all multicast addresses if there are more than we
1949                  * can handle
1950                  */
1951                 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1952                         f |= OCTNET_IFFLAG_ALLMULTI;
1953         }
1954
1955         if (netdev->flags & IFF_BROADCAST)
1956                 f |= OCTNET_IFFLAG_BROADCAST;
1957
1958         return f;
1959 }
1960
1961 /**
1962  * \brief Net device set_multicast_list
1963  * @param netdev network device
1964  */
1965 static void liquidio_set_mcast_list(struct net_device *netdev)
1966 {
1967         struct lio *lio = GET_LIO(netdev);
1968         struct octeon_device *oct = lio->oct_dev;
1969         struct octnic_ctrl_pkt nctrl;
1970         struct netdev_hw_addr *ha;
1971         u64 *mc;
1972         int ret;
1973         int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1974
1975         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1976
1977         /* Create a ctrl pkt command to be sent to core app. */
1978         nctrl.ncmd.u64 = 0;
1979         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1980         nctrl.ncmd.s.param1 = get_new_flags(netdev);
1981         nctrl.ncmd.s.param2 = mc_count;
1982         nctrl.ncmd.s.more = mc_count;
1983         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1984         nctrl.netpndev = (u64)netdev;
1985         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1986
1987         /* copy all the addresses into the udd */
1988         mc = &nctrl.udd[0];
1989         netdev_for_each_mc_addr(ha, netdev) {
1990                 *mc = 0;
1991                 memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1992                 /* no need to swap bytes */
1993
1994                 if (++mc > &nctrl.udd[mc_count])
1995                         break;
1996         }
1997
1998         /* Apparently, any activity in this call from the kernel has to
1999          * be atomic. So we won't wait for response.
2000          */
2001
2002         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2003         if (ret) {
2004                 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2005                         ret);
2006         }
2007 }
2008
2009 /**
2010  * \brief Net device set_mac_address
2011  * @param netdev network device
2012  */
2013 static int liquidio_set_mac(struct net_device *netdev, void *p)
2014 {
2015         int ret = 0;
2016         struct lio *lio = GET_LIO(netdev);
2017         struct octeon_device *oct = lio->oct_dev;
2018         struct sockaddr *addr = (struct sockaddr *)p;
2019         struct octnic_ctrl_pkt nctrl;
2020
2021         if (!is_valid_ether_addr(addr->sa_data))
2022                 return -EADDRNOTAVAIL;
2023
2024         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2025
2026         nctrl.ncmd.u64 = 0;
2027         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2028         nctrl.ncmd.s.param1 = 0;
2029         nctrl.ncmd.s.more = 1;
2030         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2031         nctrl.netpndev = (u64)netdev;
2032
2033         nctrl.udd[0] = 0;
2034         /* The MAC Address is presented in network byte order. */
2035         memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2036
2037         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2038         if (ret < 0) {
2039                 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2040                 return -ENOMEM;
2041         }
2042
2043         if (nctrl.sc_status) {
2044                 dev_err(&oct->pci_dev->dev,
2045                         "%s: MAC Address change failed. sc return=%x\n",
2046                          __func__, nctrl.sc_status);
2047                 return -EIO;
2048         }
2049
2050         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2051         memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2052
2053         return 0;
2054 }
2055
2056 static void
2057 liquidio_get_stats64(struct net_device *netdev,
2058                      struct rtnl_link_stats64 *lstats)
2059 {
2060         struct lio *lio = GET_LIO(netdev);
2061         struct octeon_device *oct;
2062         u64 pkts = 0, drop = 0, bytes = 0;
2063         struct oct_droq_stats *oq_stats;
2064         struct oct_iq_stats *iq_stats;
2065         int i, iq_no, oq_no;
2066
2067         oct = lio->oct_dev;
2068
2069         if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2070                 return;
2071
2072         for (i = 0; i < oct->num_iqs; i++) {
2073                 iq_no = lio->linfo.txpciq[i].s.q_no;
2074                 iq_stats = &oct->instr_queue[iq_no]->stats;
2075                 pkts += iq_stats->tx_done;
2076                 drop += iq_stats->tx_dropped;
2077                 bytes += iq_stats->tx_tot_bytes;
2078         }
2079
2080         lstats->tx_packets = pkts;
2081         lstats->tx_bytes = bytes;
2082         lstats->tx_dropped = drop;
2083
2084         pkts = 0;
2085         drop = 0;
2086         bytes = 0;
2087
2088         for (i = 0; i < oct->num_oqs; i++) {
2089                 oq_no = lio->linfo.rxpciq[i].s.q_no;
2090                 oq_stats = &oct->droq[oq_no]->stats;
2091                 pkts += oq_stats->rx_pkts_received;
2092                 drop += (oq_stats->rx_dropped +
2093                          oq_stats->dropped_nodispatch +
2094                          oq_stats->dropped_toomany +
2095                          oq_stats->dropped_nomem);
2096                 bytes += oq_stats->rx_bytes_received;
2097         }
2098
2099         lstats->rx_bytes = bytes;
2100         lstats->rx_packets = pkts;
2101         lstats->rx_dropped = drop;
2102
2103         lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2104         lstats->collisions = oct->link_stats.fromhost.total_collisions;
2105
2106         /* detailed rx_errors: */
2107         lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2108         /* recved pkt with crc error    */
2109         lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2110         /* recv'd frame alignment error */
2111         lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2112         /* recv'r fifo overrun */
2113         lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2114
2115         lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2116                 lstats->rx_frame_errors + lstats->rx_fifo_errors;
2117
2118         /* detailed tx_errors */
2119         lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2120         lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2121         lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2122
2123         lstats->tx_errors = lstats->tx_aborted_errors +
2124                 lstats->tx_carrier_errors +
2125                 lstats->tx_fifo_errors;
2126 }
2127
2128 /**
2129  * \brief Handler for SIOCSHWTSTAMP ioctl
2130  * @param netdev network device
2131  * @param ifr interface request
2132  * @param cmd command
2133  */
2134 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2135 {
2136         struct hwtstamp_config conf;
2137         struct lio *lio = GET_LIO(netdev);
2138
2139         if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2140                 return -EFAULT;
2141
2142         if (conf.flags)
2143                 return -EINVAL;
2144
2145         switch (conf.tx_type) {
2146         case HWTSTAMP_TX_ON:
2147         case HWTSTAMP_TX_OFF:
2148                 break;
2149         default:
2150                 return -ERANGE;
2151         }
2152
2153         switch (conf.rx_filter) {
2154         case HWTSTAMP_FILTER_NONE:
2155                 break;
2156         case HWTSTAMP_FILTER_ALL:
2157         case HWTSTAMP_FILTER_SOME:
2158         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2159         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2160         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2161         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2162         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2163         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2164         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2165         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2166         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2167         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2168         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2169         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2170         case HWTSTAMP_FILTER_NTP_ALL:
2171                 conf.rx_filter = HWTSTAMP_FILTER_ALL;
2172                 break;
2173         default:
2174                 return -ERANGE;
2175         }
2176
2177         if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2178                 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2179
2180         else
2181                 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2182
2183         return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2184 }
2185
2186 /**
2187  * \brief ioctl handler
2188  * @param netdev network device
2189  * @param ifr interface request
2190  * @param cmd command
2191  */
2192 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2193 {
2194         struct lio *lio = GET_LIO(netdev);
2195
2196         switch (cmd) {
2197         case SIOCSHWTSTAMP:
2198                 if (lio->oct_dev->ptp_enable)
2199                         return hwtstamp_ioctl(netdev, ifr);
2200                 /* fall through */
2201         default:
2202                 return -EOPNOTSUPP;
2203         }
2204 }
2205
2206 /**
2207  * \brief handle a Tx timestamp response
2208  * @param status response status
2209  * @param buf pointer to skb
2210  */
2211 static void handle_timestamp(struct octeon_device *oct,
2212                              u32 status,
2213                              void *buf)
2214 {
2215         struct octnet_buf_free_info *finfo;
2216         struct octeon_soft_command *sc;
2217         struct oct_timestamp_resp *resp;
2218         struct lio *lio;
2219         struct sk_buff *skb = (struct sk_buff *)buf;
2220
2221         finfo = (struct octnet_buf_free_info *)skb->cb;
2222         lio = finfo->lio;
2223         sc = finfo->sc;
2224         oct = lio->oct_dev;
2225         resp = (struct oct_timestamp_resp *)sc->virtrptr;
2226
2227         if (status != OCTEON_REQUEST_DONE) {
2228                 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2229                         CVM_CAST64(status));
2230                 resp->timestamp = 0;
2231         }
2232
2233         octeon_swap_8B_data(&resp->timestamp, 1);
2234
2235         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2236                 struct skb_shared_hwtstamps ts;
2237                 u64 ns = resp->timestamp;
2238
2239                 netif_info(lio, tx_done, lio->netdev,
2240                            "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2241                            skb, (unsigned long long)ns);
2242                 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2243                 skb_tstamp_tx(skb, &ts);
2244         }
2245
2246         octeon_free_soft_command(oct, sc);
2247         tx_buffer_free(skb);
2248 }
2249
2250 /* \brief Send a data packet that will be timestamped
2251  * @param oct octeon device
2252  * @param ndata pointer to network data
2253  * @param finfo pointer to private network data
2254  */
2255 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2256                                          struct octnic_data_pkt *ndata,
2257                                          struct octnet_buf_free_info *finfo,
2258                                          int xmit_more)
2259 {
2260         int retval;
2261         struct octeon_soft_command *sc;
2262         struct lio *lio;
2263         int ring_doorbell;
2264         u32 len;
2265
2266         lio = finfo->lio;
2267
2268         sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2269                                             sizeof(struct oct_timestamp_resp));
2270         finfo->sc = sc;
2271
2272         if (!sc) {
2273                 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2274                 return IQ_SEND_FAILED;
2275         }
2276
2277         if (ndata->reqtype == REQTYPE_NORESP_NET)
2278                 ndata->reqtype = REQTYPE_RESP_NET;
2279         else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2280                 ndata->reqtype = REQTYPE_RESP_NET_SG;
2281
2282         sc->callback = handle_timestamp;
2283         sc->callback_arg = finfo->skb;
2284         sc->iq_no = ndata->q_no;
2285
2286         if (OCTEON_CN23XX_PF(oct))
2287                 len = (u32)((struct octeon_instr_ih3 *)
2288                             (&sc->cmd.cmd3.ih3))->dlengsz;
2289         else
2290                 len = (u32)((struct octeon_instr_ih2 *)
2291                             (&sc->cmd.cmd2.ih2))->dlengsz;
2292
2293         ring_doorbell = !xmit_more;
2294
2295         retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2296                                      sc, len, ndata->reqtype);
2297
2298         if (retval == IQ_SEND_FAILED) {
2299                 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2300                         retval);
2301                 octeon_free_soft_command(oct, sc);
2302         } else {
2303                 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2304         }
2305
2306         return retval;
2307 }
2308
2309 /** \brief Transmit networks packets to the Octeon interface
2310  * @param skbuff   skbuff struct to be passed to network layer.
2311  * @param netdev    pointer to network device
2312  * @returns whether the packet was transmitted to the device okay or not
2313  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2314  */
2315 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2316 {
2317         struct lio *lio;
2318         struct octnet_buf_free_info *finfo;
2319         union octnic_cmd_setup cmdsetup;
2320         struct octnic_data_pkt ndata;
2321         struct octeon_device *oct;
2322         struct oct_iq_stats *stats;
2323         struct octeon_instr_irh *irh;
2324         union tx_info *tx_info;
2325         int status = 0;
2326         int q_idx = 0, iq_no = 0;
2327         int j, xmit_more = 0;
2328         u64 dptr = 0;
2329         u32 tag = 0;
2330
2331         lio = GET_LIO(netdev);
2332         oct = lio->oct_dev;
2333
2334         q_idx = skb_iq(oct, skb);
2335         tag = q_idx;
2336         iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2337
2338         stats = &oct->instr_queue[iq_no]->stats;
2339
2340         /* Check for all conditions in which the current packet cannot be
2341          * transmitted.
2342          */
2343         if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2344             (!lio->linfo.link.s.link_up) ||
2345             (skb->len <= 0)) {
2346                 netif_info(lio, tx_err, lio->netdev,
2347                            "Transmit failed link_status : %d\n",
2348                            lio->linfo.link.s.link_up);
2349                 goto lio_xmit_failed;
2350         }
2351
2352         /* Use space in skb->cb to store info used to unmap and
2353          * free the buffers.
2354          */
2355         finfo = (struct octnet_buf_free_info *)skb->cb;
2356         finfo->lio = lio;
2357         finfo->skb = skb;
2358         finfo->sc = NULL;
2359
2360         /* Prepare the attributes for the data to be passed to OSI. */
2361         memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2362
2363         ndata.buf = (void *)finfo;
2364
2365         ndata.q_no = iq_no;
2366
2367         if (octnet_iq_is_full(oct, ndata.q_no)) {
2368                 /* defer sending if queue is full */
2369                 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2370                            ndata.q_no);
2371                 stats->tx_iq_busy++;
2372                 return NETDEV_TX_BUSY;
2373         }
2374
2375         /* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2376          *      lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2377          */
2378
2379         ndata.datasize = skb->len;
2380
2381         cmdsetup.u64 = 0;
2382         cmdsetup.s.iq_no = iq_no;
2383
2384         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2385                 if (skb->encapsulation) {
2386                         cmdsetup.s.tnl_csum = 1;
2387                         stats->tx_vxlan++;
2388                 } else {
2389                         cmdsetup.s.transport_csum = 1;
2390                 }
2391         }
2392         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2393                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2394                 cmdsetup.s.timestamp = 1;
2395         }
2396
2397         if (skb_shinfo(skb)->nr_frags == 0) {
2398                 cmdsetup.s.u.datasize = skb->len;
2399                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2400
2401                 /* Offload checksum calculation for TCP/UDP packets */
2402                 dptr = dma_map_single(&oct->pci_dev->dev,
2403                                       skb->data,
2404                                       skb->len,
2405                                       DMA_TO_DEVICE);
2406                 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2407                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2408                                 __func__);
2409                         stats->tx_dmamap_fail++;
2410                         return NETDEV_TX_BUSY;
2411                 }
2412
2413                 if (OCTEON_CN23XX_PF(oct))
2414                         ndata.cmd.cmd3.dptr = dptr;
2415                 else
2416                         ndata.cmd.cmd2.dptr = dptr;
2417                 finfo->dptr = dptr;
2418                 ndata.reqtype = REQTYPE_NORESP_NET;
2419
2420         } else {
2421                 int i, frags;
2422                 struct skb_frag_struct *frag;
2423                 struct octnic_gather *g;
2424
2425                 spin_lock(&lio->glist_lock[q_idx]);
2426                 g = (struct octnic_gather *)
2427                         lio_list_delete_head(&lio->glist[q_idx]);
2428                 spin_unlock(&lio->glist_lock[q_idx]);
2429
2430                 if (!g) {
2431                         netif_info(lio, tx_err, lio->netdev,
2432                                    "Transmit scatter gather: glist null!\n");
2433                         goto lio_xmit_failed;
2434                 }
2435
2436                 cmdsetup.s.gather = 1;
2437                 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2438                 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2439
2440                 memset(g->sg, 0, g->sg_size);
2441
2442                 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2443                                                  skb->data,
2444                                                  (skb->len - skb->data_len),
2445                                                  DMA_TO_DEVICE);
2446                 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2447                         dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2448                                 __func__);
2449                         stats->tx_dmamap_fail++;
2450                         return NETDEV_TX_BUSY;
2451                 }
2452                 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2453
2454                 frags = skb_shinfo(skb)->nr_frags;
2455                 i = 1;
2456                 while (frags--) {
2457                         frag = &skb_shinfo(skb)->frags[i - 1];
2458
2459                         g->sg[(i >> 2)].ptr[(i & 3)] =
2460                                 dma_map_page(&oct->pci_dev->dev,
2461                                              frag->page.p,
2462                                              frag->page_offset,
2463                                              frag->size,
2464                                              DMA_TO_DEVICE);
2465
2466                         if (dma_mapping_error(&oct->pci_dev->dev,
2467                                               g->sg[i >> 2].ptr[i & 3])) {
2468                                 dma_unmap_single(&oct->pci_dev->dev,
2469                                                  g->sg[0].ptr[0],
2470                                                  skb->len - skb->data_len,
2471                                                  DMA_TO_DEVICE);
2472                                 for (j = 1; j < i; j++) {
2473                                         frag = &skb_shinfo(skb)->frags[j - 1];
2474                                         dma_unmap_page(&oct->pci_dev->dev,
2475                                                        g->sg[j >> 2].ptr[j & 3],
2476                                                        frag->size,
2477                                                        DMA_TO_DEVICE);
2478                                 }
2479                                 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2480                                         __func__);
2481                                 return NETDEV_TX_BUSY;
2482                         }
2483
2484                         add_sg_size(&g->sg[(i >> 2)], frag->size, (i & 3));
2485                         i++;
2486                 }
2487
2488                 dptr = g->sg_dma_ptr;
2489
2490                 if (OCTEON_CN23XX_PF(oct))
2491                         ndata.cmd.cmd3.dptr = dptr;
2492                 else
2493                         ndata.cmd.cmd2.dptr = dptr;
2494                 finfo->dptr = dptr;
2495                 finfo->g = g;
2496
2497                 ndata.reqtype = REQTYPE_NORESP_NET_SG;
2498         }
2499
2500         if (OCTEON_CN23XX_PF(oct)) {
2501                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2502                 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2503         } else {
2504                 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2505                 tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2506         }
2507
2508         if (skb_shinfo(skb)->gso_size) {
2509                 tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2510                 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2511                 stats->tx_gso++;
2512         }
2513
2514         /* HW insert VLAN tag */
2515         if (skb_vlan_tag_present(skb)) {
2516                 irh->priority = skb_vlan_tag_get(skb) >> 13;
2517                 irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2518         }
2519
2520         xmit_more = skb->xmit_more;
2521
2522         if (unlikely(cmdsetup.s.timestamp))
2523                 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2524         else
2525                 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2526         if (status == IQ_SEND_FAILED)
2527                 goto lio_xmit_failed;
2528
2529         netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2530
2531         if (status == IQ_SEND_STOP)
2532                 netif_stop_subqueue(netdev, q_idx);
2533
2534         netif_trans_update(netdev);
2535
2536         if (tx_info->s.gso_segs)
2537                 stats->tx_done += tx_info->s.gso_segs;
2538         else
2539                 stats->tx_done++;
2540         stats->tx_tot_bytes += ndata.datasize;
2541
2542         return NETDEV_TX_OK;
2543
2544 lio_xmit_failed:
2545         stats->tx_dropped++;
2546         netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2547                    iq_no, stats->tx_dropped);
2548         if (dptr)
2549                 dma_unmap_single(&oct->pci_dev->dev, dptr,
2550                                  ndata.datasize, DMA_TO_DEVICE);
2551
2552         octeon_ring_doorbell_locked(oct, iq_no);
2553
2554         tx_buffer_free(skb);
2555         return NETDEV_TX_OK;
2556 }
2557
2558 /** \brief Network device Tx timeout
2559  * @param netdev    pointer to network device
2560  */
2561 static void liquidio_tx_timeout(struct net_device *netdev)
2562 {
2563         struct lio *lio;
2564
2565         lio = GET_LIO(netdev);
2566
2567         netif_info(lio, tx_err, lio->netdev,
2568                    "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2569                    netdev->stats.tx_dropped);
2570         netif_trans_update(netdev);
2571         wake_txqs(netdev);
2572 }
2573
2574 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2575                                     __be16 proto __attribute__((unused)),
2576                                     u16 vid)
2577 {
2578         struct lio *lio = GET_LIO(netdev);
2579         struct octeon_device *oct = lio->oct_dev;
2580         struct octnic_ctrl_pkt nctrl;
2581         int ret = 0;
2582
2583         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2584
2585         nctrl.ncmd.u64 = 0;
2586         nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2587         nctrl.ncmd.s.param1 = vid;
2588         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2589         nctrl.netpndev = (u64)netdev;
2590         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2591
2592         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2593         if (ret) {
2594                 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2595                         ret);
2596                 if (ret > 0)
2597                         ret = -EIO;
2598         }
2599
2600         return ret;
2601 }
2602
2603 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2604                                      __be16 proto __attribute__((unused)),
2605                                      u16 vid)
2606 {
2607         struct lio *lio = GET_LIO(netdev);
2608         struct octeon_device *oct = lio->oct_dev;
2609         struct octnic_ctrl_pkt nctrl;
2610         int ret = 0;
2611
2612         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2613
2614         nctrl.ncmd.u64 = 0;
2615         nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2616         nctrl.ncmd.s.param1 = vid;
2617         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2618         nctrl.netpndev = (u64)netdev;
2619         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2620
2621         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2622         if (ret) {
2623                 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2624                         ret);
2625                 if (ret > 0)
2626                         ret = -EIO;
2627         }
2628         return ret;
2629 }
2630
2631 /** Sending command to enable/disable RX checksum offload
2632  * @param netdev                pointer to network device
2633  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2634  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2635  *                              OCTNET_CMD_RXCSUM_DISABLE
2636  * @returns                     SUCCESS or FAILURE
2637  */
2638 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2639                                        u8 rx_cmd)
2640 {
2641         struct lio *lio = GET_LIO(netdev);
2642         struct octeon_device *oct = lio->oct_dev;
2643         struct octnic_ctrl_pkt nctrl;
2644         int ret = 0;
2645
2646         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2647
2648         nctrl.ncmd.u64 = 0;
2649         nctrl.ncmd.s.cmd = command;
2650         nctrl.ncmd.s.param1 = rx_cmd;
2651         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2652         nctrl.netpndev = (u64)netdev;
2653         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2654
2655         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2656         if (ret) {
2657                 dev_err(&oct->pci_dev->dev,
2658                         "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2659                         ret);
2660                 if (ret > 0)
2661                         ret = -EIO;
2662         }
2663         return ret;
2664 }
2665
2666 /** Sending command to add/delete VxLAN UDP port to firmware
2667  * @param netdev                pointer to network device
2668  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2669  * @param vxlan_port            VxLAN port to be added or deleted
2670  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2671  *                              OCTNET_CMD_VXLAN_PORT_DEL
2672  * @returns                     SUCCESS or FAILURE
2673  */
2674 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2675                                        u16 vxlan_port, u8 vxlan_cmd_bit)
2676 {
2677         struct lio *lio = GET_LIO(netdev);
2678         struct octeon_device *oct = lio->oct_dev;
2679         struct octnic_ctrl_pkt nctrl;
2680         int ret = 0;
2681
2682         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2683
2684         nctrl.ncmd.u64 = 0;
2685         nctrl.ncmd.s.cmd = command;
2686         nctrl.ncmd.s.more = vxlan_cmd_bit;
2687         nctrl.ncmd.s.param1 = vxlan_port;
2688         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2689         nctrl.netpndev = (u64)netdev;
2690         nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2691
2692         ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2693         if (ret) {
2694                 dev_err(&oct->pci_dev->dev,
2695                         "VxLAN port add/delete failed in core (ret:0x%x)\n",
2696                         ret);
2697                 if (ret > 0)
2698                         ret = -EIO;
2699         }
2700         return ret;
2701 }
2702
2703 /** \brief Net device fix features
2704  * @param netdev  pointer to network device
2705  * @param request features requested
2706  * @returns updated features list
2707  */
2708 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2709                                                netdev_features_t request)
2710 {
2711         struct lio *lio = netdev_priv(netdev);
2712
2713         if ((request & NETIF_F_RXCSUM) &&
2714             !(lio->dev_capability & NETIF_F_RXCSUM))
2715                 request &= ~NETIF_F_RXCSUM;
2716
2717         if ((request & NETIF_F_HW_CSUM) &&
2718             !(lio->dev_capability & NETIF_F_HW_CSUM))
2719                 request &= ~NETIF_F_HW_CSUM;
2720
2721         if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2722                 request &= ~NETIF_F_TSO;
2723
2724         if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2725                 request &= ~NETIF_F_TSO6;
2726
2727         if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2728                 request &= ~NETIF_F_LRO;
2729
2730         /*Disable LRO if RXCSUM is off */
2731         if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2732             (lio->dev_capability & NETIF_F_LRO))
2733                 request &= ~NETIF_F_LRO;
2734
2735         if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2736             !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2737                 request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2738
2739         return request;
2740 }
2741
2742 /** \brief Net device set features
2743  * @param netdev  pointer to network device
2744  * @param features features to enable/disable
2745  */
2746 static int liquidio_set_features(struct net_device *netdev,
2747                                  netdev_features_t features)
2748 {
2749         struct lio *lio = netdev_priv(netdev);
2750
2751         if ((features & NETIF_F_LRO) &&
2752             (lio->dev_capability & NETIF_F_LRO) &&
2753             !(netdev->features & NETIF_F_LRO))
2754                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2755                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2756         else if (!(features & NETIF_F_LRO) &&
2757                  (lio->dev_capability & NETIF_F_LRO) &&
2758                  (netdev->features & NETIF_F_LRO))
2759                 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2760                                      OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2761
2762         /* Sending command to firmware to enable/disable RX checksum
2763          * offload settings using ethtool
2764          */
2765         if (!(netdev->features & NETIF_F_RXCSUM) &&
2766             (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2767             (features & NETIF_F_RXCSUM))
2768                 liquidio_set_rxcsum_command(netdev,
2769                                             OCTNET_CMD_TNL_RX_CSUM_CTL,
2770                                             OCTNET_CMD_RXCSUM_ENABLE);
2771         else if ((netdev->features & NETIF_F_RXCSUM) &&
2772                  (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2773                  !(features & NETIF_F_RXCSUM))
2774                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2775                                             OCTNET_CMD_RXCSUM_DISABLE);
2776
2777         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2778             (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2779             !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2780                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2781                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
2782         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2783                  (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2784                  (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2785                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2786                                      OCTNET_CMD_VLAN_FILTER_DISABLE);
2787
2788         return 0;
2789 }
2790
2791 static void liquidio_add_vxlan_port(struct net_device *netdev,
2792                                     struct udp_tunnel_info *ti)
2793 {
2794         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2795                 return;
2796
2797         liquidio_vxlan_port_command(netdev,
2798                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
2799                                     htons(ti->port),
2800                                     OCTNET_CMD_VXLAN_PORT_ADD);
2801 }
2802
2803 static void liquidio_del_vxlan_port(struct net_device *netdev,
2804                                     struct udp_tunnel_info *ti)
2805 {
2806         if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2807                 return;
2808
2809         liquidio_vxlan_port_command(netdev,
2810                                     OCTNET_CMD_VXLAN_PORT_CONFIG,
2811                                     htons(ti->port),
2812                                     OCTNET_CMD_VXLAN_PORT_DEL);
2813 }
2814
2815 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2816                                  u8 *mac, bool is_admin_assigned)
2817 {
2818         struct lio *lio = GET_LIO(netdev);
2819         struct octeon_device *oct = lio->oct_dev;
2820         struct octnic_ctrl_pkt nctrl;
2821         int ret = 0;
2822
2823         if (!is_valid_ether_addr(mac))
2824                 return -EINVAL;
2825
2826         if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2827                 return -EINVAL;
2828
2829         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2830
2831         nctrl.ncmd.u64 = 0;
2832         nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2833         /* vfidx is 0 based, but vf_num (param1) is 1 based */
2834         nctrl.ncmd.s.param1 = vfidx + 1;
2835         nctrl.ncmd.s.more = 1;
2836         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2837         nctrl.netpndev = (u64)netdev;
2838         if (is_admin_assigned) {
2839                 nctrl.ncmd.s.param2 = true;
2840                 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2841         }
2842
2843         nctrl.udd[0] = 0;
2844         /* The MAC Address is presented in network byte order. */
2845         ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2846
2847         oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2848
2849         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2850         if (ret > 0)
2851                 ret = -EIO;
2852
2853         return ret;
2854 }
2855
2856 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2857 {
2858         struct lio *lio = GET_LIO(netdev);
2859         struct octeon_device *oct = lio->oct_dev;
2860         int retval;
2861
2862         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2863                 return -EINVAL;
2864
2865         retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2866         if (!retval)
2867                 cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2868
2869         return retval;
2870 }
2871
2872 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2873                                     bool enable)
2874 {
2875         struct lio *lio = GET_LIO(netdev);
2876         struct octeon_device *oct = lio->oct_dev;
2877         struct octnic_ctrl_pkt nctrl;
2878         int retval;
2879
2880         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2881                 netif_info(lio, drv, lio->netdev,
2882                            "firmware does not support spoofchk\n");
2883                 return -EOPNOTSUPP;
2884         }
2885
2886         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2887                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2888                 return -EINVAL;
2889         }
2890
2891         if (enable) {
2892                 if (oct->sriov_info.vf_spoofchk[vfidx])
2893                         return 0;
2894         } else {
2895                 /* Clear */
2896                 if (!oct->sriov_info.vf_spoofchk[vfidx])
2897                         return 0;
2898         }
2899
2900         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2901         nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2902         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2903         nctrl.ncmd.s.param1 =
2904                 vfidx + 1; /* vfidx is 0 based,
2905                             * but vf_num (param1) is 1 based
2906                             */
2907         nctrl.ncmd.s.param2 = enable;
2908         nctrl.ncmd.s.more = 0;
2909         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2910         nctrl.cb_fn = NULL;
2911
2912         retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2913
2914         if (retval) {
2915                 netif_info(lio, drv, lio->netdev,
2916                            "Failed to set VF %d spoofchk %s\n", vfidx,
2917                         enable ? "on" : "off");
2918                 return -1;
2919         }
2920
2921         oct->sriov_info.vf_spoofchk[vfidx] = enable;
2922         netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2923                    enable ? "on" : "off");
2924
2925         return 0;
2926 }
2927
2928 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2929                                 u16 vlan, u8 qos, __be16 vlan_proto)
2930 {
2931         struct lio *lio = GET_LIO(netdev);
2932         struct octeon_device *oct = lio->oct_dev;
2933         struct octnic_ctrl_pkt nctrl;
2934         u16 vlantci;
2935         int ret = 0;
2936
2937         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2938                 return -EINVAL;
2939
2940         if (vlan_proto != htons(ETH_P_8021Q))
2941                 return -EPROTONOSUPPORT;
2942
2943         if (vlan >= VLAN_N_VID || qos > 7)
2944                 return -EINVAL;
2945
2946         if (vlan)
2947                 vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2948         else
2949                 vlantci = 0;
2950
2951         if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2952                 return 0;
2953
2954         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2955
2956         if (vlan)
2957                 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2958         else
2959                 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2960
2961         nctrl.ncmd.s.param1 = vlantci;
2962         nctrl.ncmd.s.param2 =
2963             vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2964         nctrl.ncmd.s.more = 0;
2965         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2966         nctrl.cb_fn = NULL;
2967
2968         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2969         if (ret) {
2970                 if (ret > 0)
2971                         ret = -EIO;
2972                 return ret;
2973         }
2974
2975         oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2976
2977         return ret;
2978 }
2979
2980 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2981                                   struct ifla_vf_info *ivi)
2982 {
2983         struct lio *lio = GET_LIO(netdev);
2984         struct octeon_device *oct = lio->oct_dev;
2985         u8 *macaddr;
2986
2987         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2988                 return -EINVAL;
2989
2990         memset(ivi, 0, sizeof(struct ifla_vf_info));
2991
2992         ivi->vf = vfidx;
2993         macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2994         ether_addr_copy(&ivi->mac[0], macaddr);
2995         ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
2996         ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
2997         if (oct->sriov_info.trusted_vf.active &&
2998             oct->sriov_info.trusted_vf.id == vfidx)
2999                 ivi->trusted = true;
3000         else
3001                 ivi->trusted = false;
3002         ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
3003         ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
3004         ivi->max_tx_rate = lio->linfo.link.s.speed;
3005         ivi->min_tx_rate = 0;
3006
3007         return 0;
3008 }
3009
3010 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
3011 {
3012         struct octeon_device *oct = lio->oct_dev;
3013         struct octeon_soft_command *sc;
3014         int retval;
3015
3016         sc = octeon_alloc_soft_command(oct, 0, 16, 0);
3017         if (!sc)
3018                 return -ENOMEM;
3019
3020         sc->iq_no = lio->linfo.txpciq[0].s.q_no;
3021
3022         /* vfidx is 0 based, but vf_num (param1) is 1 based */
3023         octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
3024                                     OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
3025                                     trusted);
3026
3027         init_completion(&sc->complete);
3028         sc->sc_status = OCTEON_REQUEST_PENDING;
3029
3030         retval = octeon_send_soft_command(oct, sc);
3031         if (retval == IQ_SEND_FAILED) {
3032                 octeon_free_soft_command(oct, sc);
3033                 retval = -1;
3034         } else {
3035                 /* Wait for response or timeout */
3036                 retval = wait_for_sc_completion_timeout(oct, sc, 0);
3037                 if (retval)
3038                         return (retval);
3039
3040                 WRITE_ONCE(sc->caller_is_done, true);
3041         }
3042
3043         return retval;
3044 }
3045
3046 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3047                                  bool setting)
3048 {
3049         struct lio *lio = GET_LIO(netdev);
3050         struct octeon_device *oct = lio->oct_dev;
3051
3052         if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3053                 /* trusted vf is not supported by firmware older than 1.7.1 */
3054                 return -EOPNOTSUPP;
3055         }
3056
3057         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3058                 netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3059                 return -EINVAL;
3060         }
3061
3062         if (setting) {
3063                 /* Set */
3064
3065                 if (oct->sriov_info.trusted_vf.active &&
3066                     oct->sriov_info.trusted_vf.id == vfidx)
3067                         return 0;
3068
3069                 if (oct->sriov_info.trusted_vf.active) {
3070                         netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3071                         return -EPERM;
3072                 }
3073         } else {
3074                 /* Clear */
3075
3076                 if (!oct->sriov_info.trusted_vf.active)
3077                         return 0;
3078         }
3079
3080         if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3081                 if (setting) {
3082                         oct->sriov_info.trusted_vf.id = vfidx;
3083                         oct->sriov_info.trusted_vf.active = true;
3084                 } else {
3085                         oct->sriov_info.trusted_vf.active = false;
3086                 }
3087
3088                 netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3089                            setting ? "" : "not ");
3090         } else {
3091                 netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3092                 return -1;
3093         }
3094
3095         return 0;
3096 }
3097
3098 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3099                                       int linkstate)
3100 {
3101         struct lio *lio = GET_LIO(netdev);
3102         struct octeon_device *oct = lio->oct_dev;
3103         struct octnic_ctrl_pkt nctrl;
3104         int ret = 0;
3105
3106         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3107                 return -EINVAL;
3108
3109         if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3110                 return 0;
3111
3112         memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3113         nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3114         nctrl.ncmd.s.param1 =
3115             vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3116         nctrl.ncmd.s.param2 = linkstate;
3117         nctrl.ncmd.s.more = 0;
3118         nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3119         nctrl.cb_fn = NULL;
3120
3121         ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3122
3123         if (!ret)
3124                 oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3125         else if (ret > 0)
3126                 ret = -EIO;
3127
3128         return ret;
3129 }
3130
3131 static int
3132 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3133 {
3134         struct lio_devlink_priv *priv;
3135         struct octeon_device *oct;
3136
3137         priv = devlink_priv(devlink);
3138         oct = priv->oct;
3139
3140         *mode = oct->eswitch_mode;
3141
3142         return 0;
3143 }
3144
3145 static int
3146 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3147                           struct netlink_ext_ack *extack)
3148 {
3149         struct lio_devlink_priv *priv;
3150         struct octeon_device *oct;
3151         int ret = 0;
3152
3153         priv = devlink_priv(devlink);
3154         oct = priv->oct;
3155
3156         if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3157                 return -EINVAL;
3158
3159         if (oct->eswitch_mode == mode)
3160                 return 0;
3161
3162         switch (mode) {
3163         case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3164                 oct->eswitch_mode = mode;
3165                 ret = lio_vf_rep_create(oct);
3166                 break;
3167
3168         case DEVLINK_ESWITCH_MODE_LEGACY:
3169                 lio_vf_rep_destroy(oct);
3170                 oct->eswitch_mode = mode;
3171                 break;
3172
3173         default:
3174                 ret = -EINVAL;
3175         }
3176
3177         return ret;
3178 }
3179
3180 static const struct devlink_ops liquidio_devlink_ops = {
3181         .eswitch_mode_get = liquidio_eswitch_mode_get,
3182         .eswitch_mode_set = liquidio_eswitch_mode_set,
3183 };
3184
3185 static int
3186 liquidio_get_port_parent_id(struct net_device *dev,
3187                             struct netdev_phys_item_id *ppid)
3188 {
3189         struct lio *lio = GET_LIO(dev);
3190         struct octeon_device *oct = lio->oct_dev;
3191
3192         if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3193                 return -EOPNOTSUPP;
3194
3195         ppid->id_len = ETH_ALEN;
3196         ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3197
3198         return 0;
3199 }
3200
3201 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3202                                  struct ifla_vf_stats *vf_stats)
3203 {
3204         struct lio *lio = GET_LIO(netdev);
3205         struct octeon_device *oct = lio->oct_dev;
3206         struct oct_vf_stats stats;
3207         int ret;
3208
3209         if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3210                 return -EINVAL;
3211
3212         memset(&stats, 0, sizeof(struct oct_vf_stats));
3213         ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3214         if (!ret) {
3215                 vf_stats->rx_packets = stats.rx_packets;
3216                 vf_stats->tx_packets = stats.tx_packets;
3217                 vf_stats->rx_bytes = stats.rx_bytes;
3218                 vf_stats->tx_bytes = stats.tx_bytes;
3219                 vf_stats->broadcast = stats.broadcast;
3220                 vf_stats->multicast = stats.multicast;
3221         }
3222
3223         return ret;
3224 }
3225
3226 static const struct net_device_ops lionetdevops = {
3227         .ndo_open               = liquidio_open,
3228         .ndo_stop               = liquidio_stop,
3229         .ndo_start_xmit         = liquidio_xmit,
3230         .ndo_get_stats64        = liquidio_get_stats64,
3231         .ndo_set_mac_address    = liquidio_set_mac,
3232         .ndo_set_rx_mode        = liquidio_set_mcast_list,
3233         .ndo_tx_timeout         = liquidio_tx_timeout,
3234
3235         .ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3236         .ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3237         .ndo_change_mtu         = liquidio_change_mtu,
3238         .ndo_do_ioctl           = liquidio_ioctl,
3239         .ndo_fix_features       = liquidio_fix_features,
3240         .ndo_set_features       = liquidio_set_features,
3241         .ndo_udp_tunnel_add     = liquidio_add_vxlan_port,
3242         .ndo_udp_tunnel_del     = liquidio_del_vxlan_port,
3243         .ndo_set_vf_mac         = liquidio_set_vf_mac,
3244         .ndo_set_vf_vlan        = liquidio_set_vf_vlan,
3245         .ndo_get_vf_config      = liquidio_get_vf_config,
3246         .ndo_set_vf_spoofchk    = liquidio_set_vf_spoofchk,
3247         .ndo_set_vf_trust       = liquidio_set_vf_trust,
3248         .ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3249         .ndo_get_vf_stats       = liquidio_get_vf_stats,
3250         .ndo_get_port_parent_id = liquidio_get_port_parent_id,
3251 };
3252
3253 /** \brief Entry point for the liquidio module
3254  */
3255 static int __init liquidio_init(void)
3256 {
3257         int i;
3258         struct handshake *hs;
3259
3260         init_completion(&first_stage);
3261
3262         octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3263
3264         if (liquidio_init_pci())
3265                 return -EINVAL;
3266
3267         wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3268
3269         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3270                 hs = &handshake[i];
3271                 if (hs->pci_dev) {
3272                         wait_for_completion(&hs->init);
3273                         if (!hs->init_ok) {
3274                                 /* init handshake failed */
3275                                 dev_err(&hs->pci_dev->dev,
3276                                         "Failed to init device\n");
3277                                 liquidio_deinit_pci();
3278                                 return -EIO;
3279                         }
3280                 }
3281         }
3282
3283         for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3284                 hs = &handshake[i];
3285                 if (hs->pci_dev) {
3286                         wait_for_completion_timeout(&hs->started,
3287                                                     msecs_to_jiffies(30000));
3288                         if (!hs->started_ok) {
3289                                 /* starter handshake failed */
3290                                 dev_err(&hs->pci_dev->dev,
3291                                         "Firmware failed to start\n");
3292                                 liquidio_deinit_pci();
3293                                 return -EIO;
3294                         }
3295                 }
3296         }
3297
3298         return 0;
3299 }
3300
3301 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3302 {
3303         struct octeon_device *oct = (struct octeon_device *)buf;
3304         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3305         int gmxport = 0;
3306         union oct_link_status *ls;
3307         int i;
3308
3309         if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3310                 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3311                         recv_pkt->buffer_size[0],
3312                         recv_pkt->rh.r_nic_info.gmxport);
3313                 goto nic_info_err;
3314         }
3315
3316         gmxport = recv_pkt->rh.r_nic_info.gmxport;
3317         ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3318                 OCT_DROQ_INFO_SIZE);
3319
3320         octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3321         for (i = 0; i < oct->ifcount; i++) {
3322                 if (oct->props[i].gmxport == gmxport) {
3323                         update_link_status(oct->props[i].netdev, ls);
3324                         break;
3325                 }
3326         }
3327
3328 nic_info_err:
3329         for (i = 0; i < recv_pkt->buffer_count; i++)
3330                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
3331         octeon_free_recv_info(recv_info);
3332         return 0;
3333 }
3334
3335 /**
3336  * \brief Setup network interfaces
3337  * @param octeon_dev  octeon device
3338  *
3339  * Called during init time for each device. It assumes the NIC
3340  * is already up and running.  The link information for each
3341  * interface is passed in link_info.
3342  */
3343 static int setup_nic_devices(struct octeon_device *octeon_dev)
3344 {
3345         struct lio *lio = NULL;
3346         struct net_device *netdev;
3347         u8 mac[6], i, j, *fw_ver, *micro_ver;
3348         unsigned long micro;
3349         u32 cur_ver;
3350         struct octeon_soft_command *sc;
3351         struct liquidio_if_cfg_resp *resp;
3352         struct octdev_props *props;
3353         int retval, num_iqueues, num_oqueues;
3354         int max_num_queues = 0;
3355         union oct_nic_if_cfg if_cfg;
3356         unsigned int base_queue;
3357         unsigned int gmx_port_id;
3358         u32 resp_size, data_size;
3359         u32 ifidx_or_pfnum;
3360         struct lio_version *vdata;
3361         struct devlink *devlink;
3362         struct lio_devlink_priv *lio_devlink;
3363
3364         /* This is to handle link status changes */
3365         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3366                                     OPCODE_NIC_INFO,
3367                                     lio_nic_info, octeon_dev);
3368
3369         /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3370          * They are handled directly.
3371          */
3372         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3373                                         free_netbuf);
3374
3375         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3376                                         free_netsgbuf);
3377
3378         octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3379                                         free_netsgbuf_with_resp);
3380
3381         for (i = 0; i < octeon_dev->ifcount; i++) {
3382                 resp_size = sizeof(struct liquidio_if_cfg_resp);
3383                 data_size = sizeof(struct lio_version);
3384                 sc = (struct octeon_soft_command *)
3385                         octeon_alloc_soft_command(octeon_dev, data_size,
3386                                                   resp_size, 0);
3387                 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3388                 vdata = (struct lio_version *)sc->virtdptr;
3389
3390                 *((u64 *)vdata) = 0;
3391                 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3392                 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3393                 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3394
3395                 if (OCTEON_CN23XX_PF(octeon_dev)) {
3396                         num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3397                         num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3398                         base_queue = octeon_dev->sriov_info.pf_srn;
3399
3400                         gmx_port_id = octeon_dev->pf_num;
3401                         ifidx_or_pfnum = octeon_dev->pf_num;
3402                 } else {
3403                         num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3404                                                 octeon_get_conf(octeon_dev), i);
3405                         num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3406                                                 octeon_get_conf(octeon_dev), i);
3407                         base_queue = CFG_GET_BASE_QUE_NIC_IF(
3408                                                 octeon_get_conf(octeon_dev), i);
3409                         gmx_port_id = CFG_GET_GMXID_NIC_IF(
3410                                                 octeon_get_conf(octeon_dev), i);
3411                         ifidx_or_pfnum = i;
3412                 }
3413
3414                 dev_dbg(&octeon_dev->pci_dev->dev,
3415                         "requesting config for interface %d, iqs %d, oqs %d\n",
3416                         ifidx_or_pfnum, num_iqueues, num_oqueues);
3417
3418                 if_cfg.u64 = 0;
3419                 if_cfg.s.num_iqueues = num_iqueues;
3420                 if_cfg.s.num_oqueues = num_oqueues;
3421                 if_cfg.s.base_queue = base_queue;
3422                 if_cfg.s.gmx_port_id = gmx_port_id;
3423
3424                 sc->iq_no = 0;
3425
3426                 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3427                                             OPCODE_NIC_IF_CFG, 0,
3428                                             if_cfg.u64, 0);
3429
3430                 init_completion(&sc->complete);
3431                 sc->sc_status = OCTEON_REQUEST_PENDING;
3432
3433                 retval = octeon_send_soft_command(octeon_dev, sc);
3434                 if (retval == IQ_SEND_FAILED) {
3435                         dev_err(&octeon_dev->pci_dev->dev,
3436                                 "iq/oq config failed status: %x\n",
3437                                 retval);
3438                         /* Soft instr is freed by driver in case of failure. */
3439                         octeon_free_soft_command(octeon_dev, sc);
3440                         return(-EIO);
3441                 }
3442
3443                 /* Sleep on a wait queue till the cond flag indicates that the
3444                  * response arrived or timed-out.
3445                  */
3446                 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3447                 if (retval)
3448                         return retval;
3449
3450                 retval = resp->status;
3451                 if (retval) {
3452                         dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3453                         WRITE_ONCE(sc->caller_is_done, true);
3454                         goto setup_nic_dev_done;
3455                 }
3456                 snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3457                          32, "%s",
3458                          resp->cfg_info.liquidio_firmware_version);
3459
3460                 /* Verify f/w version (in case of 'auto' loading from flash) */
3461                 fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3462                 if (memcmp(LIQUIDIO_BASE_VERSION,
3463                            fw_ver,
3464                            strlen(LIQUIDIO_BASE_VERSION))) {
3465                         dev_err(&octeon_dev->pci_dev->dev,
3466                                 "Unmatched firmware version. Expected %s.x, got %s.\n",
3467                                 LIQUIDIO_BASE_VERSION, fw_ver);
3468                         WRITE_ONCE(sc->caller_is_done, true);
3469                         goto setup_nic_dev_done;
3470                 } else if (atomic_read(octeon_dev->adapter_fw_state) ==
3471                            FW_IS_PRELOADED) {
3472                         dev_info(&octeon_dev->pci_dev->dev,
3473                                  "Using auto-loaded firmware version %s.\n",
3474                                  fw_ver);
3475                 }
3476
3477                 /* extract micro version field; point past '<maj>.<min>.' */
3478                 micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3479                 if (kstrtoul(micro_ver, 10, &micro) != 0)
3480                         micro = 0;
3481                 octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3482                 octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3483                 octeon_dev->fw_info.ver.rev = micro;
3484
3485                 octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3486                                     (sizeof(struct liquidio_if_cfg_info)) >> 3);
3487
3488                 num_iqueues = hweight64(resp->cfg_info.iqmask);
3489                 num_oqueues = hweight64(resp->cfg_info.oqmask);
3490
3491                 if (!(num_iqueues) || !(num_oqueues)) {
3492                         dev_err(&octeon_dev->pci_dev->dev,
3493                                 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3494                                 resp->cfg_info.iqmask,
3495                                 resp->cfg_info.oqmask);
3496                         WRITE_ONCE(sc->caller_is_done, true);
3497                         goto setup_nic_dev_done;
3498                 }
3499
3500                 if (OCTEON_CN6XXX(octeon_dev)) {
3501                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3502                                                                     cn6xxx));
3503                 } else if (OCTEON_CN23XX_PF(octeon_dev)) {
3504                         max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3505                                                                     cn23xx_pf));
3506                 }
3507
3508                 dev_dbg(&octeon_dev->pci_dev->dev,
3509                         "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3510                         i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3511                         num_iqueues, num_oqueues, max_num_queues);
3512                 netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3513
3514                 if (!netdev) {
3515                         dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3516                         WRITE_ONCE(sc->caller_is_done, true);
3517                         goto setup_nic_dev_done;
3518                 }
3519
3520                 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3521
3522                 /* Associate the routines that will handle different
3523                  * netdev tasks.
3524                  */
3525                 netdev->netdev_ops = &lionetdevops;
3526
3527                 retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3528                 if (retval) {
3529                         dev_err(&octeon_dev->pci_dev->dev,
3530                                 "setting real number rx failed\n");
3531                         WRITE_ONCE(sc->caller_is_done, true);
3532                         goto setup_nic_dev_free;
3533                 }
3534
3535                 retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3536                 if (retval) {
3537                         dev_err(&octeon_dev->pci_dev->dev,
3538                                 "setting real number tx failed\n");
3539                         WRITE_ONCE(sc->caller_is_done, true);
3540                         goto setup_nic_dev_free;
3541                 }
3542
3543                 lio = GET_LIO(netdev);
3544
3545                 memset(lio, 0, sizeof(struct lio));
3546
3547                 lio->ifidx = ifidx_or_pfnum;
3548
3549                 props = &octeon_dev->props[i];
3550                 props->gmxport = resp->cfg_info.linfo.gmxport;
3551                 props->netdev = netdev;
3552
3553                 lio->linfo.num_rxpciq = num_oqueues;
3554                 lio->linfo.num_txpciq = num_iqueues;
3555                 for (j = 0; j < num_oqueues; j++) {
3556                         lio->linfo.rxpciq[j].u64 =
3557                                 resp->cfg_info.linfo.rxpciq[j].u64;
3558                 }
3559                 for (j = 0; j < num_iqueues; j++) {
3560                         lio->linfo.txpciq[j].u64 =
3561                                 resp->cfg_info.linfo.txpciq[j].u64;
3562                 }
3563                 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3564                 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3565                 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3566
3567                 WRITE_ONCE(sc->caller_is_done, true);
3568
3569                 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3570
3571                 if (OCTEON_CN23XX_PF(octeon_dev) ||
3572                     OCTEON_CN6XXX(octeon_dev)) {
3573                         lio->dev_capability = NETIF_F_HIGHDMA
3574                                               | NETIF_F_IP_CSUM
3575                                               | NETIF_F_IPV6_CSUM
3576                                               | NETIF_F_SG | NETIF_F_RXCSUM
3577                                               | NETIF_F_GRO
3578                                               | NETIF_F_TSO | NETIF_F_TSO6
3579                                               | NETIF_F_LRO;
3580                 }
3581                 netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3582
3583                 /*  Copy of transmit encapsulation capabilities:
3584                  *  TSO, TSO6, Checksums for this device
3585                  */
3586                 lio->enc_dev_capability = NETIF_F_IP_CSUM
3587                                           | NETIF_F_IPV6_CSUM
3588                                           | NETIF_F_GSO_UDP_TUNNEL
3589                                           | NETIF_F_HW_CSUM | NETIF_F_SG
3590                                           | NETIF_F_RXCSUM
3591                                           | NETIF_F_TSO | NETIF_F_TSO6
3592                                           | NETIF_F_LRO;
3593
3594                 netdev->hw_enc_features = (lio->enc_dev_capability &
3595                                            ~NETIF_F_LRO);
3596
3597                 lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3598
3599                 netdev->vlan_features = lio->dev_capability;
3600                 /* Add any unchangeable hw features */
3601                 lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3602                                         NETIF_F_HW_VLAN_CTAG_RX |
3603                                         NETIF_F_HW_VLAN_CTAG_TX;
3604
3605                 netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3606
3607                 netdev->hw_features = lio->dev_capability;
3608                 /*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3609                 netdev->hw_features = netdev->hw_features &
3610                         ~NETIF_F_HW_VLAN_CTAG_RX;
3611
3612                 /* MTU range: 68 - 16000 */
3613                 netdev->min_mtu = LIO_MIN_MTU_SIZE;
3614                 netdev->max_mtu = LIO_MAX_MTU_SIZE;
3615
3616                 /* Point to the  properties for octeon device to which this
3617                  * interface belongs.
3618                  */
3619                 lio->oct_dev = octeon_dev;
3620                 lio->octprops = props;
3621                 lio->netdev = netdev;
3622
3623                 dev_dbg(&octeon_dev->pci_dev->dev,
3624                         "if%d gmx: %d hw_addr: 0x%llx\n", i,
3625                         lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3626
3627                 for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3628                         u8 vfmac[ETH_ALEN];
3629
3630                         eth_random_addr(vfmac);
3631                         if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3632                                 dev_err(&octeon_dev->pci_dev->dev,
3633                                         "Error setting VF%d MAC address\n",
3634                                         j);
3635                                 goto setup_nic_dev_free;
3636                         }
3637                 }
3638
3639                 /* 64-bit swap required on LE machines */
3640                 octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3641                 for (j = 0; j < 6; j++)
3642                         mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3643
3644                 /* Copy MAC Address to OS network device structure */
3645
3646                 ether_addr_copy(netdev->dev_addr, mac);
3647
3648                 /* By default all interfaces on a single Octeon uses the same
3649                  * tx and rx queues
3650                  */
3651                 lio->txq = lio->linfo.txpciq[0].s.q_no;
3652                 lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3653                 if (liquidio_setup_io_queues(octeon_dev, i,
3654                                              lio->linfo.num_txpciq,
3655                                              lio->linfo.num_rxpciq)) {
3656                         dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3657                         goto setup_nic_dev_free;
3658                 }
3659
3660                 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3661
3662                 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3663                 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3664
3665                 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3666                         dev_err(&octeon_dev->pci_dev->dev,
3667                                 "Gather list allocation failed\n");
3668                         goto setup_nic_dev_free;
3669                 }
3670
3671                 /* Register ethtool support */
3672                 liquidio_set_ethtool_ops(netdev);
3673                 if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3674                         octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3675                 else
3676                         octeon_dev->priv_flags = 0x0;
3677
3678                 if (netdev->features & NETIF_F_LRO)
3679                         liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3680                                              OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3681
3682                 liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3683                                      OCTNET_CMD_VLAN_FILTER_ENABLE);
3684
3685                 if ((debug != -1) && (debug & NETIF_MSG_HW))
3686                         liquidio_set_feature(netdev,
3687                                              OCTNET_CMD_VERBOSE_ENABLE, 0);
3688
3689                 if (setup_link_status_change_wq(netdev))
3690                         goto setup_nic_dev_free;
3691
3692                 if ((octeon_dev->fw_info.app_cap_flags &
3693                      LIQUIDIO_TIME_SYNC_CAP) &&
3694                     setup_sync_octeon_time_wq(netdev))
3695                         goto setup_nic_dev_free;
3696
3697                 if (setup_rx_oom_poll_fn(netdev))
3698                         goto setup_nic_dev_free;
3699
3700                 /* Register the network device with the OS */
3701                 if (register_netdev(netdev)) {
3702                         dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3703                         goto setup_nic_dev_free;
3704                 }
3705
3706                 dev_dbg(&octeon_dev->pci_dev->dev,
3707                         "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3708                         i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3709                 netif_carrier_off(netdev);
3710                 lio->link_changes++;
3711
3712                 ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3713
3714                 /* Sending command to firmware to enable Rx checksum offload
3715                  * by default at the time of setup of Liquidio driver for
3716                  * this device
3717                  */
3718                 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3719                                             OCTNET_CMD_RXCSUM_ENABLE);
3720                 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3721                                      OCTNET_CMD_TXCSUM_ENABLE);
3722
3723                 dev_dbg(&octeon_dev->pci_dev->dev,
3724                         "NIC ifidx:%d Setup successful\n", i);
3725
3726                 if (octeon_dev->subsystem_id ==
3727                         OCTEON_CN2350_25GB_SUBSYS_ID ||
3728                     octeon_dev->subsystem_id ==
3729                         OCTEON_CN2360_25GB_SUBSYS_ID) {
3730                         cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3731                                              octeon_dev->fw_info.ver.min,
3732                                              octeon_dev->fw_info.ver.rev);
3733
3734                         /* speed control unsupported in f/w older than 1.7.2 */
3735                         if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3736                                 dev_info(&octeon_dev->pci_dev->dev,
3737                                          "speed setting not supported by f/w.");
3738                                 octeon_dev->speed_setting = 25;
3739                                 octeon_dev->no_speed_setting = 1;
3740                         } else {
3741                                 liquidio_get_speed(lio);
3742                         }
3743
3744                         if (octeon_dev->speed_setting == 0) {
3745                                 octeon_dev->speed_setting = 25;
3746                                 octeon_dev->no_speed_setting = 1;
3747                         }
3748                 } else {
3749                         octeon_dev->no_speed_setting = 1;
3750                         octeon_dev->speed_setting = 10;
3751                 }
3752                 octeon_dev->speed_boot = octeon_dev->speed_setting;
3753
3754                 /* don't read FEC setting if unsupported by f/w (see above) */
3755                 if (octeon_dev->speed_boot == 25 &&
3756                     !octeon_dev->no_speed_setting) {
3757                         liquidio_get_fec(lio);
3758                         octeon_dev->props[lio->ifidx].fec_boot =
3759                                 octeon_dev->props[lio->ifidx].fec;
3760                 }
3761         }
3762
3763         devlink = devlink_alloc(&liquidio_devlink_ops,
3764                                 sizeof(struct lio_devlink_priv));
3765         if (!devlink) {
3766                 dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3767                 goto setup_nic_dev_free;
3768         }
3769
3770         lio_devlink = devlink_priv(devlink);
3771         lio_devlink->oct = octeon_dev;
3772
3773         if (devlink_register(devlink, &octeon_dev->pci_dev->dev)) {
3774                 devlink_free(devlink);
3775                 dev_err(&octeon_dev->pci_dev->dev,
3776                         "devlink registration failed\n");
3777                 goto setup_nic_dev_free;
3778         }
3779
3780         octeon_dev->devlink = devlink;
3781         octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3782
3783         return 0;
3784
3785 setup_nic_dev_free:
3786
3787         while (i--) {
3788                 dev_err(&octeon_dev->pci_dev->dev,
3789                         "NIC ifidx:%d Setup failed\n", i);
3790                 liquidio_destroy_nic_device(octeon_dev, i);
3791         }
3792
3793 setup_nic_dev_done:
3794
3795         return -ENODEV;
3796 }
3797
3798 #ifdef CONFIG_PCI_IOV
3799 static int octeon_enable_sriov(struct octeon_device *oct)
3800 {
3801         unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3802         struct pci_dev *vfdev;
3803         int err;
3804         u32 u;
3805
3806         if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3807                 err = pci_enable_sriov(oct->pci_dev,
3808                                        oct->sriov_info.num_vfs_alloced);
3809                 if (err) {
3810                         dev_err(&oct->pci_dev->dev,
3811                                 "OCTEON: Failed to enable PCI sriov: %d\n",
3812                                 err);
3813                         oct->sriov_info.num_vfs_alloced = 0;
3814                         return err;
3815                 }
3816                 oct->sriov_info.sriov_enabled = 1;
3817
3818                 /* init lookup table that maps DPI ring number to VF pci_dev
3819                  * struct pointer
3820                  */
3821                 u = 0;
3822                 vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3823                                        OCTEON_CN23XX_VF_VID, NULL);
3824                 while (vfdev) {
3825                         if (vfdev->is_virtfn &&
3826                             (vfdev->physfn == oct->pci_dev)) {
3827                                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3828                                         vfdev;
3829                                 u += oct->sriov_info.rings_per_vf;
3830                         }
3831                         vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3832                                                OCTEON_CN23XX_VF_VID, vfdev);
3833                 }
3834         }
3835
3836         return num_vfs_alloced;
3837 }
3838
3839 static int lio_pci_sriov_disable(struct octeon_device *oct)
3840 {
3841         int u;
3842
3843         if (pci_vfs_assigned(oct->pci_dev)) {
3844                 dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3845                 return -EPERM;
3846         }
3847
3848         pci_disable_sriov(oct->pci_dev);
3849
3850         u = 0;
3851         while (u < MAX_POSSIBLE_VFS) {
3852                 oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3853                 u += oct->sriov_info.rings_per_vf;
3854         }
3855
3856         oct->sriov_info.num_vfs_alloced = 0;
3857         dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3858                  oct->pf_num);
3859
3860         return 0;
3861 }
3862
3863 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3864 {
3865         struct octeon_device *oct = pci_get_drvdata(dev);
3866         int ret = 0;
3867
3868         if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3869             (oct->sriov_info.sriov_enabled)) {
3870                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3871                          oct->pf_num, num_vfs);
3872                 return 0;
3873         }
3874
3875         if (!num_vfs) {
3876                 lio_vf_rep_destroy(oct);
3877                 ret = lio_pci_sriov_disable(oct);
3878         } else if (num_vfs > oct->sriov_info.max_vfs) {
3879                 dev_err(&oct->pci_dev->dev,
3880                         "OCTEON: Max allowed VFs:%d user requested:%d",
3881                         oct->sriov_info.max_vfs, num_vfs);
3882                 ret = -EPERM;
3883         } else {
3884                 oct->sriov_info.num_vfs_alloced = num_vfs;
3885                 ret = octeon_enable_sriov(oct);
3886                 dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3887                          oct->pf_num, num_vfs);
3888                 ret = lio_vf_rep_create(oct);
3889                 if (ret)
3890                         dev_info(&oct->pci_dev->dev,
3891                                  "vf representor create failed");
3892         }
3893
3894         return ret;
3895 }
3896 #endif
3897
3898 /**
3899  * \brief initialize the NIC
3900  * @param oct octeon device
3901  *
3902  * This initialization routine is called once the Octeon device application is
3903  * up and running
3904  */
3905 static int liquidio_init_nic_module(struct octeon_device *oct)
3906 {
3907         int i, retval = 0;
3908         int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3909
3910         dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3911
3912         /* only default iq and oq were initialized
3913          * initialize the rest as well
3914          */
3915         /* run port_config command for each port */
3916         oct->ifcount = num_nic_ports;
3917
3918         memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3919
3920         for (i = 0; i < MAX_OCTEON_LINKS; i++)
3921                 oct->props[i].gmxport = -1;
3922
3923         retval = setup_nic_devices(oct);
3924         if (retval) {
3925                 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3926                 goto octnet_init_failure;
3927         }
3928
3929         /* Call vf_rep_modinit if the firmware is switchdev capable
3930          * and do it from the first liquidio function probed.
3931          */
3932         if (!oct->octeon_id &&
3933             oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3934                 retval = lio_vf_rep_modinit();
3935                 if (retval) {
3936                         liquidio_stop_nic_module(oct);
3937                         goto octnet_init_failure;
3938                 }
3939         }
3940
3941         liquidio_ptp_init(oct);
3942
3943         dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3944
3945         return retval;
3946
3947 octnet_init_failure:
3948
3949         oct->ifcount = 0;
3950
3951         return retval;
3952 }
3953
3954 /**
3955  * \brief starter callback that invokes the remaining initialization work after
3956  * the NIC is up and running.
3957  * @param octptr  work struct work_struct
3958  */
3959 static void nic_starter(struct work_struct *work)
3960 {
3961         struct octeon_device *oct;
3962         struct cavium_wk *wk = (struct cavium_wk *)work;
3963
3964         oct = (struct octeon_device *)wk->ctxptr;
3965
3966         if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3967                 return;
3968
3969         /* If the status of the device is CORE_OK, the core
3970          * application has reported its application type. Call
3971          * any registered handlers now and move to the RUNNING
3972          * state.
3973          */
3974         if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3975                 schedule_delayed_work(&oct->nic_poll_work.work,
3976                                       LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3977                 return;
3978         }
3979
3980         atomic_set(&oct->status, OCT_DEV_RUNNING);
3981
3982         if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3983                 dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3984
3985                 if (liquidio_init_nic_module(oct))
3986                         dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3987                 else
3988                         handshake[oct->octeon_id].started_ok = 1;
3989         } else {
3990                 dev_err(&oct->pci_dev->dev,
3991                         "Unexpected application running on NIC (%d). Check firmware.\n",
3992                         oct->app_mode);
3993         }
3994
3995         complete(&handshake[oct->octeon_id].started);
3996 }
3997
3998 static int
3999 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
4000 {
4001         struct octeon_device *oct = (struct octeon_device *)buf;
4002         struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
4003         int i, notice, vf_idx;
4004         bool cores_crashed;
4005         u64 *data, vf_num;
4006
4007         notice = recv_pkt->rh.r.ossp;
4008         data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4009
4010         /* the first 64-bit word of data is the vf_num */
4011         vf_num = data[0];
4012         octeon_swap_8B_data(&vf_num, 1);
4013         vf_idx = (int)vf_num - 1;
4014
4015         cores_crashed = READ_ONCE(oct->cores_crashed);
4016
4017         if (notice == VF_DRV_LOADED) {
4018                 if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4019                         oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4020                         dev_info(&oct->pci_dev->dev,
4021                                  "driver for VF%d was loaded\n", vf_idx);
4022                         if (!cores_crashed)
4023                                 try_module_get(THIS_MODULE);
4024                 }
4025         } else if (notice == VF_DRV_REMOVED) {
4026                 if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4027                         oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4028                         dev_info(&oct->pci_dev->dev,
4029                                  "driver for VF%d was removed\n", vf_idx);
4030                         if (!cores_crashed)
4031                                 module_put(THIS_MODULE);
4032                 }
4033         } else if (notice == VF_DRV_MACADDR_CHANGED) {
4034                 u8 *b = (u8 *)&data[1];
4035
4036                 oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4037                 dev_info(&oct->pci_dev->dev,
4038                          "VF driver changed VF%d's MAC address to %pM\n",
4039                          vf_idx, b + 2);
4040         }
4041
4042         for (i = 0; i < recv_pkt->buffer_count; i++)
4043                 recv_buffer_free(recv_pkt->buffer_ptr[i]);
4044         octeon_free_recv_info(recv_info);
4045
4046         return 0;
4047 }
4048
4049 /**
4050  * \brief Device initialization for each Octeon device that is probed
4051  * @param octeon_dev  octeon device
4052  */
4053 static int octeon_device_init(struct octeon_device *octeon_dev)
4054 {
4055         int j, ret;
4056         char bootcmd[] = "\n";
4057         char *dbg_enb = NULL;
4058         enum lio_fw_state fw_state;
4059         struct octeon_device_priv *oct_priv =
4060                 (struct octeon_device_priv *)octeon_dev->priv;
4061         atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4062
4063         /* Enable access to the octeon device and make its DMA capability
4064          * known to the OS.
4065          */
4066         if (octeon_pci_os_setup(octeon_dev))
4067                 return 1;
4068
4069         atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4070
4071         /* Identify the Octeon type and map the BAR address space. */
4072         if (octeon_chip_specific_setup(octeon_dev)) {
4073                 dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4074                 return 1;
4075         }
4076
4077         atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4078
4079         /* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4080          * since that is what is required for the reference to be removed
4081          * during de-initialization (see 'octeon_destroy_resources').
4082          */
4083         octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4084                                PCI_SLOT(octeon_dev->pci_dev->devfn),
4085                                PCI_FUNC(octeon_dev->pci_dev->devfn),
4086                                true);
4087
4088         octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4089
4090         /* CN23XX supports preloaded firmware if the following is true:
4091          *
4092          * The adapter indicates that firmware is currently running AND
4093          * 'fw_type' is 'auto'.
4094          *
4095          * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4096          */
4097         if (OCTEON_CN23XX_PF(octeon_dev) &&
4098             cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4099                 atomic_cmpxchg(octeon_dev->adapter_fw_state,
4100                                FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4101         }
4102
4103         /* If loading firmware, only first device of adapter needs to do so. */
4104         fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4105                                   FW_NEEDS_TO_BE_LOADED,
4106                                   FW_IS_BEING_LOADED);
4107
4108         /* Here, [local variable] 'fw_state' is set to one of:
4109          *
4110          *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)
4111          *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4112          *                          firmware to the adapter.
4113          *   FW_IS_BEING_LOADED:    The driver's second instance will not load
4114          *                          firmware to the adapter.
4115          */
4116
4117         /* Prior to f/w load, perform a soft reset of the Octeon device;
4118          * if error resetting, return w/error.
4119          */
4120         if (fw_state == FW_NEEDS_TO_BE_LOADED)
4121                 if (octeon_dev->fn_list.soft_reset(octeon_dev))
4122                         return 1;
4123
4124         /* Initialize the dispatch mechanism used to push packets arriving on
4125          * Octeon Output queues.
4126          */
4127         if (octeon_init_dispatch_list(octeon_dev))
4128                 return 1;
4129
4130         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4131                                     OPCODE_NIC_CORE_DRV_ACTIVE,
4132                                     octeon_core_drv_init,
4133                                     octeon_dev);
4134
4135         octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4136                                     OPCODE_NIC_VF_DRV_NOTICE,
4137                                     octeon_recv_vf_drv_notice, octeon_dev);
4138         INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4139         octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4140         schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4141                               LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4142
4143         atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4144
4145         if (octeon_set_io_queues_off(octeon_dev)) {
4146                 dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4147                 return 1;
4148         }
4149
4150         if (OCTEON_CN23XX_PF(octeon_dev)) {
4151                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4152                 if (ret) {
4153                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4154                         return ret;
4155                 }
4156         }
4157
4158         /* Initialize soft command buffer pool
4159          */
4160         if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4161                 dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4162                 return 1;
4163         }
4164         atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4165
4166         /*  Setup the data structures that manage this Octeon's Input queues. */
4167         if (octeon_setup_instr_queues(octeon_dev)) {
4168                 dev_err(&octeon_dev->pci_dev->dev,
4169                         "instruction queue initialization failed\n");
4170                 return 1;
4171         }
4172         atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4173
4174         /* Initialize lists to manage the requests of different types that
4175          * arrive from user & kernel applications for this octeon device.
4176          */
4177         if (octeon_setup_response_list(octeon_dev)) {
4178                 dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4179                 return 1;
4180         }
4181         atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4182
4183         if (octeon_setup_output_queues(octeon_dev)) {
4184                 dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4185                 return 1;
4186         }
4187
4188         atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4189
4190         if (OCTEON_CN23XX_PF(octeon_dev)) {
4191                 if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4192                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4193                         return 1;
4194                 }
4195                 atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4196
4197                 if (octeon_allocate_ioq_vector
4198                                 (octeon_dev,
4199                                  octeon_dev->sriov_info.num_pf_rings)) {
4200                         dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4201                         return 1;
4202                 }
4203                 atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4204
4205         } else {
4206                 /* The input and output queue registers were setup earlier (the
4207                  * queues were not enabled). Any additional registers
4208                  * that need to be programmed should be done now.
4209                  */
4210                 ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4211                 if (ret) {
4212                         dev_err(&octeon_dev->pci_dev->dev,
4213                                 "Failed to configure device registers\n");
4214                         return ret;
4215                 }
4216         }
4217
4218         /* Initialize the tasklet that handles output queue packet processing.*/
4219         dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4220         tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4221                      (unsigned long)octeon_dev);
4222
4223         /* Setup the interrupt handler and record the INT SUM register address
4224          */
4225         if (octeon_setup_interrupt(octeon_dev,
4226                                    octeon_dev->sriov_info.num_pf_rings))
4227                 return 1;
4228
4229         /* Enable Octeon device interrupts */
4230         octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4231
4232         atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4233
4234         /* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4235          * the output queue is enabled.
4236          * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4237          * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4238          * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4239          * before any credits have been issued, causing the ring to be reset
4240          * (and the f/w appear to never have started).
4241          */
4242         for (j = 0; j < octeon_dev->num_oqs; j++)
4243                 writel(octeon_dev->droq[j]->max_count,
4244                        octeon_dev->droq[j]->pkts_credit_reg);
4245
4246         /* Enable the input and output queues for this Octeon device */
4247         ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4248         if (ret) {
4249                 dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4250                 return ret;
4251         }
4252
4253         atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4254
4255         if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4256                 dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4257                 if (!ddr_timeout) {
4258                         dev_info(&octeon_dev->pci_dev->dev,
4259                                  "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4260                 }
4261
4262                 schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4263
4264                 /* Wait for the octeon to initialize DDR after the soft-reset.*/
4265                 while (!ddr_timeout) {
4266                         set_current_state(TASK_INTERRUPTIBLE);
4267                         if (schedule_timeout(HZ / 10)) {
4268                                 /* user probably pressed Control-C */
4269                                 return 1;
4270                         }
4271                 }
4272                 ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4273                 if (ret) {
4274                         dev_err(&octeon_dev->pci_dev->dev,
4275                                 "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4276                                 ret);
4277                         return 1;
4278                 }
4279
4280                 if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4281                         dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4282                         return 1;
4283                 }
4284
4285                 /* Divert uboot to take commands from host instead. */
4286                 ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4287
4288                 dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4289                 ret = octeon_init_consoles(octeon_dev);
4290                 if (ret) {
4291                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4292                         return 1;
4293                 }
4294                 /* If console debug enabled, specify empty string to use default
4295                  * enablement ELSE specify NULL string for 'disabled'.
4296                  */
4297                 dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4298                 ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4299                 if (ret) {
4300                         dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4301                         return 1;
4302                 } else if (octeon_console_debug_enabled(0)) {
4303                         /* If console was added AND we're logging console output
4304                          * then set our console print function.
4305                          */
4306                         octeon_dev->console[0].print = octeon_dbg_console_print;
4307                 }
4308
4309                 atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4310
4311                 dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4312                 ret = load_firmware(octeon_dev);
4313                 if (ret) {
4314                         dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4315                         return 1;
4316                 }
4317
4318                 atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4319         }
4320
4321         handshake[octeon_dev->octeon_id].init_ok = 1;
4322         complete(&handshake[octeon_dev->octeon_id].init);
4323
4324         atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4325
4326         return 0;
4327 }
4328
4329 /**
4330  * \brief Debug console print function
4331  * @param octeon_dev  octeon device
4332  * @param console_num console number
4333  * @param prefix      first portion of line to display
4334  * @param suffix      second portion of line to display
4335  *
4336  * The OCTEON debug console outputs entire lines (excluding '\n').
4337  * Normally, the line will be passed in the 'prefix' parameter.
4338  * However, due to buffering, it is possible for a line to be split into two
4339  * parts, in which case they will be passed as the 'prefix' parameter and
4340  * 'suffix' parameter.
4341  */
4342 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4343                                     char *prefix, char *suffix)
4344 {
4345         if (prefix && suffix)
4346                 dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4347                          suffix);
4348         else if (prefix)
4349                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4350         else if (suffix)
4351                 dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4352
4353         return 0;
4354 }
4355
4356 /**
4357  * \brief Exits the module
4358  */
4359 static void __exit liquidio_exit(void)
4360 {
4361         liquidio_deinit_pci();
4362
4363         pr_info("LiquidIO network module is now unloaded\n");
4364 }
4365
4366 module_init(liquidio_init);
4367 module_exit(liquidio_exit);