drivers/net/ethernet/marvell/octeontx2/af/rvu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTx2 RVU Admin Function driver
   3  *
   4  * Copyright (C) 2018 Marvell International Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/delay.h>
  14 #include <linux/irq.h>
  15 #include <linux/pci.h>
  16 #include <linux/sysfs.h>
  17
  18 #include "cgx.h"
  19 #include "rvu.h"
  20 #include "rvu_reg.h"
  21
  22 #define DRV_NAME        "octeontx2-af"
  23 #define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
  24 #define DRV_VERSION     "1.0"
  25
  26 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
  27
  28 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  29                                 struct rvu_block *block, int lf);
  30 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  31                                   struct rvu_block *block, int lf);
  32 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
  33
  34 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
  35                          int type, int num,
  36                          void (mbox_handler)(struct work_struct *),
  37                          void (mbox_up_handler)(struct work_struct *));
  38 enum {
  39         TYPE_AFVF,
  40         TYPE_AFPF,
  41 };
  42
  43 /* Supported devices */
  44 static const struct pci_device_id rvu_id_table[] = {
  45         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },
  46         { 0, }  /* end of table */
  47 };
  48
  49 MODULE_AUTHOR("Marvell International Ltd.");
  50 MODULE_DESCRIPTION(DRV_STRING);
  51 MODULE_LICENSE("GPL v2");
  52 MODULE_VERSION(DRV_VERSION);
  53 MODULE_DEVICE_TABLE(pci, rvu_id_table);
  54
  55 static char *mkex_profile; /* MKEX profile name */
  56 module_param(mkex_profile, charp, 0000);
  57 MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");
  58
  59 /* Poll a RVU block's register 'offset', for a 'zero'
  60  * or 'nonzero' at bits specified by 'mask'
  61  */
  62 int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)
  63 {
  64         unsigned long timeout = jiffies + usecs_to_jiffies(100);
  65         void __iomem *reg;
  66         u64 reg_val;
  67
  68         reg = rvu->afreg_base + ((block << 28) | offset);
  69         while (time_before(jiffies, timeout)) {
  70                 reg_val = readq(reg);
  71                 if (zero && !(reg_val & mask))
  72                         return 0;
  73                 if (!zero && (reg_val & mask))
  74                         return 0;
  75                 usleep_range(1, 5);
  76                 timeout--;
  77         }
  78         return -EBUSY;
  79 }
  80
  81 int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)
  82 {
  83         int id;
  84
  85         if (!rsrc->bmap)
  86                 return -EINVAL;
  87
  88         id = find_first_zero_bit(rsrc->bmap, rsrc->max);
  89         if (id >= rsrc->max)
  90                 return -ENOSPC;
  91
  92         __set_bit(id, rsrc->bmap);
  93
  94         return id;
  95 }
  96
  97 int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)
  98 {
  99         int start;
 100
 101         if (!rsrc->bmap)
 102                 return -EINVAL;
 103
 104         start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 105         if (start >= rsrc->max)
 106                 return -ENOSPC;
 107
 108         bitmap_set(rsrc->bmap, start, nrsrc);
 109         return start;
 110 }
 111
 112 static void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)
 113 {
 114         if (!rsrc->bmap)
 115                 return;
 116         if (start >= rsrc->max)
 117                 return;
 118
 119         bitmap_clear(rsrc->bmap, start, nrsrc);
 120 }
 121
 122 bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)
 123 {
 124         int start;
 125
 126         if (!rsrc->bmap)
 127                 return false;
 128
 129         start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 130         if (start >= rsrc->max)
 131                 return false;
 132
 133         return true;
 134 }
 135
 136 void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)
 137 {
 138         if (!rsrc->bmap)
 139                 return;
 140
 141         __clear_bit(id, rsrc->bmap);
 142 }
 143
 144 int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)
 145 {
 146         int used;
 147
 148         if (!rsrc->bmap)
 149                 return 0;
 150
 151         used = bitmap_weight(rsrc->bmap, rsrc->max);
 152         return (rsrc->max - used);
 153 }
 154
 155 int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)
 156 {
 157         rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),
 158                              sizeof(long), GFP_KERNEL);
 159         if (!rsrc->bmap)
 160                 return -ENOMEM;
 161         return 0;
 162 }
 163
 164 /* Get block LF's HW index from a PF_FUNC's block slot number */
 165 int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)
 166 {
 167         u16 match = 0;
 168         int lf;
 169
 170         mutex_lock(&rvu->rsrc_lock);
 171         for (lf = 0; lf < block->lf.max; lf++) {
 172                 if (block->fn_map[lf] == pcifunc) {
 173                         if (slot == match) {
 174                                 mutex_unlock(&rvu->rsrc_lock);
 175                                 return lf;
 176                         }
 177                         match++;
 178                 }
 179         }
 180         mutex_unlock(&rvu->rsrc_lock);
 181         return -ENODEV;
 182 }
 183
 184 /* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.
 185  * Some silicon variants of OcteonTX2 supports
 186  * multiple blocks of same type.
 187  *
 188  * @pcifunc has to be zero when no LF is yet attached.
 189  */
 190 int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)
 191 {
 192         int devnum, blkaddr = -ENODEV;
 193         u64 cfg, reg;
 194         bool is_pf;
 195
 196         switch (blktype) {
 197         case BLKTYPE_NPC:
 198                 blkaddr = BLKADDR_NPC;
 199                 goto exit;
 200         case BLKTYPE_NPA:
 201                 blkaddr = BLKADDR_NPA;
 202                 goto exit;
 203         case BLKTYPE_NIX:
 204                 /* For now assume NIX0 */
 205                 if (!pcifunc) {
 206                         blkaddr = BLKADDR_NIX0;
 207                         goto exit;
 208                 }
 209                 break;
 210         case BLKTYPE_SSO:
 211                 blkaddr = BLKADDR_SSO;
 212                 goto exit;
 213         case BLKTYPE_SSOW:
 214                 blkaddr = BLKADDR_SSOW;
 215                 goto exit;
 216         case BLKTYPE_TIM:
 217                 blkaddr = BLKADDR_TIM;
 218                 goto exit;
 219         case BLKTYPE_CPT:
 220                 /* For now assume CPT0 */
 221                 if (!pcifunc) {
 222                         blkaddr = BLKADDR_CPT0;
 223                         goto exit;
 224                 }
 225                 break;
 226         }
 227
 228         /* Check if this is a RVU PF or VF */
 229         if (pcifunc & RVU_PFVF_FUNC_MASK) {
 230                 is_pf = false;
 231                 devnum = rvu_get_hwvf(rvu, pcifunc);
 232         } else {
 233                 is_pf = true;
 234                 devnum = rvu_get_pf(pcifunc);
 235         }
 236
 237         /* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' */
 238         if (blktype == BLKTYPE_NIX) {
 239                 reg = is_pf ? RVU_PRIV_PFX_NIX0_CFG : RVU_PRIV_HWVFX_NIX0_CFG;
 240                 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 241                 if (cfg)
 242                         blkaddr = BLKADDR_NIX0;
 243         }
 244
 245         /* Check if the 'pcifunc' has a CPT LF from 'BLKADDR_CPT0' */
 246         if (blktype == BLKTYPE_CPT) {
 247                 reg = is_pf ? RVU_PRIV_PFX_CPT0_CFG : RVU_PRIV_HWVFX_CPT0_CFG;
 248                 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 249                 if (cfg)
 250                         blkaddr = BLKADDR_CPT0;
 251         }
 252
 253 exit:
 254         if (is_block_implemented(rvu->hw, blkaddr))
 255                 return blkaddr;
 256         return -ENODEV;
 257 }
 258
 259 static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,
 260                                 struct rvu_block *block, u16 pcifunc,
 261                                 u16 lf, bool attach)
 262 {
 263         int devnum, num_lfs = 0;
 264         bool is_pf;
 265         u64 reg;
 266
 267         if (lf >= block->lf.max) {
 268                 dev_err(&rvu->pdev->dev,
 269                         "%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
 270                         __func__, lf, block->name, block->lf.max);
 271                 return;
 272         }
 273
 274         /* Check if this is for a RVU PF or VF */
 275         if (pcifunc & RVU_PFVF_FUNC_MASK) {
 276                 is_pf = false;
 277                 devnum = rvu_get_hwvf(rvu, pcifunc);
 278         } else {
 279                 is_pf = true;
 280                 devnum = rvu_get_pf(pcifunc);
 281         }
 282
 283         block->fn_map[lf] = attach ? pcifunc : 0;
 284
 285         switch (block->type) {
 286         case BLKTYPE_NPA:
 287                 pfvf->npalf = attach ? true : false;
 288                 num_lfs = pfvf->npalf;
 289                 break;
 290         case BLKTYPE_NIX:
 291                 pfvf->nixlf = attach ? true : false;
 292                 num_lfs = pfvf->nixlf;
 293                 break;
 294         case BLKTYPE_SSO:
 295                 attach ? pfvf->sso++ : pfvf->sso--;
 296                 num_lfs = pfvf->sso;
 297                 break;
 298         case BLKTYPE_SSOW:
 299                 attach ? pfvf->ssow++ : pfvf->ssow--;
 300                 num_lfs = pfvf->ssow;
 301                 break;
 302         case BLKTYPE_TIM:
 303                 attach ? pfvf->timlfs++ : pfvf->timlfs--;
 304                 num_lfs = pfvf->timlfs;
 305                 break;
 306         case BLKTYPE_CPT:
 307                 attach ? pfvf->cptlfs++ : pfvf->cptlfs--;
 308                 num_lfs = pfvf->cptlfs;
 309                 break;
 310         }
 311
 312         reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;
 313         rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);
 314 }
 315
 316 inline int rvu_get_pf(u16 pcifunc)
 317 {
 318         return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
 319 }
 320
 321 void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
 322 {
 323         u64 cfg;
 324
 325         /* Get numVFs attached to this PF and first HWVF */
 326         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 327         *numvfs = (cfg >> 12) & 0xFF;
 328         *hwvf = cfg & 0xFFF;
 329 }
 330
 331 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
 332 {
 333         int pf, func;
 334         u64 cfg;
 335
 336         pf = rvu_get_pf(pcifunc);
 337         func = pcifunc & RVU_PFVF_FUNC_MASK;
 338
 339         /* Get first HWVF attached to this PF */
 340         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 341
 342         return ((cfg & 0xFFF) + func - 1);
 343 }
 344
 345 struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
 346 {
 347         /* Check if it is a PF or VF */
 348         if (pcifunc & RVU_PFVF_FUNC_MASK)
 349                 return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];
 350         else
 351                 return &rvu->pf[rvu_get_pf(pcifunc)];
 352 }
 353
 354 static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
 355 {
 356         int pf, vf, nvfs;
 357         u64 cfg;
 358
 359         pf = rvu_get_pf(pcifunc);
 360         if (pf >= rvu->hw->total_pfs)
 361                 return false;
 362
 363         if (!(pcifunc & RVU_PFVF_FUNC_MASK))
 364                 return true;
 365
 366         /* Check if VF is within number of VFs attached to this PF */
 367         vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
 368         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 369         nvfs = (cfg >> 12) & 0xFF;
 370         if (vf >= nvfs)
 371                 return false;
 372
 373         return true;
 374 }
 375
 376 bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)
 377 {
 378         struct rvu_block *block;
 379
 380         if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)
 381                 return false;
 382
 383         block = &hw->block[blkaddr];
 384         return block->implemented;
 385 }
 386
 387 static void rvu_check_block_implemented(struct rvu *rvu)
 388 {
 389         struct rvu_hwinfo *hw = rvu->hw;
 390         struct rvu_block *block;
 391         int blkid;
 392         u64 cfg;
 393
 394         /* For each block check if 'implemented' bit is set */
 395         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
 396                 block = &hw->block[blkid];
 397                 cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));
 398                 if (cfg & BIT_ULL(11))
 399                         block->implemented = true;
 400         }
 401 }
 402
 403 int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)
 404 {
 405         int err;
 406
 407         if (!block->implemented)
 408                 return 0;
 409
 410         rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));
 411         err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),
 412                            true);
 413         return err;
 414 }
 415
 416 static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
 417 {
 418         struct rvu_block *block = &rvu->hw->block[blkaddr];
 419
 420         if (!block->implemented)
 421                 return;
 422
 423         rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
 424         rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
 425 }
 426
 427 static void rvu_reset_all_blocks(struct rvu *rvu)
 428 {
 429         /* Do a HW reset of all RVU blocks */
 430         rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);
 431         rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);
 432         rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);
 433         rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);
 434         rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);
 435         rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);
 436         rvu_block_reset(rvu, BLKADDR_NDC_NIX0_RX, NDC_AF_BLK_RST);
 437         rvu_block_reset(rvu, BLKADDR_NDC_NIX0_TX, NDC_AF_BLK_RST);
 438         rvu_block_reset(rvu, BLKADDR_NDC_NPA0, NDC_AF_BLK_RST);
 439 }
 440
 441 static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)
 442 {
 443         struct rvu_pfvf *pfvf;
 444         u64 cfg;
 445         int lf;
 446
 447         for (lf = 0; lf < block->lf.max; lf++) {
 448                 cfg = rvu_read64(rvu, block->addr,
 449                                  block->lfcfg_reg | (lf << block->lfshift));
 450                 if (!(cfg & BIT_ULL(63)))
 451                         continue;
 452
 453                 /* Set this resource as being used */
 454                 __set_bit(lf, block->lf.bmap);
 455
 456                 /* Get, to whom this LF is attached */
 457                 pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);
 458                 rvu_update_rsrc_map(rvu, pfvf, block,
 459                                     (cfg >> 8) & 0xFFFF, lf, true);
 460
 461                 /* Set start MSIX vector for this LF within this PF/VF */
 462                 rvu_set_msix_offset(rvu, pfvf, block, lf);
 463         }
 464 }
 465
 466 static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
 467 {
 468         int min_vecs;
 469
 470         if (!vf)
 471                 goto check_pf;
 472
 473         if (!nvecs) {
 474                 dev_warn(rvu->dev,
 475                          "PF%d:VF%d is configured with zero msix vectors, %d\n",
 476                          pf, vf - 1, nvecs);
 477         }
 478         return;
 479
 480 check_pf:
 481         if (pf == 0)
 482                 min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;
 483         else
 484                 min_vecs = RVU_PF_INT_VEC_CNT;
 485
 486         if (!(nvecs < min_vecs))
 487                 return;
 488         dev_warn(rvu->dev,
 489                  "PF%d is configured with too few vectors, %d, min is %d\n",
 490                  pf, nvecs, min_vecs);
 491 }
 492
 493 static int rvu_setup_msix_resources(struct rvu *rvu)
 494 {
 495         struct rvu_hwinfo *hw = rvu->hw;
 496         int pf, vf, numvfs, hwvf, err;
 497         int nvecs, offset, max_msix;
 498         struct rvu_pfvf *pfvf;
 499         u64 cfg, phy_addr;
 500         dma_addr_t iova;
 501
 502         for (pf = 0; pf < hw->total_pfs; pf++) {
 503                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 504                 /* If PF is not enabled, nothing to do */
 505                 if (!((cfg >> 20) & 0x01))
 506                         continue;
 507
 508                 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
 509
 510                 pfvf = &rvu->pf[pf];
 511                 /* Get num of MSIX vectors attached to this PF */
 512                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));
 513                 pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;
 514                 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);
 515
 516                 /* Alloc msix bitmap for this PF */
 517                 err = rvu_alloc_bitmap(&pfvf->msix);
 518                 if (err)
 519                         return err;
 520
 521                 /* Allocate memory for MSIX vector to RVU block LF mapping */
 522                 pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,
 523                                                 sizeof(u16), GFP_KERNEL);
 524                 if (!pfvf->msix_lfmap)
 525                         return -ENOMEM;
 526
 527                 /* For PF0 (AF) firmware will set msix vector offsets for
 528                  * AF, block AF and PF0_INT vectors, so jump to VFs.
 529                  */
 530                 if (!pf)
 531                         goto setup_vfmsix;
 532
 533                 /* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.
 534                  * These are allocated on driver init and never freed,
 535                  * so no need to set 'msix_lfmap' for these.
 536                  */
 537                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));
 538                 nvecs = (cfg >> 12) & 0xFF;
 539                 cfg &= ~0x7FFULL;
 540                 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 541                 rvu_write64(rvu, BLKADDR_RVUM,
 542                             RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);
 543 setup_vfmsix:
 544                 /* Alloc msix bitmap for VFs */
 545                 for (vf = 0; vf < numvfs; vf++) {
 546                         pfvf =  &rvu->hwvf[hwvf + vf];
 547                         /* Get num of MSIX vectors attached to this VF */
 548                         cfg = rvu_read64(rvu, BLKADDR_RVUM,
 549                                          RVU_PRIV_PFX_MSIX_CFG(pf));
 550                         pfvf->msix.max = (cfg & 0xFFF) + 1;
 551                         rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);
 552
 553                         /* Alloc msix bitmap for this VF */
 554                         err = rvu_alloc_bitmap(&pfvf->msix);
 555                         if (err)
 556                                 return err;
 557
 558                         pfvf->msix_lfmap =
 559                                 devm_kcalloc(rvu->dev, pfvf->msix.max,
 560                                              sizeof(u16), GFP_KERNEL);
 561                         if (!pfvf->msix_lfmap)
 562                                 return -ENOMEM;
 563
 564                         /* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.
 565                          * These are allocated on driver init and never freed,
 566                          * so no need to set 'msix_lfmap' for these.
 567                          */
 568                         cfg = rvu_read64(rvu, BLKADDR_RVUM,
 569                                          RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));
 570                         nvecs = (cfg >> 12) & 0xFF;
 571                         cfg &= ~0x7FFULL;
 572                         offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 573                         rvu_write64(rvu, BLKADDR_RVUM,
 574                                     RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),
 575                                     cfg | offset);
 576                 }
 577         }
 578
 579         /* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence
 580          * create a IOMMU mapping for the physcial address configured by
 581          * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.
 582          */
 583         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 584         max_msix = cfg & 0xFFFFF;
 585         phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);
 586         iova = dma_map_resource(rvu->dev, phy_addr,
 587                                 max_msix * PCI_MSIX_ENTRY_SIZE,
 588                                 DMA_BIDIRECTIONAL, 0);
 589
 590         if (dma_mapping_error(rvu->dev, iova))
 591                 return -ENOMEM;
 592
 593         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);
 594         rvu->msix_base_iova = iova;
 595
 596         return 0;
 597 }
 598
 599 static void rvu_free_hw_resources(struct rvu *rvu)
 600 {
 601         struct rvu_hwinfo *hw = rvu->hw;
 602         struct rvu_block *block;
 603         struct rvu_pfvf  *pfvf;
 604         int id, max_msix;
 605         u64 cfg;
 606
 607         rvu_npa_freemem(rvu);
 608         rvu_npc_freemem(rvu);
 609         rvu_nix_freemem(rvu);
 610
 611         /* Free block LF bitmaps */
 612         for (id = 0; id < BLK_COUNT; id++) {
 613                 block = &hw->block[id];
 614                 kfree(block->lf.bmap);
 615         }
 616
 617         /* Free MSIX bitmaps */
 618         for (id = 0; id < hw->total_pfs; id++) {
 619                 pfvf = &rvu->pf[id];
 620                 kfree(pfvf->msix.bmap);
 621         }
 622
 623         for (id = 0; id < hw->total_vfs; id++) {
 624                 pfvf = &rvu->hwvf[id];
 625                 kfree(pfvf->msix.bmap);
 626         }
 627
 628         /* Unmap MSIX vector base IOVA mapping */
 629         if (!rvu->msix_base_iova)
 630                 return;
 631         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 632         max_msix = cfg & 0xFFFFF;
 633         dma_unmap_resource(rvu->dev, rvu->msix_base_iova,
 634                            max_msix * PCI_MSIX_ENTRY_SIZE,
 635                            DMA_BIDIRECTIONAL, 0);
 636
 637         mutex_destroy(&rvu->rsrc_lock);
 638 }
 639
 640 static int rvu_setup_hw_resources(struct rvu *rvu)
 641 {
 642         struct rvu_hwinfo *hw = rvu->hw;
 643         struct rvu_block *block;
 644         int blkid, err;
 645         u64 cfg;
 646
 647         /* Get HW supported max RVU PF & VF count */
 648         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 649         hw->total_pfs = (cfg >> 32) & 0xFF;
 650         hw->total_vfs = (cfg >> 20) & 0xFFF;
 651         hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;
 652
 653         /* Init NPA LF's bitmap */
 654         block = &hw->block[BLKADDR_NPA];
 655         if (!block->implemented)
 656                 goto nix;
 657         cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);
 658         block->lf.max = (cfg >> 16) & 0xFFF;
 659         block->addr = BLKADDR_NPA;
 660         block->type = BLKTYPE_NPA;
 661         block->lfshift = 8;
 662         block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;
 663         block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;
 664         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;
 665         block->lfcfg_reg = NPA_PRIV_LFX_CFG;
 666         block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;
 667         block->lfreset_reg = NPA_AF_LF_RST;
 668         sprintf(block->name, "NPA");
 669         err = rvu_alloc_bitmap(&block->lf);
 670         if (err)
 671                 return err;
 672
 673 nix:
 674         /* Init NIX LF's bitmap */
 675         block = &hw->block[BLKADDR_NIX0];
 676         if (!block->implemented)
 677                 goto sso;
 678         cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2);
 679         block->lf.max = cfg & 0xFFF;
 680         block->addr = BLKADDR_NIX0;
 681         block->type = BLKTYPE_NIX;
 682         block->lfshift = 8;
 683         block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;
 684         block->pf_lfcnt_reg = RVU_PRIV_PFX_NIX0_CFG;
 685         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIX0_CFG;
 686         block->lfcfg_reg = NIX_PRIV_LFX_CFG;
 687         block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;
 688         block->lfreset_reg = NIX_AF_LF_RST;
 689         sprintf(block->name, "NIX");
 690         err = rvu_alloc_bitmap(&block->lf);
 691         if (err)
 692                 return err;
 693
 694 sso:
 695         /* Init SSO group's bitmap */
 696         block = &hw->block[BLKADDR_SSO];
 697         if (!block->implemented)
 698                 goto ssow;
 699         cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);
 700         block->lf.max = cfg & 0xFFFF;
 701         block->addr = BLKADDR_SSO;
 702         block->type = BLKTYPE_SSO;
 703         block->multislot = true;
 704         block->lfshift = 3;
 705         block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;
 706         block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;
 707         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;
 708         block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;
 709         block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;
 710         block->lfreset_reg = SSO_AF_LF_HWGRP_RST;
 711         sprintf(block->name, "SSO GROUP");
 712         err = rvu_alloc_bitmap(&block->lf);
 713         if (err)
 714                 return err;
 715
 716 ssow:
 717         /* Init SSO workslot's bitmap */
 718         block = &hw->block[BLKADDR_SSOW];
 719         if (!block->implemented)
 720                 goto tim;
 721         block->lf.max = (cfg >> 56) & 0xFF;
 722         block->addr = BLKADDR_SSOW;
 723         block->type = BLKTYPE_SSOW;
 724         block->multislot = true;
 725         block->lfshift = 3;
 726         block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;
 727         block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;
 728         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;
 729         block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;
 730         block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;
 731         block->lfreset_reg = SSOW_AF_LF_HWS_RST;
 732         sprintf(block->name, "SSOWS");
 733         err = rvu_alloc_bitmap(&block->lf);
 734         if (err)
 735                 return err;
 736
 737 tim:
 738         /* Init TIM LF's bitmap */
 739         block = &hw->block[BLKADDR_TIM];
 740         if (!block->implemented)
 741                 goto cpt;
 742         cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);
 743         block->lf.max = cfg & 0xFFFF;
 744         block->addr = BLKADDR_TIM;
 745         block->type = BLKTYPE_TIM;
 746         block->multislot = true;
 747         block->lfshift = 3;
 748         block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;
 749         block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;
 750         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;
 751         block->lfcfg_reg = TIM_PRIV_LFX_CFG;
 752         block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;
 753         block->lfreset_reg = TIM_AF_LF_RST;
 754         sprintf(block->name, "TIM");
 755         err = rvu_alloc_bitmap(&block->lf);
 756         if (err)
 757                 return err;
 758
 759 cpt:
 760         /* Init CPT LF's bitmap */
 761         block = &hw->block[BLKADDR_CPT0];
 762         if (!block->implemented)
 763                 goto init;
 764         cfg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS0);
 765         block->lf.max = cfg & 0xFF;
 766         block->addr = BLKADDR_CPT0;
 767         block->type = BLKTYPE_CPT;
 768         block->multislot = true;
 769         block->lfshift = 3;
 770         block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;
 771         block->pf_lfcnt_reg = RVU_PRIV_PFX_CPT0_CFG;
 772         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPT0_CFG;
 773         block->lfcfg_reg = CPT_PRIV_LFX_CFG;
 774         block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;
 775         block->lfreset_reg = CPT_AF_LF_RST;
 776         sprintf(block->name, "CPT");
 777         err = rvu_alloc_bitmap(&block->lf);
 778         if (err)
 779                 return err;
 780
 781 init:
 782         /* Allocate memory for PFVF data */
 783         rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,
 784                                sizeof(struct rvu_pfvf), GFP_KERNEL);
 785         if (!rvu->pf)
 786                 return -ENOMEM;
 787
 788         rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,
 789                                  sizeof(struct rvu_pfvf), GFP_KERNEL);
 790         if (!rvu->hwvf)
 791                 return -ENOMEM;
 792
 793         mutex_init(&rvu->rsrc_lock);
 794
 795         err = rvu_setup_msix_resources(rvu);
 796         if (err)
 797                 return err;
 798
 799         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
 800                 block = &hw->block[blkid];
 801                 if (!block->lf.bmap)
 802                         continue;
 803
 804                 /* Allocate memory for block LF/slot to pcifunc mapping info */
 805                 block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,
 806                                              sizeof(u16), GFP_KERNEL);
 807                 if (!block->fn_map)
 808                         return -ENOMEM;
 809
 810                 /* Scan all blocks to check if low level firmware has
 811                  * already provisioned any of the resources to a PF/VF.
 812                  */
 813                 rvu_scan_block(rvu, block);
 814         }
 815
 816         err = rvu_npc_init(rvu);
 817         if (err)
 818                 goto exit;
 819
 820         err = rvu_cgx_init(rvu);
 821         if (err)
 822                 goto exit;
 823
 824         err = rvu_npa_init(rvu);
 825         if (err)
 826                 goto cgx_err;
 827
 828         err = rvu_nix_init(rvu);
 829         if (err)
 830                 goto cgx_err;
 831
 832         return 0;
 833
 834 cgx_err:
 835         rvu_cgx_exit(rvu);
 836 exit:
 837         return err;
 838 }
 839
 840 /* NPA and NIX admin queue APIs */
 841 void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)
 842 {
 843         if (!aq)
 844                 return;
 845
 846         qmem_free(rvu->dev, aq->inst);
 847         qmem_free(rvu->dev, aq->res);
 848         devm_kfree(rvu->dev, aq);
 849 }
 850
 851 int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,
 852                  int qsize, int inst_size, int res_size)
 853 {
 854         struct admin_queue *aq;
 855         int err;
 856
 857         *ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);
 858         if (!*ad_queue)
 859                 return -ENOMEM;
 860         aq = *ad_queue;
 861
 862         /* Alloc memory for instructions i.e AQ */
 863         err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);
 864         if (err) {
 865                 devm_kfree(rvu->dev, aq);
 866                 return err;
 867         }
 868
 869         /* Alloc memory for results */
 870         err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);
 871         if (err) {
 872                 rvu_aq_free(rvu, aq);
 873                 return err;
 874         }
 875
 876         spin_lock_init(&aq->lock);
 877         return 0;
 878 }
 879
 880 static int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,
 881                                   struct ready_msg_rsp *rsp)
 882 {
 883         return 0;
 884 }
 885
 886 /* Get current count of a RVU block's LF/slots
 887  * provisioned to a given RVU func.
 888  */
 889 static u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blktype)
 890 {
 891         switch (blktype) {
 892         case BLKTYPE_NPA:
 893                 return pfvf->npalf ? 1 : 0;
 894         case BLKTYPE_NIX:
 895                 return pfvf->nixlf ? 1 : 0;
 896         case BLKTYPE_SSO:
 897                 return pfvf->sso;
 898         case BLKTYPE_SSOW:
 899                 return pfvf->ssow;
 900         case BLKTYPE_TIM:
 901                 return pfvf->timlfs;
 902         case BLKTYPE_CPT:
 903                 return pfvf->cptlfs;
 904         }
 905         return 0;
 906 }
 907
 908 bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)
 909 {
 910         struct rvu_pfvf *pfvf;
 911
 912         if (!is_pf_func_valid(rvu, pcifunc))
 913                 return false;
 914
 915         pfvf = rvu_get_pfvf(rvu, pcifunc);
 916
 917         /* Check if this PFFUNC has a LF of type blktype attached */
 918         if (!rvu_get_rsrc_mapcount(pfvf, blktype))
 919                 return false;
 920
 921         return true;
 922 }
 923
 924 static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,
 925                            int pcifunc, int slot)
 926 {
 927         u64 val;
 928
 929         val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);
 930         rvu_write64(rvu, block->addr, block->lookup_reg, val);
 931         /* Wait for the lookup to finish */
 932         /* TODO: put some timeout here */
 933         while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))
 934                 ;
 935
 936         val = rvu_read64(rvu, block->addr, block->lookup_reg);
 937
 938         /* Check LF valid bit */
 939         if (!(val & (1ULL << 12)))
 940                 return -1;
 941
 942         return (val & 0xFFF);
 943 }
 944
 945 static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
 946 {
 947         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 948         struct rvu_hwinfo *hw = rvu->hw;
 949         struct rvu_block *block;
 950         int slot, lf, num_lfs;
 951         int blkaddr;
 952
 953         blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);
 954         if (blkaddr < 0)
 955                 return;
 956
 957         block = &hw->block[blkaddr];
 958
 959         num_lfs = rvu_get_rsrc_mapcount(pfvf, block->type);
 960         if (!num_lfs)
 961                 return;
 962
 963         for (slot = 0; slot < num_lfs; slot++) {
 964                 lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);
 965                 if (lf < 0) /* This should never happen */
 966                         continue;
 967
 968                 /* Disable the LF */
 969                 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
 970                             (lf << block->lfshift), 0x00ULL);
 971
 972                 /* Update SW maintained mapping info as well */
 973                 rvu_update_rsrc_map(rvu, pfvf, block,
 974                                     pcifunc, lf, false);
 975
 976                 /* Free the resource */
 977                 rvu_free_rsrc(&block->lf, lf);
 978
 979                 /* Clear MSIX vector offset for this LF */
 980                 rvu_clear_msix_offset(rvu, pfvf, block, lf);
 981         }
 982 }
 983
 984 static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
 985                             u16 pcifunc)
 986 {
 987         struct rvu_hwinfo *hw = rvu->hw;
 988         bool detach_all = true;
 989         struct rvu_block *block;
 990         int blkid;
 991
 992         mutex_lock(&rvu->rsrc_lock);
 993
 994         /* Check for partial resource detach */
 995         if (detach && detach->partial)
 996                 detach_all = false;
 997
 998         /* Check for RVU block's LFs attached to this func,
 999          * if so, detach them.
1000          */
1001         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1002                 block = &hw->block[blkid];
1003                 if (!block->lf.bmap)
1004                         continue;
1005                 if (!detach_all && detach) {
1006                         if (blkid == BLKADDR_NPA && !detach->npalf)
1007                                 continue;
1008                         else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)
1009                                 continue;
1010                         else if ((blkid == BLKADDR_SSO) && !detach->sso)
1011                                 continue;
1012                         else if ((blkid == BLKADDR_SSOW) && !detach->ssow)
1013                                 continue;
1014                         else if ((blkid == BLKADDR_TIM) && !detach->timlfs)
1015                                 continue;
1016                         else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)
1017                                 continue;
1018                 }
1019                 rvu_detach_block(rvu, pcifunc, block->type);
1020         }
1021
1022         mutex_unlock(&rvu->rsrc_lock);
1023         return 0;
1024 }
1025
1026 static int rvu_mbox_handler_detach_resources(struct rvu *rvu,
1027                                              struct rsrc_detach *detach,
1028                                              struct msg_rsp *rsp)
1029 {
1030         return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);
1031 }
1032
1033 static void rvu_attach_block(struct rvu *rvu, int pcifunc,
1034                              int blktype, int num_lfs)
1035 {
1036         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1037         struct rvu_hwinfo *hw = rvu->hw;
1038         struct rvu_block *block;
1039         int slot, lf;
1040         int blkaddr;
1041         u64 cfg;
1042
1043         if (!num_lfs)
1044                 return;
1045
1046         blkaddr = rvu_get_blkaddr(rvu, blktype, 0);
1047         if (blkaddr < 0)
1048                 return;
1049
1050         block = &hw->block[blkaddr];
1051         if (!block->lf.bmap)
1052                 return;
1053
1054         for (slot = 0; slot < num_lfs; slot++) {
1055                 /* Allocate the resource */
1056                 lf = rvu_alloc_rsrc(&block->lf);
1057                 if (lf < 0)
1058                         return;
1059
1060                 cfg = (1ULL << 63) | (pcifunc << 8) | slot;
1061                 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1062                             (lf << block->lfshift), cfg);
1063                 rvu_update_rsrc_map(rvu, pfvf, block,
1064                                     pcifunc, lf, true);
1065
1066                 /* Set start MSIX vector for this LF within this PF/VF */
1067                 rvu_set_msix_offset(rvu, pfvf, block, lf);
1068         }
1069 }
1070
1071 static int rvu_check_rsrc_availability(struct rvu *rvu,
1072                                        struct rsrc_attach *req, u16 pcifunc)
1073 {
1074         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1075         struct rvu_hwinfo *hw = rvu->hw;
1076         struct rvu_block *block;
1077         int free_lfs, mappedlfs;
1078
1079         /* Only one NPA LF can be attached */
1080         if (req->npalf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NPA)) {
1081                 block = &hw->block[BLKADDR_NPA];
1082                 free_lfs = rvu_rsrc_free_count(&block->lf);
1083                 if (!free_lfs)
1084                         goto fail;
1085         } else if (req->npalf) {
1086                 dev_err(&rvu->pdev->dev,
1087                         "Func 0x%x: Invalid req, already has NPA\n",
1088                          pcifunc);
1089                 return -EINVAL;
1090         }
1091
1092         /* Only one NIX LF can be attached */
1093         if (req->nixlf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NIX)) {
1094                 block = &hw->block[BLKADDR_NIX0];
1095                 free_lfs = rvu_rsrc_free_count(&block->lf);
1096                 if (!free_lfs)
1097                         goto fail;
1098         } else if (req->nixlf) {
1099                 dev_err(&rvu->pdev->dev,
1100                         "Func 0x%x: Invalid req, already has NIX\n",
1101                         pcifunc);
1102                 return -EINVAL;
1103         }
1104
1105         if (req->sso) {
1106                 block = &hw->block[BLKADDR_SSO];
1107                 /* Is request within limits ? */
1108                 if (req->sso > block->lf.max) {
1109                         dev_err(&rvu->pdev->dev,
1110                                 "Func 0x%x: Invalid SSO req, %d > max %d\n",
1111                                  pcifunc, req->sso, block->lf.max);
1112                         return -EINVAL;
1113                 }
1114                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1115                 free_lfs = rvu_rsrc_free_count(&block->lf);
1116                 /* Check if additional resources are available */
1117                 if (req->sso > mappedlfs &&
1118                     ((req->sso - mappedlfs) > free_lfs))
1119                         goto fail;
1120         }
1121
1122         if (req->ssow) {
1123                 block = &hw->block[BLKADDR_SSOW];
1124                 if (req->ssow > block->lf.max) {
1125                         dev_err(&rvu->pdev->dev,
1126                                 "Func 0x%x: Invalid SSOW req, %d > max %d\n",
1127                                  pcifunc, req->sso, block->lf.max);
1128                         return -EINVAL;
1129                 }
1130                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1131                 free_lfs = rvu_rsrc_free_count(&block->lf);
1132                 if (req->ssow > mappedlfs &&
1133                     ((req->ssow - mappedlfs) > free_lfs))
1134                         goto fail;
1135         }
1136
1137         if (req->timlfs) {
1138                 block = &hw->block[BLKADDR_TIM];
1139                 if (req->timlfs > block->lf.max) {
1140                         dev_err(&rvu->pdev->dev,
1141                                 "Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1142                                  pcifunc, req->timlfs, block->lf.max);
1143                         return -EINVAL;
1144                 }
1145                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1146                 free_lfs = rvu_rsrc_free_count(&block->lf);
1147                 if (req->timlfs > mappedlfs &&
1148                     ((req->timlfs - mappedlfs) > free_lfs))
1149                         goto fail;
1150         }
1151
1152         if (req->cptlfs) {
1153                 block = &hw->block[BLKADDR_CPT0];
1154                 if (req->cptlfs > block->lf.max) {
1155                         dev_err(&rvu->pdev->dev,
1156                                 "Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1157                                  pcifunc, req->cptlfs, block->lf.max);
1158                         return -EINVAL;
1159                 }
1160                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1161                 free_lfs = rvu_rsrc_free_count(&block->lf);
1162                 if (req->cptlfs > mappedlfs &&
1163                     ((req->cptlfs - mappedlfs) > free_lfs))
1164                         goto fail;
1165         }
1166
1167         return 0;
1168
1169 fail:
1170         dev_info(rvu->dev, "Request for %s failed\n", block->name);
1171         return -ENOSPC;
1172 }
1173
1174 static int rvu_mbox_handler_attach_resources(struct rvu *rvu,
1175                                              struct rsrc_attach *attach,
1176                                              struct msg_rsp *rsp)
1177 {
1178         u16 pcifunc = attach->hdr.pcifunc;
1179         int err;
1180
1181         /* If first request, detach all existing attached resources */
1182         if (!attach->modify)
1183                 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1184
1185         mutex_lock(&rvu->rsrc_lock);
1186
1187         /* Check if the request can be accommodated */
1188         err = rvu_check_rsrc_availability(rvu, attach, pcifunc);
1189         if (err)
1190                 goto exit;
1191
1192         /* Now attach the requested resources */
1193         if (attach->npalf)
1194                 rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1);
1195
1196         if (attach->nixlf)
1197                 rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1);
1198
1199         if (attach->sso) {
1200                 /* RVU func doesn't know which exact LF or slot is attached
1201                  * to it, it always sees as slot 0,1,2. So for a 'modify'
1202                  * request, simply detach all existing attached LFs/slots
1203                  * and attach a fresh.
1204                  */
1205                 if (attach->modify)
1206                         rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1207                 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso);
1208         }
1209
1210         if (attach->ssow) {
1211                 if (attach->modify)
1212                         rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1213                 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow);
1214         }
1215
1216         if (attach->timlfs) {
1217                 if (attach->modify)
1218                         rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1219                 rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs);
1220         }
1221
1222         if (attach->cptlfs) {
1223                 if (attach->modify)
1224                         rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);
1225                 rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs);
1226         }
1227
1228 exit:
1229         mutex_unlock(&rvu->rsrc_lock);
1230         return err;
1231 }
1232
1233 static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1234                                int blkaddr, int lf)
1235 {
1236         u16 vec;
1237
1238         if (lf < 0)
1239                 return MSIX_VECTOR_INVALID;
1240
1241         for (vec = 0; vec < pfvf->msix.max; vec++) {
1242                 if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))
1243                         return vec;
1244         }
1245         return MSIX_VECTOR_INVALID;
1246 }
1247
1248 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1249                                 struct rvu_block *block, int lf)
1250 {
1251         u16 nvecs, vec, offset;
1252         u64 cfg;
1253
1254         cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1255                          (lf << block->lfshift));
1256         nvecs = (cfg >> 12) & 0xFF;
1257
1258         /* Check and alloc MSIX vectors, must be contiguous */
1259         if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))
1260                 return;
1261
1262         offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
1263
1264         /* Config MSIX offset in LF */
1265         rvu_write64(rvu, block->addr, block->msixcfg_reg |
1266                     (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);
1267
1268         /* Update the bitmap as well */
1269         for (vec = 0; vec < nvecs; vec++)
1270                 pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);
1271 }
1272
1273 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1274                                   struct rvu_block *block, int lf)
1275 {
1276         u16 nvecs, vec, offset;
1277         u64 cfg;
1278
1279         cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1280                          (lf << block->lfshift));
1281         nvecs = (cfg >> 12) & 0xFF;
1282
1283         /* Clear MSIX offset in LF */
1284         rvu_write64(rvu, block->addr, block->msixcfg_reg |
1285                     (lf << block->lfshift), cfg & ~0x7FFULL);
1286
1287         offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);
1288
1289         /* Update the mapping */
1290         for (vec = 0; vec < nvecs; vec++)
1291                 pfvf->msix_lfmap[offset + vec] = 0;
1292
1293         /* Free the same in MSIX bitmap */
1294         rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);
1295 }
1296
1297 static int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,
1298                                         struct msix_offset_rsp *rsp)
1299 {
1300         struct rvu_hwinfo *hw = rvu->hw;
1301         u16 pcifunc = req->hdr.pcifunc;
1302         struct rvu_pfvf *pfvf;
1303         int lf, slot;
1304
1305         pfvf = rvu_get_pfvf(rvu, pcifunc);
1306         if (!pfvf->msix.bmap)
1307                 return 0;
1308
1309         /* Set MSIX offsets for each block's LFs attached to this PF/VF */
1310         lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);
1311         rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);
1312
1313         lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NIX0], pcifunc, 0);
1314         rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NIX0, lf);
1315
1316         rsp->sso = pfvf->sso;
1317         for (slot = 0; slot < rsp->sso; slot++) {
1318                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);
1319                 rsp->sso_msixoff[slot] =
1320                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);
1321         }
1322
1323         rsp->ssow = pfvf->ssow;
1324         for (slot = 0; slot < rsp->ssow; slot++) {
1325                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);
1326                 rsp->ssow_msixoff[slot] =
1327                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);
1328         }
1329
1330         rsp->timlfs = pfvf->timlfs;
1331         for (slot = 0; slot < rsp->timlfs; slot++) {
1332                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);
1333                 rsp->timlf_msixoff[slot] =
1334                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);
1335         }
1336
1337         rsp->cptlfs = pfvf->cptlfs;
1338         for (slot = 0; slot < rsp->cptlfs; slot++) {
1339                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);
1340                 rsp->cptlf_msixoff[slot] =
1341                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);
1342         }
1343         return 0;
1344 }
1345
1346 static int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,
1347                                    struct msg_rsp *rsp)
1348 {
1349         u16 pcifunc = req->hdr.pcifunc;
1350         u16 vf, numvfs;
1351         u64 cfg;
1352
1353         vf = pcifunc & RVU_PFVF_FUNC_MASK;
1354         cfg = rvu_read64(rvu, BLKADDR_RVUM,
1355                          RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc)));
1356         numvfs = (cfg >> 12) & 0xFF;
1357
1358         if (vf && vf <= numvfs)
1359                 __rvu_flr_handler(rvu, pcifunc);
1360         else
1361                 return RVU_INVALID_VF_ID;
1362
1363         return 0;
1364 }
1365
1366 static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
1367                                 struct mbox_msghdr *req)
1368 {
1369         struct rvu *rvu = pci_get_drvdata(mbox->pdev);
1370
1371         /* Check if valid, if not reply with a invalid msg */
1372         if (req->sig != OTX2_MBOX_REQ_SIG)
1373                 goto bad_message;
1374
1375         switch (req->id) {
1376 #define M(_name, _id, _fn_name, _req_type, _rsp_type)                   \
1377         case _id: {                                                     \
1378                 struct _rsp_type *rsp;                                  \
1379                 int err;                                                \
1380                                                                         \
1381                 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg(          \
1382                         mbox, devid,                                    \
1383                         sizeof(struct _rsp_type));                      \
1384                 /* some handlers should complete even if reply */       \
1385                 /* could not be allocated */                            \
1386                 if (!rsp &&                                             \
1387                     _id != MBOX_MSG_DETACH_RESOURCES &&                 \
1388                     _id != MBOX_MSG_NIX_TXSCH_FREE &&                   \
1389                     _id != MBOX_MSG_VF_FLR)                             \
1390                         return -ENOMEM;                                 \
1391                 if (rsp) {                                              \
1392                         rsp->hdr.id = _id;                              \
1393                         rsp->hdr.sig = OTX2_MBOX_RSP_SIG;               \
1394                         rsp->hdr.pcifunc = req->pcifunc;                \
1395                         rsp->hdr.rc = 0;                                \
1396                 }                                                       \
1397                                                                         \
1398                 err = rvu_mbox_handler_ ## _fn_name(rvu,                \
1399                                                     (struct _req_type *)req, \
1400                                                     rsp);               \
1401                 if (rsp && err)                                         \
1402                         rsp->hdr.rc = err;                              \
1403                                                                         \
1404                 return rsp ? err : -ENOMEM;                             \
1405         }
1406 MBOX_MESSAGES
1407 #undef M
1408
1409 bad_message:
1410         default:
1411                 otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);
1412                 return -ENODEV;
1413         }
1414 }
1415
1416 static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
1417 {
1418         struct rvu *rvu = mwork->rvu;
1419         int offset, err, id, devid;
1420         struct otx2_mbox_dev *mdev;
1421         struct mbox_hdr *req_hdr;
1422         struct mbox_msghdr *msg;
1423         struct mbox_wq_info *mw;
1424         struct otx2_mbox *mbox;
1425
1426         switch (type) {
1427         case TYPE_AFPF:
1428                 mw = &rvu->afpf_wq_info;
1429                 break;
1430         case TYPE_AFVF:
1431                 mw = &rvu->afvf_wq_info;
1432                 break;
1433         default:
1434                 return;
1435         }
1436
1437         devid = mwork - mw->mbox_wrk;
1438         mbox = &mw->mbox;
1439         mdev = &mbox->dev[devid];
1440
1441         /* Process received mbox messages */
1442         req_hdr = mdev->mbase + mbox->rx_start;
1443         if (mw->mbox_wrk[devid].num_msgs == 0)
1444                 return;
1445
1446         offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
1447
1448         for (id = 0; id < mw->mbox_wrk[devid].num_msgs; id++) {
1449                 msg = mdev->mbase + offset;
1450
1451                 /* Set which PF/VF sent this message based on mbox IRQ */
1452                 switch (type) {
1453                 case TYPE_AFPF:
1454                         msg->pcifunc &=
1455                                 ~(RVU_PFVF_PF_MASK << RVU_PFVF_PF_SHIFT);
1456                         msg->pcifunc |= (devid << RVU_PFVF_PF_SHIFT);
1457                         break;
1458                 case TYPE_AFVF:
1459                         msg->pcifunc &=
1460                                 ~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);
1461                         msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;
1462                         break;
1463                 }
1464
1465                 err = rvu_process_mbox_msg(mbox, devid, msg);
1466                 if (!err) {
1467                         offset = mbox->rx_start + msg->next_msgoff;
1468                         continue;
1469                 }
1470
1471                 if (msg->pcifunc & RVU_PFVF_FUNC_MASK)
1472                         dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
1473                                  err, otx2_mbox_id2name(msg->id),
1474                                  msg->id, rvu_get_pf(msg->pcifunc),
1475                                  (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);
1476                 else
1477                         dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",
1478                                  err, otx2_mbox_id2name(msg->id),
1479                                  msg->id, devid);
1480         }
1481         mw->mbox_wrk[devid].num_msgs = 0;
1482
1483         /* Send mbox responses to VF/PF */
1484         otx2_mbox_msg_send(mbox, devid);
1485 }
1486
1487 static inline void rvu_afpf_mbox_handler(struct work_struct *work)
1488 {
1489         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1490
1491         __rvu_mbox_handler(mwork, TYPE_AFPF);
1492 }
1493
1494 static inline void rvu_afvf_mbox_handler(struct work_struct *work)
1495 {
1496         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1497
1498         __rvu_mbox_handler(mwork, TYPE_AFVF);
1499 }
1500
1501 static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
1502 {
1503         struct rvu *rvu = mwork->rvu;
1504         struct otx2_mbox_dev *mdev;
1505         struct mbox_hdr *rsp_hdr;
1506         struct mbox_msghdr *msg;
1507         struct mbox_wq_info *mw;
1508         struct otx2_mbox *mbox;
1509         int offset, id, devid;
1510
1511         switch (type) {
1512         case TYPE_AFPF:
1513                 mw = &rvu->afpf_wq_info;
1514                 break;
1515         case TYPE_AFVF:
1516                 mw = &rvu->afvf_wq_info;
1517                 break;
1518         default:
1519                 return;
1520         }
1521
1522         devid = mwork - mw->mbox_wrk_up;
1523         mbox = &mw->mbox_up;
1524         mdev = &mbox->dev[devid];
1525
1526         rsp_hdr = mdev->mbase + mbox->rx_start;
1527         if (mw->mbox_wrk_up[devid].up_num_msgs == 0) {
1528                 dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");
1529                 return;
1530         }
1531
1532         offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1533
1534         for (id = 0; id < mw->mbox_wrk_up[devid].up_num_msgs; id++) {
1535                 msg = mdev->mbase + offset;
1536
1537                 if (msg->id >= MBOX_MSG_MAX) {
1538                         dev_err(rvu->dev,
1539                                 "Mbox msg with unknown ID 0x%x\n", msg->id);
1540                         goto end;
1541                 }
1542
1543                 if (msg->sig != OTX2_MBOX_RSP_SIG) {
1544                         dev_err(rvu->dev,
1545                                 "Mbox msg with wrong signature %x, ID 0x%x\n",
1546                                 msg->sig, msg->id);
1547                         goto end;
1548                 }
1549
1550                 switch (msg->id) {
1551                 case MBOX_MSG_CGX_LINK_EVENT:
1552                         break;
1553                 default:
1554                         if (msg->rc)
1555                                 dev_err(rvu->dev,
1556                                         "Mbox msg response has err %d, ID 0x%x\n",
1557                                         msg->rc, msg->id);
1558                         break;
1559                 }
1560 end:
1561                 offset = mbox->rx_start + msg->next_msgoff;
1562                 mdev->msgs_acked++;
1563         }
1564         mw->mbox_wrk_up[devid].up_num_msgs = 0;
1565
1566         otx2_mbox_reset(mbox, devid);
1567 }
1568
1569 static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)
1570 {
1571         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1572
1573         __rvu_mbox_up_handler(mwork, TYPE_AFPF);
1574 }
1575
1576 static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
1577 {
1578         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1579
1580         __rvu_mbox_up_handler(mwork, TYPE_AFVF);
1581 }
1582
1583 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
1584                          int type, int num,
1585                          void (mbox_handler)(struct work_struct *),
1586                          void (mbox_up_handler)(struct work_struct *))
1587 {
1588         void __iomem *hwbase = NULL, *reg_base;
1589         int err, i, dir, dir_up;
1590         struct rvu_work *mwork;
1591         const char *name;
1592         u64 bar4_addr;
1593
1594         switch (type) {
1595         case TYPE_AFPF:
1596                 name = "rvu_afpf_mailbox";
1597                 bar4_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PF_BAR4_ADDR);
1598                 dir = MBOX_DIR_AFPF;
1599                 dir_up = MBOX_DIR_AFPF_UP;
1600                 reg_base = rvu->afreg_base;
1601                 break;
1602         case TYPE_AFVF:
1603                 name = "rvu_afvf_mailbox";
1604                 bar4_addr = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
1605                 dir = MBOX_DIR_PFVF;
1606                 dir_up = MBOX_DIR_PFVF_UP;
1607                 reg_base = rvu->pfreg_base;
1608                 break;
1609         default:
1610                 return -EINVAL;
1611         }
1612
1613         mw->mbox_wq = alloc_workqueue(name,
1614                                       WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
1615                                       num);
1616         if (!mw->mbox_wq)
1617                 return -ENOMEM;
1618
1619         mw->mbox_wrk = devm_kcalloc(rvu->dev, num,
1620                                     sizeof(struct rvu_work), GFP_KERNEL);
1621         if (!mw->mbox_wrk) {
1622                 err = -ENOMEM;
1623                 goto exit;
1624         }
1625
1626         mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,
1627                                        sizeof(struct rvu_work), GFP_KERNEL);
1628         if (!mw->mbox_wrk_up) {
1629                 err = -ENOMEM;
1630                 goto exit;
1631         }
1632
1633         /* Mailbox is a reserved memory (in RAM) region shared between
1634          * RVU devices, shouldn't be mapped as device memory to allow
1635          * unaligned accesses.
1636          */
1637         hwbase = ioremap_wc(bar4_addr, MBOX_SIZE * num);
1638         if (!hwbase) {
1639                 dev_err(rvu->dev, "Unable to map mailbox region\n");
1640                 err = -ENOMEM;
1641                 goto exit;
1642         }
1643
1644         err = otx2_mbox_init(&mw->mbox, hwbase, rvu->pdev, reg_base, dir, num);
1645         if (err)
1646                 goto exit;
1647
1648         err = otx2_mbox_init(&mw->mbox_up, hwbase, rvu->pdev,
1649                              reg_base, dir_up, num);
1650         if (err)
1651                 goto exit;
1652
1653         for (i = 0; i < num; i++) {
1654                 mwork = &mw->mbox_wrk[i];
1655                 mwork->rvu = rvu;
1656                 INIT_WORK(&mwork->work, mbox_handler);
1657
1658                 mwork = &mw->mbox_wrk_up[i];
1659                 mwork->rvu = rvu;
1660                 INIT_WORK(&mwork->work, mbox_up_handler);
1661         }
1662
1663         return 0;
1664 exit:
1665         if (hwbase)
1666                 iounmap((void __iomem *)hwbase);
1667         destroy_workqueue(mw->mbox_wq);
1668         return err;
1669 }
1670
1671 static void rvu_mbox_destroy(struct mbox_wq_info *mw)
1672 {
1673         if (mw->mbox_wq) {
1674                 flush_workqueue(mw->mbox_wq);
1675                 destroy_workqueue(mw->mbox_wq);
1676                 mw->mbox_wq = NULL;
1677         }
1678
1679         if (mw->mbox.hwbase)
1680                 iounmap((void __iomem *)mw->mbox.hwbase);
1681
1682         otx2_mbox_destroy(&mw->mbox);
1683         otx2_mbox_destroy(&mw->mbox_up);
1684 }
1685
1686 static void rvu_queue_work(struct mbox_wq_info *mw, int first,
1687                            int mdevs, u64 intr)
1688 {
1689         struct otx2_mbox_dev *mdev;
1690         struct otx2_mbox *mbox;
1691         struct mbox_hdr *hdr;
1692         int i;
1693
1694         for (i = first; i < mdevs; i++) {
1695                 /* start from 0 */
1696                 if (!(intr & BIT_ULL(i - first)))
1697                         continue;
1698
1699                 mbox = &mw->mbox;
1700                 mdev = &mbox->dev[i];
1701                 hdr = mdev->mbase + mbox->rx_start;
1702
1703                 /*The hdr->num_msgs is set to zero immediately in the interrupt
1704                  * handler to  ensure that it holds a correct value next time
1705                  * when the interrupt handler is called.
1706                  * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
1707                  * pf>mbox.up_num_msgs holds the data for use in
1708                  * pfaf_mbox_up_handler.
1709                  */
1710
1711                 if (hdr->num_msgs) {
1712                         mw->mbox_wrk[i].num_msgs = hdr->num_msgs;
1713                         hdr->num_msgs = 0;
1714                         queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);
1715                 }
1716                 mbox = &mw->mbox_up;
1717                 mdev = &mbox->dev[i];
1718                 hdr = mdev->mbase + mbox->rx_start;
1719                 if (hdr->num_msgs) {
1720                         mw->mbox_wrk_up[i].up_num_msgs = hdr->num_msgs;
1721                         hdr->num_msgs = 0;
1722                         queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);
1723                 }
1724         }
1725 }
1726
1727 static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
1728 {
1729         struct rvu *rvu = (struct rvu *)rvu_irq;
1730         int vfs = rvu->vfs;
1731         u64 intr;
1732
1733         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
1734         /* Clear interrupts */
1735         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
1736
1737         /* Sync with mbox memory region */
1738         rmb();
1739
1740         rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
1741
1742         /* Handle VF interrupts */
1743         if (vfs > 64) {
1744                 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
1745                 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);
1746
1747                 rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);
1748                 vfs -= 64;
1749         }
1750
1751         intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
1752         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
1753
1754         rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
1755
1756         return IRQ_HANDLED;
1757 }
1758
1759 static void rvu_enable_mbox_intr(struct rvu *rvu)
1760 {
1761         struct rvu_hwinfo *hw = rvu->hw;
1762
1763         /* Clear spurious irqs, if any */
1764         rvu_write64(rvu, BLKADDR_RVUM,
1765                     RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));
1766
1767         /* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */
1768         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,
1769                     INTR_MASK(hw->total_pfs) & ~1ULL);
1770 }
1771
1772 static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
1773 {
1774         struct rvu_block *block;
1775         int slot, lf, num_lfs;
1776         int err;
1777
1778         block = &rvu->hw->block[blkaddr];
1779         num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1780                                         block->type);
1781         if (!num_lfs)
1782                 return;
1783         for (slot = 0; slot < num_lfs; slot++) {
1784                 lf = rvu_get_lf(rvu, block, pcifunc, slot);
1785                 if (lf < 0)
1786                         continue;
1787
1788                 /* Cleanup LF and reset it */
1789                 if (block->addr == BLKADDR_NIX0)
1790                         rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);
1791                 else if (block->addr == BLKADDR_NPA)
1792                         rvu_npa_lf_teardown(rvu, pcifunc, lf);
1793
1794                 err = rvu_lf_reset(rvu, block, lf);
1795                 if (err) {
1796                         dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
1797                                 block->addr, lf);
1798                 }
1799         }
1800 }
1801
1802 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)
1803 {
1804         mutex_lock(&rvu->flr_lock);
1805         /* Reset order should reflect inter-block dependencies:
1806          * 1. Reset any packet/work sources (NIX, CPT, TIM)
1807          * 2. Flush and reset SSO/SSOW
1808          * 3. Cleanup pools (NPA)
1809          */
1810         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);
1811         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);
1812         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);
1813         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);
1814         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);
1815         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);
1816         rvu_detach_rsrcs(rvu, NULL, pcifunc);
1817         mutex_unlock(&rvu->flr_lock);
1818 }
1819
1820 static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)
1821 {
1822         int reg = 0;
1823
1824         /* pcifunc = 0(PF0) | (vf + 1) */
1825         __rvu_flr_handler(rvu, vf + 1);
1826
1827         if (vf >= 64) {
1828                 reg = 1;
1829                 vf = vf - 64;
1830         }
1831
1832         /* Signal FLR finish and enable IRQ */
1833         rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
1834         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
1835 }
1836
1837 static void rvu_flr_handler(struct work_struct *work)
1838 {
1839         struct rvu_work *flrwork = container_of(work, struct rvu_work, work);
1840         struct rvu *rvu = flrwork->rvu;
1841         u16 pcifunc, numvfs, vf;
1842         u64 cfg;
1843         int pf;
1844
1845         pf = flrwork - rvu->flr_wrk;
1846         if (pf >= rvu->hw->total_pfs) {
1847                 rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);
1848                 return;
1849         }
1850
1851         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
1852         numvfs = (cfg >> 12) & 0xFF;
1853         pcifunc  = pf << RVU_PFVF_PF_SHIFT;
1854
1855         for (vf = 0; vf < numvfs; vf++)
1856                 __rvu_flr_handler(rvu, (pcifunc | (vf + 1)));
1857
1858         __rvu_flr_handler(rvu, pcifunc);
1859
1860         /* Signal FLR finish */
1861         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));
1862
1863         /* Enable interrupt */
1864         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,  BIT_ULL(pf));
1865 }
1866
1867 static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)
1868 {
1869         int dev, vf, reg = 0;
1870         u64 intr;
1871
1872         if (start_vf >= 64)
1873                 reg = 1;
1874
1875         intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));
1876         if (!intr)
1877                 return;
1878
1879         for (vf = 0; vf < numvfs; vf++) {
1880                 if (!(intr & BIT_ULL(vf)))
1881                         continue;
1882                 dev = vf + start_vf + rvu->hw->total_pfs;
1883                 queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);
1884                 /* Clear and disable the interrupt */
1885                 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
1886                 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));
1887         }
1888 }
1889
1890 static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)
1891 {
1892         struct rvu *rvu = (struct rvu *)rvu_irq;
1893         u64 intr;
1894         u8  pf;
1895
1896         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);
1897         if (!intr)
1898                 goto afvf_flr;
1899
1900         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1901                 if (intr & (1ULL << pf)) {
1902                         /* PF is already dead do only AF related operations */
1903                         queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);
1904                         /* clear interrupt */
1905                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,
1906                                     BIT_ULL(pf));
1907                         /* Disable the interrupt */
1908                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1909                                     BIT_ULL(pf));
1910                 }
1911         }
1912
1913 afvf_flr:
1914         rvu_afvf_queue_flr_work(rvu, 0, 64);
1915         if (rvu->vfs > 64)
1916                 rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);
1917
1918         return IRQ_HANDLED;
1919 }
1920
1921 static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)
1922 {
1923         int vf;
1924
1925         /* Nothing to be done here other than clearing the
1926          * TRPEND bit.
1927          */
1928         for (vf = 0; vf < 64; vf++) {
1929                 if (intr & (1ULL << vf)) {
1930                         /* clear the trpend due to ME(master enable) */
1931                         rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));
1932                         /* clear interrupt */
1933                         rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));
1934                 }
1935         }
1936 }
1937
1938 /* Handles ME interrupts from VFs of AF */
1939 static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)
1940 {
1941         struct rvu *rvu = (struct rvu *)rvu_irq;
1942         int vfset;
1943         u64 intr;
1944
1945         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1946
1947         for (vfset = 0; vfset <= 1; vfset++) {
1948                 intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));
1949                 if (intr)
1950                         rvu_me_handle_vfset(rvu, vfset, intr);
1951         }
1952
1953         return IRQ_HANDLED;
1954 }
1955
1956 /* Handles ME interrupts from PFs */
1957 static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)
1958 {
1959         struct rvu *rvu = (struct rvu *)rvu_irq;
1960         u64 intr;
1961         u8  pf;
1962
1963         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1964
1965         /* Nothing to be done here other than clearing the
1966          * TRPEND bit.
1967          */
1968         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1969                 if (intr & (1ULL << pf)) {
1970                         /* clear the trpend due to ME(master enable) */
1971                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,
1972                                     BIT_ULL(pf));
1973                         /* clear interrupt */
1974                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,
1975                                     BIT_ULL(pf));
1976                 }
1977         }
1978
1979         return IRQ_HANDLED;
1980 }
1981
1982 static void rvu_unregister_interrupts(struct rvu *rvu)
1983 {
1984         int irq;
1985
1986         /* Disable the Mbox interrupt */
1987         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,
1988                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1989
1990         /* Disable the PF FLR interrupt */
1991         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1992                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1993
1994         /* Disable the PF ME interrupt */
1995         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,
1996                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1997
1998         for (irq = 0; irq < rvu->num_vec; irq++) {
1999                 if (rvu->irq_allocated[irq])
2000                         free_irq(pci_irq_vector(rvu->pdev, irq), rvu);
2001         }
2002
2003         pci_free_irq_vectors(rvu->pdev);
2004         rvu->num_vec = 0;
2005 }
2006
2007 static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
2008 {
2009         struct rvu_pfvf *pfvf = &rvu->pf[0];
2010         int offset;
2011
2012         pfvf = &rvu->pf[0];
2013         offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2014
2015         /* Make sure there are enough MSIX vectors configured so that
2016          * VF interrupts can be handled. Offset equal to zero means
2017          * that PF vectors are not configured and overlapping AF vectors.
2018          */
2019         return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&
2020                offset;
2021 }
2022
2023 static int rvu_register_interrupts(struct rvu *rvu)
2024 {
2025         int ret, offset, pf_vec_start;
2026
2027         rvu->num_vec = pci_msix_vec_count(rvu->pdev);
2028
2029         rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
2030                                            NAME_SIZE, GFP_KERNEL);
2031         if (!rvu->irq_name)
2032                 return -ENOMEM;
2033
2034         rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,
2035                                           sizeof(bool), GFP_KERNEL);
2036         if (!rvu->irq_allocated)
2037                 return -ENOMEM;
2038
2039         /* Enable MSI-X */
2040         ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,
2041                                     rvu->num_vec, PCI_IRQ_MSIX);
2042         if (ret < 0) {
2043                 dev_err(rvu->dev,
2044                         "RVUAF: Request for %d msix vectors failed, ret %d\n",
2045                         rvu->num_vec, ret);
2046                 return ret;
2047         }
2048
2049         /* Register mailbox interrupt handler */
2050         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");
2051         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),
2052                           rvu_mbox_intr_handler, 0,
2053                           &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);
2054         if (ret) {
2055                 dev_err(rvu->dev,
2056                         "RVUAF: IRQ registration failed for mbox irq\n");
2057                 goto fail;
2058         }
2059
2060         rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;
2061
2062         /* Enable mailbox interrupts from all PFs */
2063         rvu_enable_mbox_intr(rvu);
2064
2065         /* Register FLR interrupt handler */
2066         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2067                 "RVUAF FLR");
2068         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),
2069                           rvu_flr_intr_handler, 0,
2070                           &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2071                           rvu);
2072         if (ret) {
2073                 dev_err(rvu->dev,
2074                         "RVUAF: IRQ registration failed for FLR\n");
2075                 goto fail;
2076         }
2077         rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;
2078
2079         /* Enable FLR interrupt for all PFs*/
2080         rvu_write64(rvu, BLKADDR_RVUM,
2081                     RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));
2082
2083         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,
2084                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2085
2086         /* Register ME interrupt handler */
2087         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2088                 "RVUAF ME");
2089         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),
2090                           rvu_me_pf_intr_handler, 0,
2091                           &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2092                           rvu);
2093         if (ret) {
2094                 dev_err(rvu->dev,
2095                         "RVUAF: IRQ registration failed for ME\n");
2096         }
2097         rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;
2098
2099         /* Enable ME interrupt for all PFs*/
2100         rvu_write64(rvu, BLKADDR_RVUM,
2101                     RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));
2102
2103         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,
2104                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2105
2106         if (!rvu_afvf_msix_vectors_num_ok(rvu))
2107                 return 0;
2108
2109         /* Get PF MSIX vectors offset. */
2110         pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,
2111                                   RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2112
2113         /* Register MBOX0 interrupt. */
2114         offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;
2115         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");
2116         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2117                           rvu_mbox_intr_handler, 0,
2118                           &rvu->irq_name[offset * NAME_SIZE],
2119                           rvu);
2120         if (ret)
2121                 dev_err(rvu->dev,
2122                         "RVUAF: IRQ registration failed for Mbox0\n");
2123
2124         rvu->irq_allocated[offset] = true;
2125
2126         /* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so
2127          * simply increment current offset by 1.
2128          */
2129         offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;
2130         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");
2131         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2132                           rvu_mbox_intr_handler, 0,
2133                           &rvu->irq_name[offset * NAME_SIZE],
2134                           rvu);
2135         if (ret)
2136                 dev_err(rvu->dev,
2137                         "RVUAF: IRQ registration failed for Mbox1\n");
2138
2139         rvu->irq_allocated[offset] = true;
2140
2141         /* Register FLR interrupt handler for AF's VFs */
2142         offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;
2143         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");
2144         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2145                           rvu_flr_intr_handler, 0,
2146                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2147         if (ret) {
2148                 dev_err(rvu->dev,
2149                         "RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
2150                 goto fail;
2151         }
2152         rvu->irq_allocated[offset] = true;
2153
2154         offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;
2155         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");
2156         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2157                           rvu_flr_intr_handler, 0,
2158                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2159         if (ret) {
2160                 dev_err(rvu->dev,
2161                         "RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
2162                 goto fail;
2163         }
2164         rvu->irq_allocated[offset] = true;
2165
2166         /* Register ME interrupt handler for AF's VFs */
2167         offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;
2168         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");
2169         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2170                           rvu_me_vf_intr_handler, 0,
2171                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2172         if (ret) {
2173                 dev_err(rvu->dev,
2174                         "RVUAF: IRQ registration failed for RVUAFVF ME0\n");
2175                 goto fail;
2176         }
2177         rvu->irq_allocated[offset] = true;
2178
2179         offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;
2180         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");
2181         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2182                           rvu_me_vf_intr_handler, 0,
2183                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2184         if (ret) {
2185                 dev_err(rvu->dev,
2186                         "RVUAF: IRQ registration failed for RVUAFVF ME1\n");
2187                 goto fail;
2188         }
2189         rvu->irq_allocated[offset] = true;
2190         return 0;
2191
2192 fail:
2193         rvu_unregister_interrupts(rvu);
2194         return ret;
2195 }
2196
2197 static void rvu_flr_wq_destroy(struct rvu *rvu)
2198 {
2199         if (rvu->flr_wq) {
2200                 flush_workqueue(rvu->flr_wq);
2201                 destroy_workqueue(rvu->flr_wq);
2202                 rvu->flr_wq = NULL;
2203         }
2204 }
2205
2206 static int rvu_flr_init(struct rvu *rvu)
2207 {
2208         int dev, num_devs;
2209         u64 cfg;
2210         int pf;
2211
2212         /* Enable FLR for all PFs*/
2213         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2214                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2215                 rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),
2216                             cfg | BIT_ULL(22));
2217         }
2218
2219         rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
2220                                       WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
2221                                        1);
2222         if (!rvu->flr_wq)
2223                 return -ENOMEM;
2224
2225         num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);
2226         rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,
2227                                     sizeof(struct rvu_work), GFP_KERNEL);
2228         if (!rvu->flr_wrk) {
2229                 destroy_workqueue(rvu->flr_wq);
2230                 return -ENOMEM;
2231         }
2232
2233         for (dev = 0; dev < num_devs; dev++) {
2234                 rvu->flr_wrk[dev].rvu = rvu;
2235                 INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);
2236         }
2237
2238         mutex_init(&rvu->flr_lock);
2239
2240         return 0;
2241 }
2242
2243 static void rvu_disable_afvf_intr(struct rvu *rvu)
2244 {
2245         int vfs = rvu->vfs;
2246
2247         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));
2248         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
2249         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
2250         if (vfs <= 64)
2251                 return;
2252
2253         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
2254                       INTR_MASK(vfs - 64));
2255         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2256         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2257 }
2258
2259 static void rvu_enable_afvf_intr(struct rvu *rvu)
2260 {
2261         int vfs = rvu->vfs;
2262
2263         /* Clear any pending interrupts and enable AF VF interrupts for
2264          * the first 64 VFs.
2265          */
2266         /* Mbox */
2267         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));
2268         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));
2269
2270         /* FLR */
2271         rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));
2272         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));
2273         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));
2274
2275         /* Same for remaining VFs, if any. */
2276         if (vfs <= 64)
2277                 return;
2278
2279         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));
2280         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
2281                       INTR_MASK(vfs - 64));
2282
2283         rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));
2284         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2285         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2286 }
2287
2288 #define PCI_DEVID_OCTEONTX2_LBK 0xA061
2289
2290 static int lbk_get_num_chans(void)
2291 {
2292         struct pci_dev *pdev;
2293         void __iomem *base;
2294         int ret = -EIO;
2295
2296         pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,
2297                               NULL);
2298         if (!pdev)
2299                 goto err;
2300
2301         base = pci_ioremap_bar(pdev, 0);
2302         if (!base)
2303                 goto err_put;
2304
2305         /* Read number of available LBK channels from LBK(0)_CONST register. */
2306         ret = (readq(base + 0x10) >> 32) & 0xffff;
2307         iounmap(base);
2308 err_put:
2309         pci_dev_put(pdev);
2310 err:
2311         return ret;
2312 }
2313
2314 static int rvu_enable_sriov(struct rvu *rvu)
2315 {
2316         struct pci_dev *pdev = rvu->pdev;
2317         int err, chans, vfs;
2318
2319         if (!rvu_afvf_msix_vectors_num_ok(rvu)) {
2320                 dev_warn(&pdev->dev,
2321                          "Skipping SRIOV enablement since not enough IRQs are available\n");
2322                 return 0;
2323         }
2324
2325         chans = lbk_get_num_chans();
2326         if (chans < 0)
2327                 return chans;
2328
2329         vfs = pci_sriov_get_totalvfs(pdev);
2330
2331         /* Limit VFs in case we have more VFs than LBK channels available. */
2332         if (vfs > chans)
2333                 vfs = chans;
2334
2335         /* AF's VFs work in pairs and talk over consecutive loopback channels.
2336          * Thus we want to enable maximum even number of VFs. In case
2337          * odd number of VFs are available then the last VF on the list
2338          * remains disabled.
2339          */
2340         if (vfs & 0x1) {
2341                 dev_warn(&pdev->dev,
2342                          "Number of VFs should be even. Enabling %d out of %d.\n",
2343                          vfs - 1, vfs);
2344                 vfs--;
2345         }
2346
2347         if (!vfs)
2348                 return 0;
2349
2350         /* Save VFs number for reference in VF interrupts handlers.
2351          * Since interrupts might start arriving during SRIOV enablement
2352          * ordinary API cannot be used to get number of enabled VFs.
2353          */
2354         rvu->vfs = vfs;
2355
2356         err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,
2357                             rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);
2358         if (err)
2359                 return err;
2360
2361         rvu_enable_afvf_intr(rvu);
2362         /* Make sure IRQs are enabled before SRIOV. */
2363         mb();
2364
2365         err = pci_enable_sriov(pdev, vfs);
2366         if (err) {
2367                 rvu_disable_afvf_intr(rvu);
2368                 rvu_mbox_destroy(&rvu->afvf_wq_info);
2369                 return err;
2370         }
2371
2372         return 0;
2373 }
2374
2375 static void rvu_disable_sriov(struct rvu *rvu)
2376 {
2377         rvu_disable_afvf_intr(rvu);
2378         rvu_mbox_destroy(&rvu->afvf_wq_info);
2379         pci_disable_sriov(rvu->pdev);
2380 }
2381
2382 static void rvu_update_module_params(struct rvu *rvu)
2383 {
2384         const char *default_pfl_name = "default";
2385
2386         strscpy(rvu->mkex_pfl_name,
2387                 mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);
2388 }
2389
2390 static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2391 {
2392         struct device *dev = &pdev->dev;
2393         struct rvu *rvu;
2394         int    err;
2395
2396         rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);
2397         if (!rvu)
2398                 return -ENOMEM;
2399
2400         rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);
2401         if (!rvu->hw) {
2402                 devm_kfree(dev, rvu);
2403                 return -ENOMEM;
2404         }
2405
2406         pci_set_drvdata(pdev, rvu);
2407         rvu->pdev = pdev;
2408         rvu->dev = &pdev->dev;
2409
2410         err = pci_enable_device(pdev);
2411         if (err) {
2412                 dev_err(dev, "Failed to enable PCI device\n");
2413                 goto err_freemem;
2414         }
2415
2416         err = pci_request_regions(pdev, DRV_NAME);
2417         if (err) {
2418                 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2419                 goto err_disable_device;
2420         }
2421
2422         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
2423         if (err) {
2424                 dev_err(dev, "Unable to set DMA mask\n");
2425                 goto err_release_regions;
2426         }
2427
2428         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
2429         if (err) {
2430                 dev_err(dev, "Unable to set consistent DMA mask\n");
2431                 goto err_release_regions;
2432         }
2433
2434         /* Map Admin function CSRs */
2435         rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);
2436         rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
2437         if (!rvu->afreg_base || !rvu->pfreg_base) {
2438                 dev_err(dev, "Unable to map admin function CSRs, aborting\n");
2439                 err = -ENOMEM;
2440                 goto err_release_regions;
2441         }
2442
2443         /* Store module params in rvu structure */
2444         rvu_update_module_params(rvu);
2445
2446         /* Check which blocks the HW supports */
2447         rvu_check_block_implemented(rvu);
2448
2449         rvu_reset_all_blocks(rvu);
2450
2451         err = rvu_setup_hw_resources(rvu);
2452         if (err)
2453                 goto err_release_regions;
2454
2455         /* Init mailbox btw AF and PFs */
2456         err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,
2457                             rvu->hw->total_pfs, rvu_afpf_mbox_handler,
2458                             rvu_afpf_mbox_up_handler);
2459         if (err)
2460                 goto err_hwsetup;
2461
2462         err = rvu_flr_init(rvu);
2463         if (err)
2464                 goto err_mbox;
2465
2466         err = rvu_register_interrupts(rvu);
2467         if (err)
2468                 goto err_flr;
2469
2470         /* Enable AF's VFs (if any) */
2471         err = rvu_enable_sriov(rvu);
2472         if (err)
2473                 goto err_irq;
2474
2475         /* Initialize debugfs */
2476         rvu_dbg_init(rvu);
2477
2478         return 0;
2479 err_irq:
2480         rvu_unregister_interrupts(rvu);
2481 err_flr:
2482         rvu_flr_wq_destroy(rvu);
2483 err_mbox:
2484         rvu_mbox_destroy(&rvu->afpf_wq_info);
2485 err_hwsetup:
2486         rvu_cgx_exit(rvu);
2487         rvu_reset_all_blocks(rvu);
2488         rvu_free_hw_resources(rvu);
2489 err_release_regions:
2490         pci_release_regions(pdev);
2491 err_disable_device:
2492         pci_disable_device(pdev);
2493 err_freemem:
2494         pci_set_drvdata(pdev, NULL);
2495         devm_kfree(&pdev->dev, rvu->hw);
2496         devm_kfree(dev, rvu);
2497         return err;
2498 }
2499
2500 static void rvu_remove(struct pci_dev *pdev)
2501 {
2502         struct rvu *rvu = pci_get_drvdata(pdev);
2503
2504         rvu_dbg_exit(rvu);
2505         rvu_unregister_interrupts(rvu);
2506         rvu_flr_wq_destroy(rvu);
2507         rvu_cgx_exit(rvu);
2508         rvu_mbox_destroy(&rvu->afpf_wq_info);
2509         rvu_disable_sriov(rvu);
2510         rvu_reset_all_blocks(rvu);
2511         rvu_free_hw_resources(rvu);
2512
2513         pci_release_regions(pdev);
2514         pci_disable_device(pdev);
2515         pci_set_drvdata(pdev, NULL);
2516
2517         devm_kfree(&pdev->dev, rvu->hw);
2518         devm_kfree(&pdev->dev, rvu);
2519 }
2520
2521 static struct pci_driver rvu_driver = {
2522         .name = DRV_NAME,
2523         .id_table = rvu_id_table,
2524         .probe = rvu_probe,
2525         .remove = rvu_remove,
2526 };
2527
2528 static int __init rvu_init_module(void)
2529 {
2530         int err;
2531
2532         pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
2533
2534         err = pci_register_driver(&cgx_driver);
2535         if (err < 0)
2536                 return err;
2537
2538         err =  pci_register_driver(&rvu_driver);
2539         if (err < 0)
2540                 pci_unregister_driver(&cgx_driver);
2541
2542         return err;
2543 }
2544
2545 static void __exit rvu_cleanup_module(void)
2546 {
2547         pci_unregister_driver(&rvu_driver);
2548         pci_unregister_driver(&cgx_driver);
2549 }
2550
2551 module_init(rvu_init_module);
2552 module_exit(rvu_cleanup_module);