drivers/misc/habanalabs/goya/goya.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #include "goyaP.h"
   9 #include "include/hw_ip/mmu/mmu_general.h"
  10 #include "include/hw_ip/mmu/mmu_v1_0.h"
  11 #include "include/goya/asic_reg/goya_masks.h"
  12 #include "include/goya/goya_reg_map.h"
  13
  14 #include <linux/pci.h>
  15 #include <linux/genalloc.h>
  16 #include <linux/hwmon.h>
  17 #include <linux/io-64-nonatomic-lo-hi.h>
  18 #include <linux/iommu.h>
  19 #include <linux/seq_file.h>
  20
  21 /*
  22  * GOYA security scheme:
  23  *
  24  * 1. Host is protected by:
  25  *        - Range registers (When MMU is enabled, DMA RR does NOT protect host)
  26  *        - MMU
  27  *
  28  * 2. DRAM is protected by:
  29  *        - Range registers (protect the first 512MB)
  30  *        - MMU (isolation between users)
  31  *
  32  * 3. Configuration is protected by:
  33  *        - Range registers
  34  *        - Protection bits
  35  *
  36  * When MMU is disabled:
  37  *
  38  * QMAN DMA: PQ, CQ, CP, DMA are secured.
  39  * PQ, CB and the data are on the host.
  40  *
  41  * QMAN TPC/MME:
  42  * PQ, CQ and CP are not secured.
  43  * PQ, CB and the data are on the SRAM/DRAM.
  44  *
  45  * Since QMAN DMA is secured, the driver is parsing the DMA CB:
  46  *     - checks DMA pointer
  47  *     - WREG, MSG_PROT are not allowed.
  48  *     - MSG_LONG/SHORT are allowed.
  49  *
  50  * A read/write transaction by the QMAN to a protected area will succeed if
  51  * and only if the QMAN's CP is secured and MSG_PROT is used
  52  *
  53  *
  54  * When MMU is enabled:
  55  *
  56  * QMAN DMA: PQ, CQ and CP are secured.
  57  * MMU is set to bypass on the Secure props register of the QMAN.
  58  * The reasons we don't enable MMU for PQ, CQ and CP are:
  59  *     - PQ entry is in kernel address space and the driver doesn't map it.
  60  *     - CP writes to MSIX register and to kernel address space (completion
  61  *       queue).
  62  *
  63  * DMA is not secured but because CP is secured, the driver still needs to parse
  64  * the CB, but doesn't need to check the DMA addresses.
  65  *
  66  * For QMAN DMA 0, DMA is also secured because only the driver uses this DMA and
  67  * the driver doesn't map memory in MMU.
  68  *
  69  * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
  70  *
  71  * DMA RR does NOT protect host because DMA is not secured
  72  *
  73  */
  74
  75 #define GOYA_UBOOT_FW_FILE      "habanalabs/goya/goya-u-boot.bin"
  76 #define GOYA_LINUX_FW_FILE      "habanalabs/goya/goya-fit.itb"
  77
  78 #define GOYA_MMU_REGS_NUM               63
  79
  80 #define GOYA_DMA_POOL_BLK_SIZE          0x100           /* 256 bytes */
  81
  82 #define GOYA_RESET_TIMEOUT_MSEC         500             /* 500ms */
  83 #define GOYA_PLDM_RESET_TIMEOUT_MSEC    20000           /* 20s */
  84 #define GOYA_RESET_WAIT_MSEC            1               /* 1ms */
  85 #define GOYA_CPU_RESET_WAIT_MSEC        100             /* 100ms */
  86 #define GOYA_PLDM_RESET_WAIT_MSEC       1000            /* 1s */
  87 #define GOYA_TEST_QUEUE_WAIT_USEC       100000          /* 100ms */
  88 #define GOYA_PLDM_MMU_TIMEOUT_USEC      (MMU_CONFIG_TIMEOUT_USEC * 100)
  89 #define GOYA_PLDM_QMAN0_TIMEOUT_USEC    (HL_DEVICE_TIMEOUT_USEC * 30)
  90
  91 #define GOYA_QMAN0_FENCE_VAL            0xD169B243
  92
  93 #define GOYA_MAX_STRING_LEN             20
  94
  95 #define GOYA_CB_POOL_CB_CNT             512
  96 #define GOYA_CB_POOL_CB_SIZE            0x20000         /* 128KB */
  97
  98 #define IS_QM_IDLE(engine, qm_glbl_sts0) \
  99         (((qm_glbl_sts0) & engine##_QM_IDLE_MASK) == engine##_QM_IDLE_MASK)
 100 #define IS_DMA_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(DMA, qm_glbl_sts0)
 101 #define IS_TPC_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(TPC, qm_glbl_sts0)
 102 #define IS_MME_QM_IDLE(qm_glbl_sts0)    IS_QM_IDLE(MME, qm_glbl_sts0)
 103
 104 #define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) \
 105         (((cmdq_glbl_sts0) & engine##_CMDQ_IDLE_MASK) == \
 106                         engine##_CMDQ_IDLE_MASK)
 107 #define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) \
 108         IS_CMDQ_IDLE(TPC, cmdq_glbl_sts0)
 109 #define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) \
 110         IS_CMDQ_IDLE(MME, cmdq_glbl_sts0)
 111
 112 #define IS_DMA_IDLE(dma_core_sts0) \
 113         !((dma_core_sts0) & DMA_CH_0_STS0_DMA_BUSY_MASK)
 114
 115 #define IS_TPC_IDLE(tpc_cfg_sts) \
 116         (((tpc_cfg_sts) & TPC_CFG_IDLE_MASK) == TPC_CFG_IDLE_MASK)
 117
 118 #define IS_MME_IDLE(mme_arch_sts) \
 119         (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
 120
 121
 122 static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = {
 123                 "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3",
 124                 "goya cq 4", "goya cpu eq"
 125 };
 126
 127 static u16 goya_packet_sizes[MAX_PACKET_ID] = {
 128         [PACKET_WREG_32]        = sizeof(struct packet_wreg32),
 129         [PACKET_WREG_BULK]      = sizeof(struct packet_wreg_bulk),
 130         [PACKET_MSG_LONG]       = sizeof(struct packet_msg_long),
 131         [PACKET_MSG_SHORT]      = sizeof(struct packet_msg_short),
 132         [PACKET_CP_DMA]         = sizeof(struct packet_cp_dma),
 133         [PACKET_MSG_PROT]       = sizeof(struct packet_msg_prot),
 134         [PACKET_FENCE]          = sizeof(struct packet_fence),
 135         [PACKET_LIN_DMA]        = sizeof(struct packet_lin_dma),
 136         [PACKET_NOP]            = sizeof(struct packet_nop),
 137         [PACKET_STOP]           = sizeof(struct packet_stop)
 138 };
 139
 140 static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = {
 141         mmDMA_QM_0_GLBL_NON_SECURE_PROPS,
 142         mmDMA_QM_1_GLBL_NON_SECURE_PROPS,
 143         mmDMA_QM_2_GLBL_NON_SECURE_PROPS,
 144         mmDMA_QM_3_GLBL_NON_SECURE_PROPS,
 145         mmDMA_QM_4_GLBL_NON_SECURE_PROPS,
 146         mmTPC0_QM_GLBL_SECURE_PROPS,
 147         mmTPC0_QM_GLBL_NON_SECURE_PROPS,
 148         mmTPC0_CMDQ_GLBL_SECURE_PROPS,
 149         mmTPC0_CMDQ_GLBL_NON_SECURE_PROPS,
 150         mmTPC0_CFG_ARUSER,
 151         mmTPC0_CFG_AWUSER,
 152         mmTPC1_QM_GLBL_SECURE_PROPS,
 153         mmTPC1_QM_GLBL_NON_SECURE_PROPS,
 154         mmTPC1_CMDQ_GLBL_SECURE_PROPS,
 155         mmTPC1_CMDQ_GLBL_NON_SECURE_PROPS,
 156         mmTPC1_CFG_ARUSER,
 157         mmTPC1_CFG_AWUSER,
 158         mmTPC2_QM_GLBL_SECURE_PROPS,
 159         mmTPC2_QM_GLBL_NON_SECURE_PROPS,
 160         mmTPC2_CMDQ_GLBL_SECURE_PROPS,
 161         mmTPC2_CMDQ_GLBL_NON_SECURE_PROPS,
 162         mmTPC2_CFG_ARUSER,
 163         mmTPC2_CFG_AWUSER,
 164         mmTPC3_QM_GLBL_SECURE_PROPS,
 165         mmTPC3_QM_GLBL_NON_SECURE_PROPS,
 166         mmTPC3_CMDQ_GLBL_SECURE_PROPS,
 167         mmTPC3_CMDQ_GLBL_NON_SECURE_PROPS,
 168         mmTPC3_CFG_ARUSER,
 169         mmTPC3_CFG_AWUSER,
 170         mmTPC4_QM_GLBL_SECURE_PROPS,
 171         mmTPC4_QM_GLBL_NON_SECURE_PROPS,
 172         mmTPC4_CMDQ_GLBL_SECURE_PROPS,
 173         mmTPC4_CMDQ_GLBL_NON_SECURE_PROPS,
 174         mmTPC4_CFG_ARUSER,
 175         mmTPC4_CFG_AWUSER,
 176         mmTPC5_QM_GLBL_SECURE_PROPS,
 177         mmTPC5_QM_GLBL_NON_SECURE_PROPS,
 178         mmTPC5_CMDQ_GLBL_SECURE_PROPS,
 179         mmTPC5_CMDQ_GLBL_NON_SECURE_PROPS,
 180         mmTPC5_CFG_ARUSER,
 181         mmTPC5_CFG_AWUSER,
 182         mmTPC6_QM_GLBL_SECURE_PROPS,
 183         mmTPC6_QM_GLBL_NON_SECURE_PROPS,
 184         mmTPC6_CMDQ_GLBL_SECURE_PROPS,
 185         mmTPC6_CMDQ_GLBL_NON_SECURE_PROPS,
 186         mmTPC6_CFG_ARUSER,
 187         mmTPC6_CFG_AWUSER,
 188         mmTPC7_QM_GLBL_SECURE_PROPS,
 189         mmTPC7_QM_GLBL_NON_SECURE_PROPS,
 190         mmTPC7_CMDQ_GLBL_SECURE_PROPS,
 191         mmTPC7_CMDQ_GLBL_NON_SECURE_PROPS,
 192         mmTPC7_CFG_ARUSER,
 193         mmTPC7_CFG_AWUSER,
 194         mmMME_QM_GLBL_SECURE_PROPS,
 195         mmMME_QM_GLBL_NON_SECURE_PROPS,
 196         mmMME_CMDQ_GLBL_SECURE_PROPS,
 197         mmMME_CMDQ_GLBL_NON_SECURE_PROPS,
 198         mmMME_SBA_CONTROL_DATA,
 199         mmMME_SBB_CONTROL_DATA,
 200         mmMME_SBC_CONTROL_DATA,
 201         mmMME_WBC_CONTROL_DATA,
 202         mmPCIE_WRAP_PSOC_ARUSER,
 203         mmPCIE_WRAP_PSOC_AWUSER
 204 };
 205
 206 static u32 goya_all_events[] = {
 207         GOYA_ASYNC_EVENT_ID_PCIE_IF,
 208         GOYA_ASYNC_EVENT_ID_TPC0_ECC,
 209         GOYA_ASYNC_EVENT_ID_TPC1_ECC,
 210         GOYA_ASYNC_EVENT_ID_TPC2_ECC,
 211         GOYA_ASYNC_EVENT_ID_TPC3_ECC,
 212         GOYA_ASYNC_EVENT_ID_TPC4_ECC,
 213         GOYA_ASYNC_EVENT_ID_TPC5_ECC,
 214         GOYA_ASYNC_EVENT_ID_TPC6_ECC,
 215         GOYA_ASYNC_EVENT_ID_TPC7_ECC,
 216         GOYA_ASYNC_EVENT_ID_MME_ECC,
 217         GOYA_ASYNC_EVENT_ID_MME_ECC_EXT,
 218         GOYA_ASYNC_EVENT_ID_MMU_ECC,
 219         GOYA_ASYNC_EVENT_ID_DMA_MACRO,
 220         GOYA_ASYNC_EVENT_ID_DMA_ECC,
 221         GOYA_ASYNC_EVENT_ID_CPU_IF_ECC,
 222         GOYA_ASYNC_EVENT_ID_PSOC_MEM,
 223         GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT,
 224         GOYA_ASYNC_EVENT_ID_SRAM0,
 225         GOYA_ASYNC_EVENT_ID_SRAM1,
 226         GOYA_ASYNC_EVENT_ID_SRAM2,
 227         GOYA_ASYNC_EVENT_ID_SRAM3,
 228         GOYA_ASYNC_EVENT_ID_SRAM4,
 229         GOYA_ASYNC_EVENT_ID_SRAM5,
 230         GOYA_ASYNC_EVENT_ID_SRAM6,
 231         GOYA_ASYNC_EVENT_ID_SRAM7,
 232         GOYA_ASYNC_EVENT_ID_SRAM8,
 233         GOYA_ASYNC_EVENT_ID_SRAM9,
 234         GOYA_ASYNC_EVENT_ID_SRAM10,
 235         GOYA_ASYNC_EVENT_ID_SRAM11,
 236         GOYA_ASYNC_EVENT_ID_SRAM12,
 237         GOYA_ASYNC_EVENT_ID_SRAM13,
 238         GOYA_ASYNC_EVENT_ID_SRAM14,
 239         GOYA_ASYNC_EVENT_ID_SRAM15,
 240         GOYA_ASYNC_EVENT_ID_SRAM16,
 241         GOYA_ASYNC_EVENT_ID_SRAM17,
 242         GOYA_ASYNC_EVENT_ID_SRAM18,
 243         GOYA_ASYNC_EVENT_ID_SRAM19,
 244         GOYA_ASYNC_EVENT_ID_SRAM20,
 245         GOYA_ASYNC_EVENT_ID_SRAM21,
 246         GOYA_ASYNC_EVENT_ID_SRAM22,
 247         GOYA_ASYNC_EVENT_ID_SRAM23,
 248         GOYA_ASYNC_EVENT_ID_SRAM24,
 249         GOYA_ASYNC_EVENT_ID_SRAM25,
 250         GOYA_ASYNC_EVENT_ID_SRAM26,
 251         GOYA_ASYNC_EVENT_ID_SRAM27,
 252         GOYA_ASYNC_EVENT_ID_SRAM28,
 253         GOYA_ASYNC_EVENT_ID_SRAM29,
 254         GOYA_ASYNC_EVENT_ID_GIC500,
 255         GOYA_ASYNC_EVENT_ID_PLL0,
 256         GOYA_ASYNC_EVENT_ID_PLL1,
 257         GOYA_ASYNC_EVENT_ID_PLL3,
 258         GOYA_ASYNC_EVENT_ID_PLL4,
 259         GOYA_ASYNC_EVENT_ID_PLL5,
 260         GOYA_ASYNC_EVENT_ID_PLL6,
 261         GOYA_ASYNC_EVENT_ID_AXI_ECC,
 262         GOYA_ASYNC_EVENT_ID_L2_RAM_ECC,
 263         GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET,
 264         GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT,
 265         GOYA_ASYNC_EVENT_ID_PCIE_DEC,
 266         GOYA_ASYNC_EVENT_ID_TPC0_DEC,
 267         GOYA_ASYNC_EVENT_ID_TPC1_DEC,
 268         GOYA_ASYNC_EVENT_ID_TPC2_DEC,
 269         GOYA_ASYNC_EVENT_ID_TPC3_DEC,
 270         GOYA_ASYNC_EVENT_ID_TPC4_DEC,
 271         GOYA_ASYNC_EVENT_ID_TPC5_DEC,
 272         GOYA_ASYNC_EVENT_ID_TPC6_DEC,
 273         GOYA_ASYNC_EVENT_ID_TPC7_DEC,
 274         GOYA_ASYNC_EVENT_ID_MME_WACS,
 275         GOYA_ASYNC_EVENT_ID_MME_WACSD,
 276         GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER,
 277         GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC,
 278         GOYA_ASYNC_EVENT_ID_PSOC,
 279         GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR,
 280         GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR,
 281         GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR,
 282         GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR,
 283         GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR,
 284         GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR,
 285         GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR,
 286         GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR,
 287         GOYA_ASYNC_EVENT_ID_TPC0_CMDQ,
 288         GOYA_ASYNC_EVENT_ID_TPC1_CMDQ,
 289         GOYA_ASYNC_EVENT_ID_TPC2_CMDQ,
 290         GOYA_ASYNC_EVENT_ID_TPC3_CMDQ,
 291         GOYA_ASYNC_EVENT_ID_TPC4_CMDQ,
 292         GOYA_ASYNC_EVENT_ID_TPC5_CMDQ,
 293         GOYA_ASYNC_EVENT_ID_TPC6_CMDQ,
 294         GOYA_ASYNC_EVENT_ID_TPC7_CMDQ,
 295         GOYA_ASYNC_EVENT_ID_TPC0_QM,
 296         GOYA_ASYNC_EVENT_ID_TPC1_QM,
 297         GOYA_ASYNC_EVENT_ID_TPC2_QM,
 298         GOYA_ASYNC_EVENT_ID_TPC3_QM,
 299         GOYA_ASYNC_EVENT_ID_TPC4_QM,
 300         GOYA_ASYNC_EVENT_ID_TPC5_QM,
 301         GOYA_ASYNC_EVENT_ID_TPC6_QM,
 302         GOYA_ASYNC_EVENT_ID_TPC7_QM,
 303         GOYA_ASYNC_EVENT_ID_MME_QM,
 304         GOYA_ASYNC_EVENT_ID_MME_CMDQ,
 305         GOYA_ASYNC_EVENT_ID_DMA0_QM,
 306         GOYA_ASYNC_EVENT_ID_DMA1_QM,
 307         GOYA_ASYNC_EVENT_ID_DMA2_QM,
 308         GOYA_ASYNC_EVENT_ID_DMA3_QM,
 309         GOYA_ASYNC_EVENT_ID_DMA4_QM,
 310         GOYA_ASYNC_EVENT_ID_DMA0_CH,
 311         GOYA_ASYNC_EVENT_ID_DMA1_CH,
 312         GOYA_ASYNC_EVENT_ID_DMA2_CH,
 313         GOYA_ASYNC_EVENT_ID_DMA3_CH,
 314         GOYA_ASYNC_EVENT_ID_DMA4_CH,
 315         GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU,
 316         GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU,
 317         GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU,
 318         GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU,
 319         GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU,
 320         GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU,
 321         GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU,
 322         GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU,
 323         GOYA_ASYNC_EVENT_ID_DMA_BM_CH0,
 324         GOYA_ASYNC_EVENT_ID_DMA_BM_CH1,
 325         GOYA_ASYNC_EVENT_ID_DMA_BM_CH2,
 326         GOYA_ASYNC_EVENT_ID_DMA_BM_CH3,
 327         GOYA_ASYNC_EVENT_ID_DMA_BM_CH4
 328 };
 329
 330 static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
 331 static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
 332 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
 333 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
 334
 335 void goya_get_fixed_properties(struct hl_device *hdev)
 336 {
 337         struct asic_fixed_properties *prop = &hdev->asic_prop;
 338         int i;
 339
 340         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
 341                 prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
 342                 prop->hw_queues_props[i].driver_only = 0;
 343                 prop->hw_queues_props[i].requires_kernel_cb = 1;
 344         }
 345
 346         for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
 347                 prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
 348                 prop->hw_queues_props[i].driver_only = 1;
 349                 prop->hw_queues_props[i].requires_kernel_cb = 0;
 350         }
 351
 352         for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
 353                         NUMBER_OF_INT_HW_QUEUES; i++) {
 354                 prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
 355                 prop->hw_queues_props[i].driver_only = 0;
 356                 prop->hw_queues_props[i].requires_kernel_cb = 0;
 357         }
 358
 359         for (; i < HL_MAX_QUEUES; i++)
 360                 prop->hw_queues_props[i].type = QUEUE_TYPE_NA;
 361
 362         prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
 363
 364         prop->dram_base_address = DRAM_PHYS_BASE;
 365         prop->dram_size = DRAM_PHYS_DEFAULT_SIZE;
 366         prop->dram_end_address = prop->dram_base_address + prop->dram_size;
 367         prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
 368
 369         prop->sram_base_address = SRAM_BASE_ADDR;
 370         prop->sram_size = SRAM_SIZE;
 371         prop->sram_end_address = prop->sram_base_address + prop->sram_size;
 372         prop->sram_user_base_address = prop->sram_base_address +
 373                                                 SRAM_USER_BASE_OFFSET;
 374
 375         prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
 376         prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR;
 377         if (hdev->pldm)
 378                 prop->mmu_pgt_size = 0x800000; /* 8MB */
 379         else
 380                 prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
 381         prop->mmu_pte_size = HL_PTE_SIZE;
 382         prop->mmu_hop_table_size = HOP_TABLE_SIZE;
 383         prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
 384         prop->dram_page_size = PAGE_SIZE_2MB;
 385
 386         prop->dmmu.hop0_shift = HOP0_SHIFT;
 387         prop->dmmu.hop1_shift = HOP1_SHIFT;
 388         prop->dmmu.hop2_shift = HOP2_SHIFT;
 389         prop->dmmu.hop3_shift = HOP3_SHIFT;
 390         prop->dmmu.hop4_shift = HOP4_SHIFT;
 391         prop->dmmu.hop0_mask = HOP0_MASK;
 392         prop->dmmu.hop1_mask = HOP1_MASK;
 393         prop->dmmu.hop2_mask = HOP2_MASK;
 394         prop->dmmu.hop3_mask = HOP3_MASK;
 395         prop->dmmu.hop4_mask = HOP4_MASK;
 396         prop->dmmu.huge_page_size = PAGE_SIZE_2MB;
 397
 398         /* No difference between PMMU and DMMU except of page size */
 399         memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu));
 400         prop->dmmu.page_size = PAGE_SIZE_2MB;
 401         prop->pmmu.page_size = PAGE_SIZE_4KB;
 402
 403         prop->va_space_host_start_address = VA_HOST_SPACE_START;
 404         prop->va_space_host_end_address = VA_HOST_SPACE_END;
 405         prop->va_space_dram_start_address = VA_DDR_SPACE_START;
 406         prop->va_space_dram_end_address = VA_DDR_SPACE_END;
 407         prop->dram_size_for_default_page_mapping =
 408                         prop->va_space_dram_end_address;
 409         prop->cfg_size = CFG_SIZE;
 410         prop->max_asid = MAX_ASID;
 411         prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE;
 412         prop->high_pll = PLL_HIGH_DEFAULT;
 413         prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
 414         prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
 415         prop->max_power_default = MAX_POWER_DEFAULT;
 416         prop->tpc_enabled_mask = TPC_ENABLED_MASK;
 417         prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
 418         prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
 419
 420         strncpy(prop->armcp_info.card_name, GOYA_DEFAULT_CARD_NAME,
 421                 CARD_NAME_MAX_LEN);
 422 }
 423
 424 /*
 425  * goya_pci_bars_map - Map PCI BARS of Goya device
 426  *
 427  * @hdev: pointer to hl_device structure
 428  *
 429  * Request PCI regions and map them to kernel virtual addresses.
 430  * Returns 0 on success
 431  *
 432  */
 433 static int goya_pci_bars_map(struct hl_device *hdev)
 434 {
 435         static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"};
 436         bool is_wc[3] = {false, false, true};
 437         int rc;
 438
 439         rc = hl_pci_bars_map(hdev, name, is_wc);
 440         if (rc)
 441                 return rc;
 442
 443         hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
 444                         (CFG_BASE - SRAM_BASE_ADDR);
 445
 446         return 0;
 447 }
 448
 449 static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
 450 {
 451         struct goya_device *goya = hdev->asic_specific;
 452         u64 old_addr = addr;
 453         int rc;
 454
 455         if ((goya) && (goya->ddr_bar_cur_addr == addr))
 456                 return old_addr;
 457
 458         /* Inbound Region 1 - Bar 4 - Point to DDR */
 459         rc = hl_pci_set_dram_bar_base(hdev, 1, 4, addr);
 460         if (rc)
 461                 return U64_MAX;
 462
 463         if (goya) {
 464                 old_addr = goya->ddr_bar_cur_addr;
 465                 goya->ddr_bar_cur_addr = addr;
 466         }
 467
 468         return old_addr;
 469 }
 470
 471 /*
 472  * goya_init_iatu - Initialize the iATU unit inside the PCI controller
 473  *
 474  * @hdev: pointer to hl_device structure
 475  *
 476  * This is needed in case the firmware doesn't initialize the iATU
 477  *
 478  */
 479 static int goya_init_iatu(struct hl_device *hdev)
 480 {
 481         return hl_pci_init_iatu(hdev, SRAM_BASE_ADDR, DRAM_PHYS_BASE,
 482                                 HOST_PHYS_BASE, HOST_PHYS_SIZE);
 483 }
 484
 485 /*
 486  * goya_early_init - GOYA early initialization code
 487  *
 488  * @hdev: pointer to hl_device structure
 489  *
 490  * Verify PCI bars
 491  * Set DMA masks
 492  * PCI controller initialization
 493  * Map PCI bars
 494  *
 495  */
 496 static int goya_early_init(struct hl_device *hdev)
 497 {
 498         struct asic_fixed_properties *prop = &hdev->asic_prop;
 499         struct pci_dev *pdev = hdev->pdev;
 500         u32 val;
 501         int rc;
 502
 503         goya_get_fixed_properties(hdev);
 504
 505         /* Check BAR sizes */
 506         if (pci_resource_len(pdev, SRAM_CFG_BAR_ID) != CFG_BAR_SIZE) {
 507                 dev_err(hdev->dev,
 508                         "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 509                         SRAM_CFG_BAR_ID,
 510                         (unsigned long long) pci_resource_len(pdev,
 511                                                         SRAM_CFG_BAR_ID),
 512                         CFG_BAR_SIZE);
 513                 return -ENODEV;
 514         }
 515
 516         if (pci_resource_len(pdev, MSIX_BAR_ID) != MSIX_BAR_SIZE) {
 517                 dev_err(hdev->dev,
 518                         "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
 519                         MSIX_BAR_ID,
 520                         (unsigned long long) pci_resource_len(pdev,
 521                                                                 MSIX_BAR_ID),
 522                         MSIX_BAR_SIZE);
 523                 return -ENODEV;
 524         }
 525
 526         prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
 527
 528         rc = hl_pci_init(hdev, 48);
 529         if (rc)
 530                 return rc;
 531
 532         if (!hdev->pldm) {
 533                 val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
 534                 if (val & PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_SRIOV_EN_MASK)
 535                         dev_warn(hdev->dev,
 536                                 "PCI strap is not configured correctly, PCI bus errors may occur\n");
 537         }
 538
 539         return 0;
 540 }
 541
 542 /*
 543  * goya_early_fini - GOYA early finalization code
 544  *
 545  * @hdev: pointer to hl_device structure
 546  *
 547  * Unmap PCI bars
 548  *
 549  */
 550 static int goya_early_fini(struct hl_device *hdev)
 551 {
 552         hl_pci_fini(hdev);
 553
 554         return 0;
 555 }
 556
 557 static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
 558 {
 559         /* mask to zero the MMBP and ASID bits */
 560         WREG32_AND(reg, ~0x7FF);
 561         WREG32_OR(reg, asid);
 562 }
 563
 564 static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
 565 {
 566         struct goya_device *goya = hdev->asic_specific;
 567
 568         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
 569                 return;
 570
 571         if (secure)
 572                 WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
 573         else
 574                 WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
 575
 576         RREG32(mmDMA_QM_0_GLBL_PROT);
 577 }
 578
 579 /*
 580  * goya_fetch_psoc_frequency - Fetch PSOC frequency values
 581  *
 582  * @hdev: pointer to hl_device structure
 583  *
 584  */
 585 static void goya_fetch_psoc_frequency(struct hl_device *hdev)
 586 {
 587         struct asic_fixed_properties *prop = &hdev->asic_prop;
 588
 589         prop->psoc_pci_pll_nr = RREG32(mmPSOC_PCI_PLL_NR);
 590         prop->psoc_pci_pll_nf = RREG32(mmPSOC_PCI_PLL_NF);
 591         prop->psoc_pci_pll_od = RREG32(mmPSOC_PCI_PLL_OD);
 592         prop->psoc_pci_pll_div_factor = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
 593 }
 594
 595 int goya_late_init(struct hl_device *hdev)
 596 {
 597         struct asic_fixed_properties *prop = &hdev->asic_prop;
 598         int rc;
 599
 600         goya_fetch_psoc_frequency(hdev);
 601
 602         rc = goya_mmu_clear_pgt_range(hdev);
 603         if (rc) {
 604                 dev_err(hdev->dev,
 605                         "Failed to clear MMU page tables range %d\n", rc);
 606                 return rc;
 607         }
 608
 609         rc = goya_mmu_set_dram_default_page(hdev);
 610         if (rc) {
 611                 dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc);
 612                 return rc;
 613         }
 614
 615         rc = goya_mmu_add_mappings_for_device_cpu(hdev);
 616         if (rc)
 617                 return rc;
 618
 619         rc = goya_init_cpu_queues(hdev);
 620         if (rc)
 621                 return rc;
 622
 623         rc = goya_test_cpu_queue(hdev);
 624         if (rc)
 625                 return rc;
 626
 627         rc = goya_armcp_info_get(hdev);
 628         if (rc) {
 629                 dev_err(hdev->dev, "Failed to get armcp info %d\n", rc);
 630                 return rc;
 631         }
 632
 633         /* Now that we have the DRAM size in ASIC prop, we need to check
 634          * its size and configure the DMA_IF DDR wrap protection (which is in
 635          * the MMU block) accordingly. The value is the log2 of the DRAM size
 636          */
 637         WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size));
 638
 639         rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_ENABLE_PCI_ACCESS);
 640         if (rc) {
 641                 dev_err(hdev->dev,
 642                         "Failed to enable PCI access from CPU %d\n", rc);
 643                 return rc;
 644         }
 645
 646         WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
 647                         GOYA_ASYNC_EVENT_ID_INTS_REGISTER);
 648
 649         return 0;
 650 }
 651
 652 /*
 653  * goya_late_fini - GOYA late tear-down code
 654  *
 655  * @hdev: pointer to hl_device structure
 656  *
 657  * Free sensors allocated structures
 658  */
 659 void goya_late_fini(struct hl_device *hdev)
 660 {
 661         const struct hwmon_channel_info **channel_info_arr;
 662         int i = 0;
 663
 664         if (!hdev->hl_chip_info->info)
 665                 return;
 666
 667         channel_info_arr = hdev->hl_chip_info->info;
 668
 669         while (channel_info_arr[i]) {
 670                 kfree(channel_info_arr[i]->config);
 671                 kfree(channel_info_arr[i]);
 672                 i++;
 673         }
 674
 675         kfree(channel_info_arr);
 676
 677         hdev->hl_chip_info->info = NULL;
 678 }
 679
 680 /*
 681  * goya_sw_init - Goya software initialization code
 682  *
 683  * @hdev: pointer to hl_device structure
 684  *
 685  */
 686 static int goya_sw_init(struct hl_device *hdev)
 687 {
 688         struct goya_device *goya;
 689         int rc;
 690
 691         /* Allocate device structure */
 692         goya = kzalloc(sizeof(*goya), GFP_KERNEL);
 693         if (!goya)
 694                 return -ENOMEM;
 695
 696         /* according to goya_init_iatu */
 697         goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
 698
 699         goya->mme_clk = GOYA_PLL_FREQ_LOW;
 700         goya->tpc_clk = GOYA_PLL_FREQ_LOW;
 701         goya->ic_clk = GOYA_PLL_FREQ_LOW;
 702
 703         hdev->asic_specific = goya;
 704
 705         /* Create DMA pool for small allocations */
 706         hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
 707                         &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
 708         if (!hdev->dma_pool) {
 709                 dev_err(hdev->dev, "failed to create DMA pool\n");
 710                 rc = -ENOMEM;
 711                 goto free_goya_device;
 712         }
 713
 714         hdev->cpu_accessible_dma_mem =
 715                         hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
 716                                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 717                                         &hdev->cpu_accessible_dma_address,
 718                                         GFP_KERNEL | __GFP_ZERO);
 719
 720         if (!hdev->cpu_accessible_dma_mem) {
 721                 rc = -ENOMEM;
 722                 goto free_dma_pool;
 723         }
 724
 725         dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n",
 726                 &hdev->cpu_accessible_dma_address);
 727
 728         hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
 729         if (!hdev->cpu_accessible_dma_pool) {
 730                 dev_err(hdev->dev,
 731                         "Failed to create CPU accessible DMA pool\n");
 732                 rc = -ENOMEM;
 733                 goto free_cpu_dma_mem;
 734         }
 735
 736         rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
 737                                 (uintptr_t) hdev->cpu_accessible_dma_mem,
 738                                 HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
 739         if (rc) {
 740                 dev_err(hdev->dev,
 741                         "Failed to add memory to CPU accessible DMA pool\n");
 742                 rc = -EFAULT;
 743                 goto free_cpu_accessible_dma_pool;
 744         }
 745
 746         spin_lock_init(&goya->hw_queues_lock);
 747
 748         return 0;
 749
 750 free_cpu_accessible_dma_pool:
 751         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 752 free_cpu_dma_mem:
 753         hdev->asic_funcs->asic_dma_free_coherent(hdev,
 754                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 755                         hdev->cpu_accessible_dma_mem,
 756                         hdev->cpu_accessible_dma_address);
 757 free_dma_pool:
 758         dma_pool_destroy(hdev->dma_pool);
 759 free_goya_device:
 760         kfree(goya);
 761
 762         return rc;
 763 }
 764
 765 /*
 766  * goya_sw_fini - Goya software tear-down code
 767  *
 768  * @hdev: pointer to hl_device structure
 769  *
 770  */
 771 static int goya_sw_fini(struct hl_device *hdev)
 772 {
 773         struct goya_device *goya = hdev->asic_specific;
 774
 775         gen_pool_destroy(hdev->cpu_accessible_dma_pool);
 776
 777         hdev->asic_funcs->asic_dma_free_coherent(hdev,
 778                         HL_CPU_ACCESSIBLE_MEM_SIZE,
 779                         hdev->cpu_accessible_dma_mem,
 780                         hdev->cpu_accessible_dma_address);
 781
 782         dma_pool_destroy(hdev->dma_pool);
 783
 784         kfree(goya);
 785
 786         return 0;
 787 }
 788
 789 static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
 790                 dma_addr_t bus_address)
 791 {
 792         struct goya_device *goya = hdev->asic_specific;
 793         u32 mtr_base_lo, mtr_base_hi;
 794         u32 so_base_lo, so_base_hi;
 795         u32 gic_base_lo, gic_base_hi;
 796         u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
 797
 798         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
 799         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
 800         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
 801         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
 802
 803         gic_base_lo =
 804                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 805         gic_base_hi =
 806                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 807
 808         WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
 809         WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
 810
 811         WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
 812         WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
 813         WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
 814
 815         WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
 816         WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
 817         WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
 818         WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
 819         WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
 820         WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
 821         WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
 822                         GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
 823
 824         /* PQ has buffer of 2 cache lines, while CQ has 8 lines */
 825         WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
 826         WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
 827
 828         if (goya->hw_cap_initialized & HW_CAP_MMU)
 829                 WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_PARTLY_TRUSTED);
 830         else
 831                 WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
 832
 833         WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, QMAN_DMA_ERR_MSG_EN);
 834         WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
 835 }
 836
 837 static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
 838 {
 839         u32 gic_base_lo, gic_base_hi;
 840         u64 sob_addr;
 841         u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
 842
 843         gic_base_lo =
 844                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 845         gic_base_hi =
 846                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
 847
 848         WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
 849         WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
 850         WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
 851                         GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
 852
 853         if (dma_id)
 854                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
 855                                 (dma_id - 1) * 4;
 856         else
 857                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
 858
 859         WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr));
 860         WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
 861 }
 862
 863 /*
 864  * goya_init_dma_qmans - Initialize QMAN DMA registers
 865  *
 866  * @hdev: pointer to hl_device structure
 867  *
 868  * Initialize the H/W registers of the QMAN DMA channels
 869  *
 870  */
 871 void goya_init_dma_qmans(struct hl_device *hdev)
 872 {
 873         struct goya_device *goya = hdev->asic_specific;
 874         struct hl_hw_queue *q;
 875         int i;
 876
 877         if (goya->hw_cap_initialized & HW_CAP_DMA)
 878                 return;
 879
 880         q = &hdev->kernel_queues[0];
 881
 882         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
 883                 goya_init_dma_qman(hdev, i, q->bus_address);
 884                 goya_init_dma_ch(hdev, i);
 885         }
 886
 887         goya->hw_cap_initialized |= HW_CAP_DMA;
 888 }
 889
 890 /*
 891  * goya_disable_external_queues - Disable external queues
 892  *
 893  * @hdev: pointer to hl_device structure
 894  *
 895  */
 896 static void goya_disable_external_queues(struct hl_device *hdev)
 897 {
 898         struct goya_device *goya = hdev->asic_specific;
 899
 900         if (!(goya->hw_cap_initialized & HW_CAP_DMA))
 901                 return;
 902
 903         WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
 904         WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
 905         WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
 906         WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
 907         WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
 908 }
 909
 910 static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
 911                                 u32 cp_sts_reg, u32 glbl_sts0_reg)
 912 {
 913         int rc;
 914         u32 status;
 915
 916         /* use the values of TPC0 as they are all the same*/
 917
 918         WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
 919
 920         status = RREG32(cp_sts_reg);
 921         if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
 922                 rc = hl_poll_timeout(
 923                         hdev,
 924                         cp_sts_reg,
 925                         status,
 926                         !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
 927                         1000,
 928                         QMAN_FENCE_TIMEOUT_USEC);
 929
 930                 /* if QMAN is stuck in fence no need to check for stop */
 931                 if (rc)
 932                         return 0;
 933         }
 934
 935         rc = hl_poll_timeout(
 936                 hdev,
 937                 glbl_sts0_reg,
 938                 status,
 939                 (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
 940                 1000,
 941                 QMAN_STOP_TIMEOUT_USEC);
 942
 943         if (rc) {
 944                 dev_err(hdev->dev,
 945                         "Timeout while waiting for QMAN to stop\n");
 946                 return -EINVAL;
 947         }
 948
 949         return 0;
 950 }
 951
 952 /*
 953  * goya_stop_external_queues - Stop external queues
 954  *
 955  * @hdev: pointer to hl_device structure
 956  *
 957  * Returns 0 on success
 958  *
 959  */
 960 static int goya_stop_external_queues(struct hl_device *hdev)
 961 {
 962         int rc, retval = 0;
 963
 964         struct goya_device *goya = hdev->asic_specific;
 965
 966         if (!(goya->hw_cap_initialized & HW_CAP_DMA))
 967                 return retval;
 968
 969         rc = goya_stop_queue(hdev,
 970                         mmDMA_QM_0_GLBL_CFG1,
 971                         mmDMA_QM_0_CP_STS,
 972                         mmDMA_QM_0_GLBL_STS0);
 973
 974         if (rc) {
 975                 dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
 976                 retval = -EIO;
 977         }
 978
 979         rc = goya_stop_queue(hdev,
 980                         mmDMA_QM_1_GLBL_CFG1,
 981                         mmDMA_QM_1_CP_STS,
 982                         mmDMA_QM_1_GLBL_STS0);
 983
 984         if (rc) {
 985                 dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
 986                 retval = -EIO;
 987         }
 988
 989         rc = goya_stop_queue(hdev,
 990                         mmDMA_QM_2_GLBL_CFG1,
 991                         mmDMA_QM_2_CP_STS,
 992                         mmDMA_QM_2_GLBL_STS0);
 993
 994         if (rc) {
 995                 dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
 996                 retval = -EIO;
 997         }
 998
 999         rc = goya_stop_queue(hdev,
1000                         mmDMA_QM_3_GLBL_CFG1,
1001                         mmDMA_QM_3_CP_STS,
1002                         mmDMA_QM_3_GLBL_STS0);
1003
1004         if (rc) {
1005                 dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
1006                 retval = -EIO;
1007         }
1008
1009         rc = goya_stop_queue(hdev,
1010                         mmDMA_QM_4_GLBL_CFG1,
1011                         mmDMA_QM_4_CP_STS,
1012                         mmDMA_QM_4_GLBL_STS0);
1013
1014         if (rc) {
1015                 dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
1016                 retval = -EIO;
1017         }
1018
1019         return retval;
1020 }
1021
1022 /*
1023  * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
1024  *
1025  * @hdev: pointer to hl_device structure
1026  *
1027  * Returns 0 on success
1028  *
1029  */
1030 int goya_init_cpu_queues(struct hl_device *hdev)
1031 {
1032         struct goya_device *goya = hdev->asic_specific;
1033         struct hl_eq *eq;
1034         u32 status;
1035         struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
1036         int err;
1037
1038         if (!hdev->cpu_queues_enable)
1039                 return 0;
1040
1041         if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
1042                 return 0;
1043
1044         eq = &hdev->event_queue;
1045
1046         WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
1047         WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
1048
1049         WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
1050         WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
1051
1052         WREG32(mmCPU_CQ_BASE_ADDR_LOW,
1053                         lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
1054         WREG32(mmCPU_CQ_BASE_ADDR_HIGH,
1055                         upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
1056
1057         WREG32(mmCPU_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
1058         WREG32(mmCPU_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
1059         WREG32(mmCPU_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
1060
1061         /* Used for EQ CI */
1062         WREG32(mmCPU_EQ_CI, 0);
1063
1064         WREG32(mmCPU_IF_PF_PQ_PI, 0);
1065
1066         WREG32(mmCPU_PQ_INIT_STATUS, PQ_INIT_STATUS_READY_FOR_CP);
1067
1068         WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
1069                         GOYA_ASYNC_EVENT_ID_PI_UPDATE);
1070
1071         err = hl_poll_timeout(
1072                 hdev,
1073                 mmCPU_PQ_INIT_STATUS,
1074                 status,
1075                 (status == PQ_INIT_STATUS_READY_FOR_HOST),
1076                 1000,
1077                 GOYA_CPU_TIMEOUT_USEC);
1078
1079         if (err) {
1080                 dev_err(hdev->dev,
1081                         "Failed to setup communication with device CPU\n");
1082                 return -EIO;
1083         }
1084
1085         goya->hw_cap_initialized |= HW_CAP_CPU_Q;
1086         return 0;
1087 }
1088
1089 static void goya_set_pll_refclk(struct hl_device *hdev)
1090 {
1091         WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
1092         WREG32(mmCPU_PLL_DIV_SEL_1, 0x0);
1093         WREG32(mmCPU_PLL_DIV_SEL_2, 0x0);
1094         WREG32(mmCPU_PLL_DIV_SEL_3, 0x0);
1095
1096         WREG32(mmIC_PLL_DIV_SEL_0, 0x0);
1097         WREG32(mmIC_PLL_DIV_SEL_1, 0x0);
1098         WREG32(mmIC_PLL_DIV_SEL_2, 0x0);
1099         WREG32(mmIC_PLL_DIV_SEL_3, 0x0);
1100
1101         WREG32(mmMC_PLL_DIV_SEL_0, 0x0);
1102         WREG32(mmMC_PLL_DIV_SEL_1, 0x0);
1103         WREG32(mmMC_PLL_DIV_SEL_2, 0x0);
1104         WREG32(mmMC_PLL_DIV_SEL_3, 0x0);
1105
1106         WREG32(mmPSOC_MME_PLL_DIV_SEL_0, 0x0);
1107         WREG32(mmPSOC_MME_PLL_DIV_SEL_1, 0x0);
1108         WREG32(mmPSOC_MME_PLL_DIV_SEL_2, 0x0);
1109         WREG32(mmPSOC_MME_PLL_DIV_SEL_3, 0x0);
1110
1111         WREG32(mmPSOC_PCI_PLL_DIV_SEL_0, 0x0);
1112         WREG32(mmPSOC_PCI_PLL_DIV_SEL_1, 0x0);
1113         WREG32(mmPSOC_PCI_PLL_DIV_SEL_2, 0x0);
1114         WREG32(mmPSOC_PCI_PLL_DIV_SEL_3, 0x0);
1115
1116         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_0, 0x0);
1117         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_1, 0x0);
1118         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_2, 0x0);
1119         WREG32(mmPSOC_EMMC_PLL_DIV_SEL_3, 0x0);
1120
1121         WREG32(mmTPC_PLL_DIV_SEL_0, 0x0);
1122         WREG32(mmTPC_PLL_DIV_SEL_1, 0x0);
1123         WREG32(mmTPC_PLL_DIV_SEL_2, 0x0);
1124         WREG32(mmTPC_PLL_DIV_SEL_3, 0x0);
1125 }
1126
1127 static void goya_disable_clk_rlx(struct hl_device *hdev)
1128 {
1129         WREG32(mmPSOC_MME_PLL_CLK_RLX_0, 0x100010);
1130         WREG32(mmIC_PLL_CLK_RLX_0, 0x100010);
1131 }
1132
1133 static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id)
1134 {
1135         u64 tpc_eml_address;
1136         u32 val, tpc_offset, tpc_eml_offset, tpc_slm_offset;
1137         int err, slm_index;
1138
1139         tpc_offset = tpc_id * 0x40000;
1140         tpc_eml_offset = tpc_id * 0x200000;
1141         tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE);
1142         tpc_slm_offset = tpc_eml_address + 0x100000;
1143
1144         /*
1145          * Workaround for Bug H2 #2443 :
1146          * "TPC SB is not initialized on chip reset"
1147          */
1148
1149         val = RREG32(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset);
1150         if (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK)
1151                 dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n",
1152                         tpc_id);
1153
1154         WREG32(mmTPC0_CFG_FUNC_MBIST_PAT + tpc_offset, val & 0xFFFFF000);
1155
1156         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_0 + tpc_offset, 0x37FF);
1157         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_1 + tpc_offset, 0x303F);
1158         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_2 + tpc_offset, 0x71FF);
1159         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_3 + tpc_offset, 0x71FF);
1160         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_4 + tpc_offset, 0x70FF);
1161         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_5 + tpc_offset, 0x70FF);
1162         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_6 + tpc_offset, 0x70FF);
1163         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_7 + tpc_offset, 0x70FF);
1164         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_8 + tpc_offset, 0x70FF);
1165         WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_9 + tpc_offset, 0x70FF);
1166
1167         WREG32_OR(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1168                 1 << TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT);
1169
1170         err = hl_poll_timeout(
1171                 hdev,
1172                 mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
1173                 val,
1174                 (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK),
1175                 1000,
1176                 HL_DEVICE_TIMEOUT_USEC);
1177
1178         if (err)
1179                 dev_err(hdev->dev,
1180                         "Timeout while waiting for TPC%d MBIST DONE\n", tpc_id);
1181
1182         WREG32_OR(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1183                 1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT);
1184
1185         msleep(GOYA_RESET_WAIT_MSEC);
1186
1187         WREG32_AND(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
1188                 ~(1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT));
1189
1190         msleep(GOYA_RESET_WAIT_MSEC);
1191
1192         for (slm_index = 0 ; slm_index < 256 ; slm_index++)
1193                 WREG32(tpc_slm_offset + (slm_index << 2), 0);
1194
1195         val = RREG32(tpc_slm_offset);
1196 }
1197
1198 static void goya_tpc_mbist_workaround(struct hl_device *hdev)
1199 {
1200         struct goya_device *goya = hdev->asic_specific;
1201         int i;
1202
1203         if (hdev->pldm)
1204                 return;
1205
1206         if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST)
1207                 return;
1208
1209         /* Workaround for H2 #2443 */
1210
1211         for (i = 0 ; i < TPC_MAX_NUM ; i++)
1212                 _goya_tpc_mbist_workaround(hdev, i);
1213
1214         goya->hw_cap_initialized |= HW_CAP_TPC_MBIST;
1215 }
1216
1217 /*
1218  * goya_init_golden_registers - Initialize golden registers
1219  *
1220  * @hdev: pointer to hl_device structure
1221  *
1222  * Initialize the H/W registers of the device
1223  *
1224  */
1225 static void goya_init_golden_registers(struct hl_device *hdev)
1226 {
1227         struct goya_device *goya = hdev->asic_specific;
1228         u32 polynom[10], tpc_intr_mask, offset;
1229         int i;
1230
1231         if (goya->hw_cap_initialized & HW_CAP_GOLDEN)
1232                 return;
1233
1234         polynom[0] = 0x00020080;
1235         polynom[1] = 0x00401000;
1236         polynom[2] = 0x00200800;
1237         polynom[3] = 0x00002000;
1238         polynom[4] = 0x00080200;
1239         polynom[5] = 0x00040100;
1240         polynom[6] = 0x00100400;
1241         polynom[7] = 0x00004000;
1242         polynom[8] = 0x00010000;
1243         polynom[9] = 0x00008000;
1244
1245         /* Mask all arithmetic interrupts from TPC */
1246         tpc_intr_mask = 0x7FFF;
1247
1248         for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x20000) {
1249                 WREG32(mmSRAM_Y0_X0_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1250                 WREG32(mmSRAM_Y0_X1_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1251                 WREG32(mmSRAM_Y0_X2_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1252                 WREG32(mmSRAM_Y0_X3_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1253                 WREG32(mmSRAM_Y0_X4_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
1254
1255                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_L_ARB + offset, 0x204);
1256                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_L_ARB + offset, 0x204);
1257                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_L_ARB + offset, 0x204);
1258                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_L_ARB + offset, 0x204);
1259                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_L_ARB + offset, 0x204);
1260
1261
1262                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_E_ARB + offset, 0x206);
1263                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_E_ARB + offset, 0x206);
1264                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_E_ARB + offset, 0x206);
1265                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_E_ARB + offset, 0x207);
1266                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_E_ARB + offset, 0x207);
1267
1268                 WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_W_ARB + offset, 0x207);
1269                 WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_W_ARB + offset, 0x207);
1270                 WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_W_ARB + offset, 0x206);
1271                 WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_W_ARB + offset, 0x206);
1272                 WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_W_ARB + offset, 0x206);
1273
1274                 WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_E_ARB + offset, 0x101);
1275                 WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_E_ARB + offset, 0x102);
1276                 WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_E_ARB + offset, 0x103);
1277                 WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_E_ARB + offset, 0x104);
1278                 WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_E_ARB + offset, 0x105);
1279
1280                 WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_W_ARB + offset, 0x105);
1281                 WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_W_ARB + offset, 0x104);
1282                 WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_W_ARB + offset, 0x103);
1283                 WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_W_ARB + offset, 0x102);
1284                 WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_W_ARB + offset, 0x101);
1285         }
1286
1287         WREG32(mmMME_STORE_MAX_CREDIT, 0x21);
1288         WREG32(mmMME_AGU, 0x0f0f0f10);
1289         WREG32(mmMME_SEI_MASK, ~0x0);
1290
1291         WREG32(mmMME6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1292         WREG32(mmMME5_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1293         WREG32(mmMME4_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1294         WREG32(mmMME3_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1295         WREG32(mmMME2_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1296         WREG32(mmMME1_RTR_HBW_RD_RQ_N_ARB, 0x07010701);
1297         WREG32(mmMME6_RTR_HBW_RD_RQ_S_ARB, 0x04010401);
1298         WREG32(mmMME5_RTR_HBW_RD_RQ_S_ARB, 0x04050401);
1299         WREG32(mmMME4_RTR_HBW_RD_RQ_S_ARB, 0x03070301);
1300         WREG32(mmMME3_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1301         WREG32(mmMME2_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1302         WREG32(mmMME1_RTR_HBW_RD_RQ_S_ARB, 0x01050105);
1303         WREG32(mmMME6_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1304         WREG32(mmMME5_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
1305         WREG32(mmMME4_RTR_HBW_RD_RQ_W_ARB, 0x01040301);
1306         WREG32(mmMME3_RTR_HBW_RD_RQ_W_ARB, 0x01030401);
1307         WREG32(mmMME2_RTR_HBW_RD_RQ_W_ARB, 0x01040101);
1308         WREG32(mmMME1_RTR_HBW_RD_RQ_W_ARB, 0x01050101);
1309         WREG32(mmMME6_RTR_HBW_WR_RQ_N_ARB, 0x02020202);
1310         WREG32(mmMME5_RTR_HBW_WR_RQ_N_ARB, 0x01070101);
1311         WREG32(mmMME4_RTR_HBW_WR_RQ_N_ARB, 0x02020201);
1312         WREG32(mmMME3_RTR_HBW_WR_RQ_N_ARB, 0x07020701);
1313         WREG32(mmMME2_RTR_HBW_WR_RQ_N_ARB, 0x01020101);
1314         WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1315         WREG32(mmMME6_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1316         WREG32(mmMME5_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1317         WREG32(mmMME4_RTR_HBW_WR_RQ_S_ARB, 0x07020701);
1318         WREG32(mmMME3_RTR_HBW_WR_RQ_S_ARB, 0x02020201);
1319         WREG32(mmMME2_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
1320         WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01020102);
1321         WREG32(mmMME6_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1322         WREG32(mmMME5_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
1323         WREG32(mmMME4_RTR_HBW_WR_RQ_W_ARB, 0x07020707);
1324         WREG32(mmMME3_RTR_HBW_WR_RQ_W_ARB, 0x01020201);
1325         WREG32(mmMME2_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1326         WREG32(mmMME1_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
1327         WREG32(mmMME6_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1328         WREG32(mmMME5_RTR_HBW_RD_RS_N_ARB, 0x01070102);
1329         WREG32(mmMME4_RTR_HBW_RD_RS_N_ARB, 0x01060102);
1330         WREG32(mmMME3_RTR_HBW_RD_RS_N_ARB, 0x01040102);
1331         WREG32(mmMME2_RTR_HBW_RD_RS_N_ARB, 0x01020102);
1332         WREG32(mmMME1_RTR_HBW_RD_RS_N_ARB, 0x01020107);
1333         WREG32(mmMME6_RTR_HBW_RD_RS_S_ARB, 0x01020106);
1334         WREG32(mmMME5_RTR_HBW_RD_RS_S_ARB, 0x01020102);
1335         WREG32(mmMME4_RTR_HBW_RD_RS_S_ARB, 0x01040102);
1336         WREG32(mmMME3_RTR_HBW_RD_RS_S_ARB, 0x01060102);
1337         WREG32(mmMME2_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1338         WREG32(mmMME1_RTR_HBW_RD_RS_S_ARB, 0x01070102);
1339         WREG32(mmMME6_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1340         WREG32(mmMME5_RTR_HBW_RD_RS_E_ARB, 0x01020702);
1341         WREG32(mmMME4_RTR_HBW_RD_RS_E_ARB, 0x01040602);
1342         WREG32(mmMME3_RTR_HBW_RD_RS_E_ARB, 0x01060402);
1343         WREG32(mmMME2_RTR_HBW_RD_RS_E_ARB, 0x01070202);
1344         WREG32(mmMME1_RTR_HBW_RD_RS_E_ARB, 0x01070102);
1345         WREG32(mmMME6_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1346         WREG32(mmMME5_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1347         WREG32(mmMME4_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1348         WREG32(mmMME3_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1349         WREG32(mmMME2_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1350         WREG32(mmMME1_RTR_HBW_RD_RS_W_ARB, 0x01060401);
1351         WREG32(mmMME6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1352         WREG32(mmMME5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1353         WREG32(mmMME4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1354         WREG32(mmMME3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1355         WREG32(mmMME2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1356         WREG32(mmMME1_RTR_HBW_WR_RS_N_ARB, 0x01010107);
1357         WREG32(mmMME6_RTR_HBW_WR_RS_S_ARB, 0x01010107);
1358         WREG32(mmMME5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1359         WREG32(mmMME4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1360         WREG32(mmMME3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1361         WREG32(mmMME2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1362         WREG32(mmMME1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1363         WREG32(mmMME6_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1364         WREG32(mmMME5_RTR_HBW_WR_RS_E_ARB, 0x01010501);
1365         WREG32(mmMME4_RTR_HBW_WR_RS_E_ARB, 0x01040301);
1366         WREG32(mmMME3_RTR_HBW_WR_RS_E_ARB, 0x01030401);
1367         WREG32(mmMME2_RTR_HBW_WR_RS_E_ARB, 0x01040101);
1368         WREG32(mmMME1_RTR_HBW_WR_RS_E_ARB, 0x01050101);
1369         WREG32(mmMME6_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1370         WREG32(mmMME5_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1371         WREG32(mmMME4_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1372         WREG32(mmMME3_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1373         WREG32(mmMME2_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1374         WREG32(mmMME1_RTR_HBW_WR_RS_W_ARB, 0x01010101);
1375
1376         WREG32(mmTPC1_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1377         WREG32(mmTPC1_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1378         WREG32(mmTPC1_RTR_HBW_RD_RQ_E_ARB, 0x01060101);
1379         WREG32(mmTPC1_RTR_HBW_WR_RQ_N_ARB, 0x02020102);
1380         WREG32(mmTPC1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1381         WREG32(mmTPC1_RTR_HBW_WR_RQ_E_ARB, 0x02070202);
1382         WREG32(mmTPC1_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1383         WREG32(mmTPC1_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1384         WREG32(mmTPC1_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1385         WREG32(mmTPC1_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1386         WREG32(mmTPC1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
1387         WREG32(mmTPC1_RTR_HBW_WR_RS_W_ARB, 0x01050101);
1388
1389         WREG32(mmTPC2_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
1390         WREG32(mmTPC2_RTR_HBW_RD_RQ_S_ARB, 0x01050101);
1391         WREG32(mmTPC2_RTR_HBW_RD_RQ_E_ARB, 0x01010201);
1392         WREG32(mmTPC2_RTR_HBW_WR_RQ_N_ARB, 0x02040102);
1393         WREG32(mmTPC2_RTR_HBW_WR_RQ_S_ARB, 0x01050101);
1394         WREG32(mmTPC2_RTR_HBW_WR_RQ_E_ARB, 0x02060202);
1395         WREG32(mmTPC2_RTR_HBW_RD_RS_N_ARB, 0x01020201);
1396         WREG32(mmTPC2_RTR_HBW_RD_RS_S_ARB, 0x01070201);
1397         WREG32(mmTPC2_RTR_HBW_RD_RS_W_ARB, 0x01070202);
1398         WREG32(mmTPC2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
1399         WREG32(mmTPC2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
1400         WREG32(mmTPC2_RTR_HBW_WR_RS_W_ARB, 0x01040101);
1401
1402         WREG32(mmTPC3_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
1403         WREG32(mmTPC3_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
1404         WREG32(mmTPC3_RTR_HBW_RD_RQ_E_ARB, 0x01040301);
1405         WREG32(mmTPC3_RTR_HBW_WR_RQ_N_ARB, 0x02060102);
1406         WREG32(mmTPC3_RTR_HBW_WR_RQ_S_ARB, 0x01040101);
1407         WREG32(mmTPC3_RTR_HBW_WR_RQ_E_ARB, 0x01040301);
1408         WREG32(mmTPC3_RTR_HBW_RD_RS_N_ARB, 0x01040201);
1409         WREG32(mmTPC3_RTR_HBW_RD_RS_S_ARB, 0x01060201);
1410         WREG32(mmTPC3_RTR_HBW_RD_RS_W_ARB, 0x01060402);
1411         WREG32(mmTPC3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
1412         WREG32(mmTPC3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
1413         WREG32(mmTPC3_RTR_HBW_WR_RS_W_ARB, 0x01030401);
1414
1415         WREG32(mmTPC4_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
1416         WREG32(mmTPC4_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
1417         WREG32(mmTPC4_RTR_HBW_RD_RQ_E_ARB, 0x01030401);
1418         WREG32(mmTPC4_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1419         WREG32(mmTPC4_RTR_HBW_WR_RQ_S_ARB, 0x01030101);
1420         WREG32(mmTPC4_RTR_HBW_WR_RQ_E_ARB, 0x02060702);
1421         WREG32(mmTPC4_RTR_HBW_RD_RS_N_ARB, 0x01060201);
1422         WREG32(mmTPC4_RTR_HBW_RD_RS_S_ARB, 0x01040201);
1423         WREG32(mmTPC4_RTR_HBW_RD_RS_W_ARB, 0x01040602);
1424         WREG32(mmTPC4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
1425         WREG32(mmTPC4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
1426         WREG32(mmTPC4_RTR_HBW_WR_RS_W_ARB, 0x01040301);
1427
1428         WREG32(mmTPC5_RTR_HBW_RD_RQ_N_ARB, 0x01050101);
1429         WREG32(mmTPC5_RTR_HBW_RD_RQ_S_ARB, 0x01020101);
1430         WREG32(mmTPC5_RTR_HBW_RD_RQ_E_ARB, 0x01200501);
1431         WREG32(mmTPC5_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
1432         WREG32(mmTPC5_RTR_HBW_WR_RQ_S_ARB, 0x01020101);
1433         WREG32(mmTPC5_RTR_HBW_WR_RQ_E_ARB, 0x02020602);
1434         WREG32(mmTPC5_RTR_HBW_RD_RS_N_ARB, 0x01070201);
1435         WREG32(mmTPC5_RTR_HBW_RD_RS_S_ARB, 0x01020201);
1436         WREG32(mmTPC5_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1437         WREG32(mmTPC5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
1438         WREG32(mmTPC5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1439         WREG32(mmTPC5_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1440
1441         WREG32(mmTPC6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
1442         WREG32(mmTPC6_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
1443         WREG32(mmTPC6_RTR_HBW_RD_RQ_E_ARB, 0x01010601);
1444         WREG32(mmTPC6_RTR_HBW_WR_RQ_N_ARB, 0x01010101);
1445         WREG32(mmTPC6_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
1446         WREG32(mmTPC6_RTR_HBW_WR_RQ_E_ARB, 0x02020702);
1447         WREG32(mmTPC6_RTR_HBW_RD_RS_N_ARB, 0x01010101);
1448         WREG32(mmTPC6_RTR_HBW_RD_RS_S_ARB, 0x01010101);
1449         WREG32(mmTPC6_RTR_HBW_RD_RS_W_ARB, 0x01020702);
1450         WREG32(mmTPC6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
1451         WREG32(mmTPC6_RTR_HBW_WR_RS_S_ARB, 0x01010101);
1452         WREG32(mmTPC6_RTR_HBW_WR_RS_W_ARB, 0x01010501);
1453
1454         for (i = 0, offset = 0 ; i < 10 ; i++, offset += 4) {
1455                 WREG32(mmMME1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1456                 WREG32(mmMME2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1457                 WREG32(mmMME3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1458                 WREG32(mmMME4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1459                 WREG32(mmMME5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1460                 WREG32(mmMME6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1461
1462                 WREG32(mmTPC0_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1463                 WREG32(mmTPC1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1464                 WREG32(mmTPC2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1465                 WREG32(mmTPC3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1466                 WREG32(mmTPC4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1467                 WREG32(mmTPC5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1468                 WREG32(mmTPC6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1469                 WREG32(mmTPC7_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1470
1471                 WREG32(mmPCI_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1472                 WREG32(mmDMA_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
1473         }
1474
1475         for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x40000) {
1476                 WREG32(mmMME1_RTR_SCRAMB_EN + offset,
1477                                 1 << MME1_RTR_SCRAMB_EN_VAL_SHIFT);
1478                 WREG32(mmMME1_RTR_NON_LIN_SCRAMB + offset,
1479                                 1 << MME1_RTR_NON_LIN_SCRAMB_EN_SHIFT);
1480         }
1481
1482         for (i = 0, offset = 0 ; i < 8 ; i++, offset += 0x40000) {
1483                 /*
1484                  * Workaround for Bug H2 #2441 :
1485                  * "ST.NOP set trace event illegal opcode"
1486                  */
1487                 WREG32(mmTPC0_CFG_TPC_INTR_MASK + offset, tpc_intr_mask);
1488
1489                 WREG32(mmTPC0_NRTR_SCRAMB_EN + offset,
1490                                 1 << TPC0_NRTR_SCRAMB_EN_VAL_SHIFT);
1491                 WREG32(mmTPC0_NRTR_NON_LIN_SCRAMB + offset,
1492                                 1 << TPC0_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1493
1494                 WREG32_FIELD(TPC0_CFG_MSS_CONFIG, offset,
1495                                 ICACHE_FETCH_LINE_NUM, 2);
1496         }
1497
1498         WREG32(mmDMA_NRTR_SCRAMB_EN, 1 << DMA_NRTR_SCRAMB_EN_VAL_SHIFT);
1499         WREG32(mmDMA_NRTR_NON_LIN_SCRAMB,
1500                         1 << DMA_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1501
1502         WREG32(mmPCI_NRTR_SCRAMB_EN, 1 << PCI_NRTR_SCRAMB_EN_VAL_SHIFT);
1503         WREG32(mmPCI_NRTR_NON_LIN_SCRAMB,
1504                         1 << PCI_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
1505
1506         /*
1507          * Workaround for H2 #HW-23 bug
1508          * Set DMA max outstanding read requests to 240 on DMA CH 1.
1509          * This limitation is still large enough to not affect Gen4 bandwidth.
1510          * We need to only limit that DMA channel because the user can only read
1511          * from Host using DMA CH 1
1512          */
1513         WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0);
1514
1515         WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
1516
1517         goya->hw_cap_initialized |= HW_CAP_GOLDEN;
1518 }
1519
1520 static void goya_init_mme_qman(struct hl_device *hdev)
1521 {
1522         u32 mtr_base_lo, mtr_base_hi;
1523         u32 so_base_lo, so_base_hi;
1524         u32 gic_base_lo, gic_base_hi;
1525         u64 qman_base_addr;
1526
1527         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1528         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1529         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1530         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1531
1532         gic_base_lo =
1533                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1534         gic_base_hi =
1535                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1536
1537         qman_base_addr = hdev->asic_prop.sram_base_address +
1538                                 MME_QMAN_BASE_OFFSET;
1539
1540         WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
1541         WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
1542         WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
1543         WREG32(mmMME_QM_PQ_PI, 0);
1544         WREG32(mmMME_QM_PQ_CI, 0);
1545         WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
1546         WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
1547         WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
1548         WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
1549
1550         WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1551         WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1552         WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1553         WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1554
1555         /* QMAN CQ has 8 cache lines */
1556         WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
1557
1558         WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
1559         WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
1560
1561         WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
1562
1563         WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
1564
1565         WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
1566
1567         WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
1568 }
1569
1570 static void goya_init_mme_cmdq(struct hl_device *hdev)
1571 {
1572         u32 mtr_base_lo, mtr_base_hi;
1573         u32 so_base_lo, so_base_hi;
1574         u32 gic_base_lo, gic_base_hi;
1575
1576         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1577         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1578         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1579         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1580
1581         gic_base_lo =
1582                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1583         gic_base_hi =
1584                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1585
1586         WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
1587         WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
1588         WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
1589         WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
1590
1591         /* CMDQ CQ has 20 cache lines */
1592         WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
1593
1594         WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
1595         WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
1596
1597         WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
1598
1599         WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
1600
1601         WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
1602
1603         WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
1604 }
1605
1606 void goya_init_mme_qmans(struct hl_device *hdev)
1607 {
1608         struct goya_device *goya = hdev->asic_specific;
1609         u32 so_base_lo, so_base_hi;
1610
1611         if (goya->hw_cap_initialized & HW_CAP_MME)
1612                 return;
1613
1614         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1615         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1616
1617         WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
1618         WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
1619
1620         goya_init_mme_qman(hdev);
1621         goya_init_mme_cmdq(hdev);
1622
1623         goya->hw_cap_initialized |= HW_CAP_MME;
1624 }
1625
1626 static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
1627 {
1628         u32 mtr_base_lo, mtr_base_hi;
1629         u32 so_base_lo, so_base_hi;
1630         u32 gic_base_lo, gic_base_hi;
1631         u64 qman_base_addr;
1632         u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
1633
1634         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1635         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1636         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1637         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1638
1639         gic_base_lo =
1640                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1641         gic_base_hi =
1642                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1643
1644         qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
1645
1646         WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
1647         WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
1648         WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
1649         WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
1650         WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
1651         WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
1652         WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
1653         WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
1654         WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
1655
1656         WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1657         WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1658         WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1659         WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1660
1661         WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
1662
1663         WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1664         WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1665
1666         WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
1667                         GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
1668
1669         WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
1670
1671         WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
1672
1673         WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
1674 }
1675
1676 static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
1677 {
1678         u32 mtr_base_lo, mtr_base_hi;
1679         u32 so_base_lo, so_base_hi;
1680         u32 gic_base_lo, gic_base_hi;
1681         u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
1682
1683         mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1684         mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
1685         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1686         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1687
1688         gic_base_lo =
1689                 lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1690         gic_base_hi =
1691                 upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
1692
1693         WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
1694         WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
1695         WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
1696         WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
1697
1698         WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
1699
1700         WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
1701         WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
1702
1703         WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
1704                         GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
1705
1706         WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
1707
1708         WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
1709
1710         WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
1711 }
1712
1713 void goya_init_tpc_qmans(struct hl_device *hdev)
1714 {
1715         struct goya_device *goya = hdev->asic_specific;
1716         u32 so_base_lo, so_base_hi;
1717         u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
1718                         mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
1719         int i;
1720
1721         if (goya->hw_cap_initialized & HW_CAP_TPC)
1722                 return;
1723
1724         so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1725         so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
1726
1727         for (i = 0 ; i < TPC_MAX_NUM ; i++) {
1728                 WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
1729                                 so_base_lo);
1730                 WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
1731                                 so_base_hi);
1732         }
1733
1734         goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
1735         goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
1736         goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
1737         goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
1738         goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
1739         goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
1740         goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
1741         goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
1742
1743         for (i = 0 ; i < TPC_MAX_NUM ; i++)
1744                 goya_init_tpc_cmdq(hdev, i);
1745
1746         goya->hw_cap_initialized |= HW_CAP_TPC;
1747 }
1748
1749 /*
1750  * goya_disable_internal_queues - Disable internal queues
1751  *
1752  * @hdev: pointer to hl_device structure
1753  *
1754  */
1755 static void goya_disable_internal_queues(struct hl_device *hdev)
1756 {
1757         struct goya_device *goya = hdev->asic_specific;
1758
1759         if (!(goya->hw_cap_initialized & HW_CAP_MME))
1760                 goto disable_tpc;
1761
1762         WREG32(mmMME_QM_GLBL_CFG0, 0);
1763         WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
1764
1765 disable_tpc:
1766         if (!(goya->hw_cap_initialized & HW_CAP_TPC))
1767                 return;
1768
1769         WREG32(mmTPC0_QM_GLBL_CFG0, 0);
1770         WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
1771
1772         WREG32(mmTPC1_QM_GLBL_CFG0, 0);
1773         WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
1774
1775         WREG32(mmTPC2_QM_GLBL_CFG0, 0);
1776         WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
1777
1778         WREG32(mmTPC3_QM_GLBL_CFG0, 0);
1779         WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
1780
1781         WREG32(mmTPC4_QM_GLBL_CFG0, 0);
1782         WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
1783
1784         WREG32(mmTPC5_QM_GLBL_CFG0, 0);
1785         WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
1786
1787         WREG32(mmTPC6_QM_GLBL_CFG0, 0);
1788         WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
1789
1790         WREG32(mmTPC7_QM_GLBL_CFG0, 0);
1791         WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
1792 }
1793
1794 /*
1795  * goya_stop_internal_queues - Stop internal queues
1796  *
1797  * @hdev: pointer to hl_device structure
1798  *
1799  * Returns 0 on success
1800  *
1801  */
1802 static int goya_stop_internal_queues(struct hl_device *hdev)
1803 {
1804         struct goya_device *goya = hdev->asic_specific;
1805         int rc, retval = 0;
1806
1807         if (!(goya->hw_cap_initialized & HW_CAP_MME))
1808                 goto stop_tpc;
1809
1810         /*
1811          * Each queue (QMAN) is a separate H/W logic. That means that each
1812          * QMAN can be stopped independently and failure to stop one does NOT
1813          * mandate we should not try to stop other QMANs
1814          */
1815
1816         rc = goya_stop_queue(hdev,
1817                         mmMME_QM_GLBL_CFG1,
1818                         mmMME_QM_CP_STS,
1819                         mmMME_QM_GLBL_STS0);
1820
1821         if (rc) {
1822                 dev_err(hdev->dev, "failed to stop MME QMAN\n");
1823                 retval = -EIO;
1824         }
1825
1826         rc = goya_stop_queue(hdev,
1827                         mmMME_CMDQ_GLBL_CFG1,
1828                         mmMME_CMDQ_CP_STS,
1829                         mmMME_CMDQ_GLBL_STS0);
1830
1831         if (rc) {
1832                 dev_err(hdev->dev, "failed to stop MME CMDQ\n");
1833                 retval = -EIO;
1834         }
1835
1836 stop_tpc:
1837         if (!(goya->hw_cap_initialized & HW_CAP_TPC))
1838                 return retval;
1839
1840         rc = goya_stop_queue(hdev,
1841                         mmTPC0_QM_GLBL_CFG1,
1842                         mmTPC0_QM_CP_STS,
1843                         mmTPC0_QM_GLBL_STS0);
1844
1845         if (rc) {
1846                 dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
1847                 retval = -EIO;
1848         }
1849
1850         rc = goya_stop_queue(hdev,
1851                         mmTPC0_CMDQ_GLBL_CFG1,
1852                         mmTPC0_CMDQ_CP_STS,
1853                         mmTPC0_CMDQ_GLBL_STS0);
1854
1855         if (rc) {
1856                 dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
1857                 retval = -EIO;
1858         }
1859
1860         rc = goya_stop_queue(hdev,
1861                         mmTPC1_QM_GLBL_CFG1,
1862                         mmTPC1_QM_CP_STS,
1863                         mmTPC1_QM_GLBL_STS0);
1864
1865         if (rc) {
1866                 dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
1867                 retval = -EIO;
1868         }
1869
1870         rc = goya_stop_queue(hdev,
1871                         mmTPC1_CMDQ_GLBL_CFG1,
1872                         mmTPC1_CMDQ_CP_STS,
1873                         mmTPC1_CMDQ_GLBL_STS0);
1874
1875         if (rc) {
1876                 dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
1877                 retval = -EIO;
1878         }
1879
1880         rc = goya_stop_queue(hdev,
1881                         mmTPC2_QM_GLBL_CFG1,
1882                         mmTPC2_QM_CP_STS,
1883                         mmTPC2_QM_GLBL_STS0);
1884
1885         if (rc) {
1886                 dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
1887                 retval = -EIO;
1888         }
1889
1890         rc = goya_stop_queue(hdev,
1891                         mmTPC2_CMDQ_GLBL_CFG1,
1892                         mmTPC2_CMDQ_CP_STS,
1893                         mmTPC2_CMDQ_GLBL_STS0);
1894
1895         if (rc) {
1896                 dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
1897                 retval = -EIO;
1898         }
1899
1900         rc = goya_stop_queue(hdev,
1901                         mmTPC3_QM_GLBL_CFG1,
1902                         mmTPC3_QM_CP_STS,
1903                         mmTPC3_QM_GLBL_STS0);
1904
1905         if (rc) {
1906                 dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
1907                 retval = -EIO;
1908         }
1909
1910         rc = goya_stop_queue(hdev,
1911                         mmTPC3_CMDQ_GLBL_CFG1,
1912                         mmTPC3_CMDQ_CP_STS,
1913                         mmTPC3_CMDQ_GLBL_STS0);
1914
1915         if (rc) {
1916                 dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
1917                 retval = -EIO;
1918         }
1919
1920         rc = goya_stop_queue(hdev,
1921                         mmTPC4_QM_GLBL_CFG1,
1922                         mmTPC4_QM_CP_STS,
1923                         mmTPC4_QM_GLBL_STS0);
1924
1925         if (rc) {
1926                 dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
1927                 retval = -EIO;
1928         }
1929
1930         rc = goya_stop_queue(hdev,
1931                         mmTPC4_CMDQ_GLBL_CFG1,
1932                         mmTPC4_CMDQ_CP_STS,
1933                         mmTPC4_CMDQ_GLBL_STS0);
1934
1935         if (rc) {
1936                 dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
1937                 retval = -EIO;
1938         }
1939
1940         rc = goya_stop_queue(hdev,
1941                         mmTPC5_QM_GLBL_CFG1,
1942                         mmTPC5_QM_CP_STS,
1943                         mmTPC5_QM_GLBL_STS0);
1944
1945         if (rc) {
1946                 dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
1947                 retval = -EIO;
1948         }
1949
1950         rc = goya_stop_queue(hdev,
1951                         mmTPC5_CMDQ_GLBL_CFG1,
1952                         mmTPC5_CMDQ_CP_STS,
1953                         mmTPC5_CMDQ_GLBL_STS0);
1954
1955         if (rc) {
1956                 dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
1957                 retval = -EIO;
1958         }
1959
1960         rc = goya_stop_queue(hdev,
1961                         mmTPC6_QM_GLBL_CFG1,
1962                         mmTPC6_QM_CP_STS,
1963                         mmTPC6_QM_GLBL_STS0);
1964
1965         if (rc) {
1966                 dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
1967                 retval = -EIO;
1968         }
1969
1970         rc = goya_stop_queue(hdev,
1971                         mmTPC6_CMDQ_GLBL_CFG1,
1972                         mmTPC6_CMDQ_CP_STS,
1973                         mmTPC6_CMDQ_GLBL_STS0);
1974
1975         if (rc) {
1976                 dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
1977                 retval = -EIO;
1978         }
1979
1980         rc = goya_stop_queue(hdev,
1981                         mmTPC7_QM_GLBL_CFG1,
1982                         mmTPC7_QM_CP_STS,
1983                         mmTPC7_QM_GLBL_STS0);
1984
1985         if (rc) {
1986                 dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
1987                 retval = -EIO;
1988         }
1989
1990         rc = goya_stop_queue(hdev,
1991                         mmTPC7_CMDQ_GLBL_CFG1,
1992                         mmTPC7_CMDQ_CP_STS,
1993                         mmTPC7_CMDQ_GLBL_STS0);
1994
1995         if (rc) {
1996                 dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
1997                 retval = -EIO;
1998         }
1999
2000         return retval;
2001 }
2002
2003 static void goya_dma_stall(struct hl_device *hdev)
2004 {
2005         struct goya_device *goya = hdev->asic_specific;
2006
2007         if (!(goya->hw_cap_initialized & HW_CAP_DMA))
2008                 return;
2009
2010         WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT);
2011         WREG32(mmDMA_QM_1_GLBL_CFG1, 1 << DMA_QM_1_GLBL_CFG1_DMA_STOP_SHIFT);
2012         WREG32(mmDMA_QM_2_GLBL_CFG1, 1 << DMA_QM_2_GLBL_CFG1_DMA_STOP_SHIFT);
2013         WREG32(mmDMA_QM_3_GLBL_CFG1, 1 << DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT);
2014         WREG32(mmDMA_QM_4_GLBL_CFG1, 1 << DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT);
2015 }
2016
2017 static void goya_tpc_stall(struct hl_device *hdev)
2018 {
2019         struct goya_device *goya = hdev->asic_specific;
2020
2021         if (!(goya->hw_cap_initialized & HW_CAP_TPC))
2022                 return;
2023
2024         WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
2025         WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC1_CFG_TPC_STALL_V_SHIFT);
2026         WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC2_CFG_TPC_STALL_V_SHIFT);
2027         WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC3_CFG_TPC_STALL_V_SHIFT);
2028         WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC4_CFG_TPC_STALL_V_SHIFT);
2029         WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC5_CFG_TPC_STALL_V_SHIFT);
2030         WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC6_CFG_TPC_STALL_V_SHIFT);
2031         WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC7_CFG_TPC_STALL_V_SHIFT);
2032 }
2033
2034 static void goya_mme_stall(struct hl_device *hdev)
2035 {
2036         struct goya_device *goya = hdev->asic_specific;
2037
2038         if (!(goya->hw_cap_initialized & HW_CAP_MME))
2039                 return;
2040
2041         WREG32(mmMME_STALL, 0xFFFFFFFF);
2042 }
2043
2044 static int goya_enable_msix(struct hl_device *hdev)
2045 {
2046         struct goya_device *goya = hdev->asic_specific;
2047         int cq_cnt = hdev->asic_prop.completion_queues_count;
2048         int rc, i, irq_cnt_init, irq;
2049
2050         if (goya->hw_cap_initialized & HW_CAP_MSIX)
2051                 return 0;
2052
2053         rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES,
2054                                 GOYA_MSIX_ENTRIES, PCI_IRQ_MSIX);
2055         if (rc < 0) {
2056                 dev_err(hdev->dev,
2057                         "MSI-X: Failed to enable support -- %d/%d\n",
2058                         GOYA_MSIX_ENTRIES, rc);
2059                 return rc;
2060         }
2061
2062         for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
2063                 irq = pci_irq_vector(hdev->pdev, i);
2064                 rc = request_irq(irq, hl_irq_handler_cq, 0, goya_irq_name[i],
2065                                 &hdev->completion_queue[i]);
2066                 if (rc) {
2067                         dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2068                         goto free_irqs;
2069                 }
2070         }
2071
2072         irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2073
2074         rc = request_irq(irq, hl_irq_handler_eq, 0,
2075                         goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX],
2076                         &hdev->event_queue);
2077         if (rc) {
2078                 dev_err(hdev->dev, "Failed to request IRQ %d", irq);
2079                 goto free_irqs;
2080         }
2081
2082         goya->hw_cap_initialized |= HW_CAP_MSIX;
2083         return 0;
2084
2085 free_irqs:
2086         for (i = 0 ; i < irq_cnt_init ; i++)
2087                 free_irq(pci_irq_vector(hdev->pdev, i),
2088                         &hdev->completion_queue[i]);
2089
2090         pci_free_irq_vectors(hdev->pdev);
2091         return rc;
2092 }
2093
2094 static void goya_sync_irqs(struct hl_device *hdev)
2095 {
2096         struct goya_device *goya = hdev->asic_specific;
2097         int i;
2098
2099         if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2100                 return;
2101
2102         /* Wait for all pending IRQs to be finished */
2103         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2104                 synchronize_irq(pci_irq_vector(hdev->pdev, i));
2105
2106         synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
2107 }
2108
2109 static void goya_disable_msix(struct hl_device *hdev)
2110 {
2111         struct goya_device *goya = hdev->asic_specific;
2112         int i, irq;
2113
2114         if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
2115                 return;
2116
2117         goya_sync_irqs(hdev);
2118
2119         irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
2120         free_irq(irq, &hdev->event_queue);
2121
2122         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
2123                 irq = pci_irq_vector(hdev->pdev, i);
2124                 free_irq(irq, &hdev->completion_queue[i]);
2125         }
2126
2127         pci_free_irq_vectors(hdev->pdev);
2128
2129         goya->hw_cap_initialized &= ~HW_CAP_MSIX;
2130 }
2131
2132 static void goya_enable_timestamp(struct hl_device *hdev)
2133 {
2134         /* Disable the timestamp counter */
2135         WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2136
2137         /* Zero the lower/upper parts of the 64-bit counter */
2138         WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
2139         WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
2140
2141         /* Enable the counter */
2142         WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
2143 }
2144
2145 static void goya_disable_timestamp(struct hl_device *hdev)
2146 {
2147         /* Disable the timestamp counter */
2148         WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
2149 }
2150
2151 static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
2152 {
2153         u32 wait_timeout_ms, cpu_timeout_ms;
2154
2155         dev_info(hdev->dev,
2156                 "Halting compute engines and disabling interrupts\n");
2157
2158         if (hdev->pldm) {
2159                 wait_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2160                 cpu_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
2161         } else {
2162                 wait_timeout_ms = GOYA_RESET_WAIT_MSEC;
2163                 cpu_timeout_ms = GOYA_CPU_RESET_WAIT_MSEC;
2164         }
2165
2166         if (hard_reset) {
2167                 /*
2168                  * I don't know what is the state of the CPU so make sure it is
2169                  * stopped in any means necessary
2170                  */
2171                 WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_GOTO_WFE);
2172                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2173                         GOYA_ASYNC_EVENT_ID_HALT_MACHINE);
2174                 msleep(cpu_timeout_ms);
2175         }
2176
2177         goya_stop_external_queues(hdev);
2178         goya_stop_internal_queues(hdev);
2179
2180         msleep(wait_timeout_ms);
2181
2182         goya_dma_stall(hdev);
2183         goya_tpc_stall(hdev);
2184         goya_mme_stall(hdev);
2185
2186         msleep(wait_timeout_ms);
2187
2188         goya_disable_external_queues(hdev);
2189         goya_disable_internal_queues(hdev);
2190
2191         goya_disable_timestamp(hdev);
2192
2193         if (hard_reset) {
2194                 goya_disable_msix(hdev);
2195                 goya_mmu_remove_device_cpu_mappings(hdev);
2196         } else {
2197                 goya_sync_irqs(hdev);
2198         }
2199 }
2200
2201 /*
2202  * goya_push_uboot_to_device() - Push u-boot FW code to device.
2203  * @hdev: Pointer to hl_device structure.
2204  *
2205  * Copy u-boot fw code from firmware file to SRAM BAR.
2206  *
2207  * Return: 0 on success, non-zero for failure.
2208  */
2209 static int goya_push_uboot_to_device(struct hl_device *hdev)
2210 {
2211         void __iomem *dst;
2212
2213         dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + UBOOT_FW_OFFSET;
2214
2215         return hl_fw_push_fw_to_device(hdev, GOYA_UBOOT_FW_FILE, dst);
2216 }
2217
2218 /*
2219  * goya_push_linux_to_device() - Push LINUX FW code to device.
2220  * @hdev: Pointer to hl_device structure.
2221  *
2222  * Copy LINUX fw code from firmware file to HBM BAR.
2223  *
2224  * Return: 0 on success, non-zero for failure.
2225  */
2226 static int goya_push_linux_to_device(struct hl_device *hdev)
2227 {
2228         void __iomem *dst;
2229
2230         dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
2231
2232         return hl_fw_push_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst);
2233 }
2234
2235 static int goya_pldm_init_cpu(struct hl_device *hdev)
2236 {
2237         u32 unit_rst_val;
2238         int rc;
2239
2240         /* Must initialize SRAM scrambler before pushing u-boot to SRAM */
2241         goya_init_golden_registers(hdev);
2242
2243         /* Put ARM cores into reset */
2244         WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL, CPU_RESET_ASSERT);
2245         RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
2246
2247         /* Reset the CA53 MACRO */
2248         unit_rst_val = RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2249         WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, CA53_RESET);
2250         RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2251         WREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N, unit_rst_val);
2252         RREG32(mmPSOC_GLOBAL_CONF_UNIT_RST_N);
2253
2254         rc = goya_push_uboot_to_device(hdev);
2255         if (rc)
2256                 return rc;
2257
2258         rc = goya_push_linux_to_device(hdev);
2259         if (rc)
2260                 return rc;
2261
2262         WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
2263         WREG32(mmPSOC_GLOBAL_CONF_WARM_REBOOT, CPU_BOOT_STATUS_NA);
2264
2265         WREG32(mmCPU_CA53_CFG_RST_ADDR_LSB_0,
2266                 lower_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
2267         WREG32(mmCPU_CA53_CFG_RST_ADDR_MSB_0,
2268                 upper_32_bits(SRAM_BASE_ADDR + UBOOT_FW_OFFSET));
2269
2270         /* Release ARM core 0 from reset */
2271         WREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL,
2272                                         CPU_RESET_CORE0_DEASSERT);
2273         RREG32(mmCPU_CA53_CFG_ARM_RST_CONTROL);
2274
2275         return 0;
2276 }
2277
2278 /*
2279  * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
2280  * The version string should be located by that offset.
2281  */
2282 static void goya_read_device_fw_version(struct hl_device *hdev,
2283                                         enum goya_fw_component fwc)
2284 {
2285         const char *name;
2286         u32 ver_off;
2287         char *dest;
2288
2289         switch (fwc) {
2290         case FW_COMP_UBOOT:
2291                 ver_off = RREG32(mmUBOOT_VER_OFFSET);
2292                 dest = hdev->asic_prop.uboot_ver;
2293                 name = "U-Boot";
2294                 break;
2295         case FW_COMP_PREBOOT:
2296                 ver_off = RREG32(mmPREBOOT_VER_OFFSET);
2297                 dest = hdev->asic_prop.preboot_ver;
2298                 name = "Preboot";
2299                 break;
2300         default:
2301                 dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
2302                 return;
2303         }
2304
2305         ver_off &= ~((u32)SRAM_BASE_ADDR);
2306
2307         if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
2308                 memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
2309                                                         VERSION_MAX_LEN);
2310         } else {
2311                 dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
2312                                                                 name, ver_off);
2313                 strcpy(dest, "unavailable");
2314         }
2315 }
2316
2317 static int goya_init_cpu(struct hl_device *hdev, u32 cpu_timeout)
2318 {
2319         struct goya_device *goya = hdev->asic_specific;
2320         u32 status;
2321         int rc;
2322
2323         if (!hdev->cpu_enable)
2324                 return 0;
2325
2326         if (goya->hw_cap_initialized & HW_CAP_CPU)
2327                 return 0;
2328
2329         /*
2330          * Before pushing u-boot/linux to device, need to set the ddr bar to
2331          * base address of dram
2332          */
2333         if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
2334                 dev_err(hdev->dev,
2335                         "failed to map DDR bar to DRAM base address\n");
2336                 return -EIO;
2337         }
2338
2339         if (hdev->pldm) {
2340                 rc = goya_pldm_init_cpu(hdev);
2341                 if (rc)
2342                         return rc;
2343
2344                 goto out;
2345         }
2346
2347         /* Make sure CPU boot-loader is running */
2348         rc = hl_poll_timeout(
2349                 hdev,
2350                 mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2351                 status,
2352                 (status == CPU_BOOT_STATUS_DRAM_RDY) ||
2353                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2354                 10000,
2355                 cpu_timeout);
2356
2357         /* Read U-Boot version now in case we will later fail */
2358         goya_read_device_fw_version(hdev, FW_COMP_UBOOT);
2359         goya_read_device_fw_version(hdev, FW_COMP_PREBOOT);
2360
2361         if (rc) {
2362                 dev_err(hdev->dev, "Error in ARM u-boot!");
2363                 switch (status) {
2364                 case CPU_BOOT_STATUS_NA:
2365                         dev_err(hdev->dev,
2366                                 "ARM status %d - BTL did NOT run\n", status);
2367                         break;
2368                 case CPU_BOOT_STATUS_IN_WFE:
2369                         dev_err(hdev->dev,
2370                                 "ARM status %d - Inside WFE loop\n", status);
2371                         break;
2372                 case CPU_BOOT_STATUS_IN_BTL:
2373                         dev_err(hdev->dev,
2374                                 "ARM status %d - Stuck in BTL\n", status);
2375                         break;
2376                 case CPU_BOOT_STATUS_IN_PREBOOT:
2377                         dev_err(hdev->dev,
2378                                 "ARM status %d - Stuck in Preboot\n", status);
2379                         break;
2380                 case CPU_BOOT_STATUS_IN_SPL:
2381                         dev_err(hdev->dev,
2382                                 "ARM status %d - Stuck in SPL\n", status);
2383                         break;
2384                 case CPU_BOOT_STATUS_IN_UBOOT:
2385                         dev_err(hdev->dev,
2386                                 "ARM status %d - Stuck in u-boot\n", status);
2387                         break;
2388                 case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
2389                         dev_err(hdev->dev,
2390                                 "ARM status %d - DDR initialization failed\n",
2391                                 status);
2392                         break;
2393                 case CPU_BOOT_STATUS_UBOOT_NOT_READY:
2394                         dev_err(hdev->dev,
2395                                 "ARM status %d - u-boot stopped by user\n",
2396                                 status);
2397                         break;
2398                 case CPU_BOOT_STATUS_TS_INIT_FAIL:
2399                         dev_err(hdev->dev,
2400                                 "ARM status %d - Thermal Sensor initialization failed\n",
2401                                 status);
2402                         break;
2403                 default:
2404                         dev_err(hdev->dev,
2405                                 "ARM status %d - Invalid status code\n",
2406                                 status);
2407                         break;
2408                 }
2409                 return -EIO;
2410         }
2411
2412         if (!hdev->fw_loading) {
2413                 dev_info(hdev->dev, "Skip loading FW\n");
2414                 goto out;
2415         }
2416
2417         if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
2418                 goto out;
2419
2420         rc = goya_push_linux_to_device(hdev);
2421         if (rc)
2422                 return rc;
2423
2424         WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_FIT_RDY);
2425
2426         rc = hl_poll_timeout(
2427                 hdev,
2428                 mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2429                 status,
2430                 (status == CPU_BOOT_STATUS_SRAM_AVAIL),
2431                 10000,
2432                 cpu_timeout);
2433
2434         if (rc) {
2435                 if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)
2436                         dev_err(hdev->dev,
2437                                 "ARM u-boot reports FIT image is corrupted\n");
2438                 else
2439                         dev_err(hdev->dev,
2440                                 "ARM Linux failed to load, %d\n", status);
2441                 WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_NA);
2442                 return -EIO;
2443         }
2444
2445         dev_info(hdev->dev, "Successfully loaded firmware to device\n");
2446
2447 out:
2448         goya->hw_cap_initialized |= HW_CAP_CPU;
2449
2450         return 0;
2451 }
2452
2453 static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
2454                                                 u64 phys_addr)
2455 {
2456         u32 status, timeout_usec;
2457         int rc;
2458
2459         if (hdev->pldm)
2460                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
2461         else
2462                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
2463
2464         WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
2465         WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
2466         WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
2467
2468         rc = hl_poll_timeout(
2469                 hdev,
2470                 MMU_ASID_BUSY,
2471                 status,
2472                 !(status & 0x80000000),
2473                 1000,
2474                 timeout_usec);
2475
2476         if (rc) {
2477                 dev_err(hdev->dev,
2478                         "Timeout during MMU hop0 config of asid %d\n", asid);
2479                 return rc;
2480         }
2481
2482         return 0;
2483 }
2484
2485 int goya_mmu_init(struct hl_device *hdev)
2486 {
2487         struct asic_fixed_properties *prop = &hdev->asic_prop;
2488         struct goya_device *goya = hdev->asic_specific;
2489         u64 hop0_addr;
2490         int rc, i;
2491
2492         if (!hdev->mmu_enable)
2493                 return 0;
2494
2495         if (goya->hw_cap_initialized & HW_CAP_MMU)
2496                 return 0;
2497
2498         hdev->dram_supports_virtual_memory = true;
2499         hdev->dram_default_page_mapping = true;
2500
2501         for (i = 0 ; i < prop->max_asid ; i++) {
2502                 hop0_addr = prop->mmu_pgt_addr +
2503                                 (i * prop->mmu_hop_table_size);
2504
2505                 rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
2506                 if (rc) {
2507                         dev_err(hdev->dev,
2508                                 "failed to set hop0 addr for asid %d\n", i);
2509                         goto err;
2510                 }
2511         }
2512
2513         goya->hw_cap_initialized |= HW_CAP_MMU;
2514
2515         /* init MMU cache manage page */
2516         WREG32(mmSTLB_CACHE_INV_BASE_39_8,
2517                                 lower_32_bits(MMU_CACHE_MNG_ADDR >> 8));
2518         WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
2519
2520         /* Remove follower feature due to performance bug */
2521         WREG32_AND(mmSTLB_STLB_FEATURE_EN,
2522                         (~STLB_STLB_FEATURE_EN_FOLLOWER_EN_MASK));
2523
2524         hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
2525                                         VM_TYPE_USERPTR | VM_TYPE_PHYS_PACK);
2526
2527         WREG32(mmMMU_MMU_ENABLE, 1);
2528         WREG32(mmMMU_SPI_MASK, 0xF);
2529
2530         return 0;
2531
2532 err:
2533         return rc;
2534 }
2535
2536 /*
2537  * goya_hw_init - Goya hardware initialization code
2538  *
2539  * @hdev: pointer to hl_device structure
2540  *
2541  * Returns 0 on success
2542  *
2543  */
2544 static int goya_hw_init(struct hl_device *hdev)
2545 {
2546         struct asic_fixed_properties *prop = &hdev->asic_prop;
2547         int rc;
2548
2549         dev_info(hdev->dev, "Starting initialization of H/W\n");
2550
2551         /* Perform read from the device to make sure device is up */
2552         RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2553
2554         /*
2555          * Let's mark in the H/W that we have reached this point. We check
2556          * this value in the reset_before_init function to understand whether
2557          * we need to reset the chip before doing H/W init. This register is
2558          * cleared by the H/W upon H/W reset
2559          */
2560         WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
2561
2562         rc = goya_init_cpu(hdev, GOYA_CPU_TIMEOUT_USEC);
2563         if (rc) {
2564                 dev_err(hdev->dev, "failed to initialize CPU\n");
2565                 return rc;
2566         }
2567
2568         goya_tpc_mbist_workaround(hdev);
2569
2570         goya_init_golden_registers(hdev);
2571
2572         /*
2573          * After CPU initialization is finished, change DDR bar mapping inside
2574          * iATU to point to the start address of the MMU page tables
2575          */
2576         if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE +
2577                         (MMU_PAGE_TABLES_ADDR &
2578                         ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) {
2579                 dev_err(hdev->dev,
2580                         "failed to map DDR bar to MMU page tables\n");
2581                 return -EIO;
2582         }
2583
2584         rc = goya_mmu_init(hdev);
2585         if (rc)
2586                 return rc;
2587
2588         goya_init_security(hdev);
2589
2590         goya_init_dma_qmans(hdev);
2591
2592         goya_init_mme_qmans(hdev);
2593
2594         goya_init_tpc_qmans(hdev);
2595
2596         goya_enable_timestamp(hdev);
2597
2598         /* MSI-X must be enabled before CPU queues are initialized */
2599         rc = goya_enable_msix(hdev);
2600         if (rc)
2601                 goto disable_queues;
2602
2603         /* Perform read from the device to flush all MSI-X configuration */
2604         RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
2605
2606         return 0;
2607
2608 disable_queues:
2609         goya_disable_internal_queues(hdev);
2610         goya_disable_external_queues(hdev);
2611
2612         return rc;
2613 }
2614
2615 /*
2616  * goya_hw_fini - Goya hardware tear-down code
2617  *
2618  * @hdev: pointer to hl_device structure
2619  * @hard_reset: should we do hard reset to all engines or just reset the
2620  *              compute/dma engines
2621  */
2622 static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
2623 {
2624         struct goya_device *goya = hdev->asic_specific;
2625         u32 reset_timeout_ms, status;
2626
2627         if (hdev->pldm)
2628                 reset_timeout_ms = GOYA_PLDM_RESET_TIMEOUT_MSEC;
2629         else
2630                 reset_timeout_ms = GOYA_RESET_TIMEOUT_MSEC;
2631
2632         if (hard_reset) {
2633                 goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
2634                 goya_disable_clk_rlx(hdev);
2635                 goya_set_pll_refclk(hdev);
2636
2637                 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, RESET_ALL);
2638                 dev_info(hdev->dev,
2639                         "Issued HARD reset command, going to wait %dms\n",
2640                         reset_timeout_ms);
2641         } else {
2642                 WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, DMA_MME_TPC_RESET);
2643                 dev_info(hdev->dev,
2644                         "Issued SOFT reset command, going to wait %dms\n",
2645                         reset_timeout_ms);
2646         }
2647
2648         /*
2649          * After hard reset, we can't poll the BTM_FSM register because the PSOC
2650          * itself is in reset. In either reset we need to wait until the reset
2651          * is deasserted
2652          */
2653         msleep(reset_timeout_ms);
2654
2655         status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
2656         if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
2657                 dev_err(hdev->dev,
2658                         "Timeout while waiting for device to reset 0x%x\n",
2659                         status);
2660
2661         if (!hard_reset) {
2662                 goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME |
2663                                                 HW_CAP_GOLDEN | HW_CAP_TPC);
2664                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2665                                 GOYA_ASYNC_EVENT_ID_SOFT_RESET);
2666                 return;
2667         }
2668
2669         /* Chicken bit to re-initiate boot sequencer flow */
2670         WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START,
2671                 1 << PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT);
2672         /* Move boot manager FSM to pre boot sequencer init state */
2673         WREG32(mmPSOC_GLOBAL_CONF_SW_BTM_FSM,
2674                         0xA << PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT);
2675
2676         goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
2677                                         HW_CAP_DDR_0 | HW_CAP_DDR_1 |
2678                                         HW_CAP_DMA | HW_CAP_MME |
2679                                         HW_CAP_MMU | HW_CAP_TPC_MBIST |
2680                                         HW_CAP_GOLDEN | HW_CAP_TPC);
2681         memset(goya->events_stat, 0, sizeof(goya->events_stat));
2682
2683         if (!hdev->pldm) {
2684                 int rc;
2685                 /* In case we are running inside VM and the VM is
2686                  * shutting down, we need to make sure CPU boot-loader
2687                  * is running before we can continue the VM shutdown.
2688                  * That is because the VM will send an FLR signal that
2689                  * we must answer
2690                  */
2691                 dev_info(hdev->dev,
2692                         "Going to wait up to %ds for CPU boot loader\n",
2693                         GOYA_CPU_TIMEOUT_USEC / 1000 / 1000);
2694
2695                 rc = hl_poll_timeout(
2696                         hdev,
2697                         mmPSOC_GLOBAL_CONF_WARM_REBOOT,
2698                         status,
2699                         (status == CPU_BOOT_STATUS_DRAM_RDY),
2700                         10000,
2701                         GOYA_CPU_TIMEOUT_USEC);
2702                 if (rc)
2703                         dev_err(hdev->dev,
2704                                 "failed to wait for CPU boot loader\n");
2705         }
2706 }
2707
2708 int goya_suspend(struct hl_device *hdev)
2709 {
2710         int rc;
2711
2712         rc = hl_fw_send_pci_access_msg(hdev, ARMCP_PACKET_DISABLE_PCI_ACCESS);
2713         if (rc)
2714                 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
2715
2716         return rc;
2717 }
2718
2719 int goya_resume(struct hl_device *hdev)
2720 {
2721         return goya_init_iatu(hdev);
2722 }
2723
2724 static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
2725                 u64 kaddress, phys_addr_t paddress, u32 size)
2726 {
2727         int rc;
2728
2729         vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
2730                         VM_DONTCOPY | VM_NORESERVE;
2731
2732         rc = remap_pfn_range(vma, vma->vm_start, paddress >> PAGE_SHIFT,
2733                                 size, vma->vm_page_prot);
2734         if (rc)
2735                 dev_err(hdev->dev, "remap_pfn_range error %d", rc);
2736
2737         return rc;
2738 }
2739
2740 void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
2741 {
2742         u32 db_reg_offset, db_value;
2743
2744         switch (hw_queue_id) {
2745         case GOYA_QUEUE_ID_DMA_0:
2746                 db_reg_offset = mmDMA_QM_0_PQ_PI;
2747                 break;
2748
2749         case GOYA_QUEUE_ID_DMA_1:
2750                 db_reg_offset = mmDMA_QM_1_PQ_PI;
2751                 break;
2752
2753         case GOYA_QUEUE_ID_DMA_2:
2754                 db_reg_offset = mmDMA_QM_2_PQ_PI;
2755                 break;
2756
2757         case GOYA_QUEUE_ID_DMA_3:
2758                 db_reg_offset = mmDMA_QM_3_PQ_PI;
2759                 break;
2760
2761         case GOYA_QUEUE_ID_DMA_4:
2762                 db_reg_offset = mmDMA_QM_4_PQ_PI;
2763                 break;
2764
2765         case GOYA_QUEUE_ID_CPU_PQ:
2766                 db_reg_offset = mmCPU_IF_PF_PQ_PI;
2767                 break;
2768
2769         case GOYA_QUEUE_ID_MME:
2770                 db_reg_offset = mmMME_QM_PQ_PI;
2771                 break;
2772
2773         case GOYA_QUEUE_ID_TPC0:
2774                 db_reg_offset = mmTPC0_QM_PQ_PI;
2775                 break;
2776
2777         case GOYA_QUEUE_ID_TPC1:
2778                 db_reg_offset = mmTPC1_QM_PQ_PI;
2779                 break;
2780
2781         case GOYA_QUEUE_ID_TPC2:
2782                 db_reg_offset = mmTPC2_QM_PQ_PI;
2783                 break;
2784
2785         case GOYA_QUEUE_ID_TPC3:
2786                 db_reg_offset = mmTPC3_QM_PQ_PI;
2787                 break;
2788
2789         case GOYA_QUEUE_ID_TPC4:
2790                 db_reg_offset = mmTPC4_QM_PQ_PI;
2791                 break;
2792
2793         case GOYA_QUEUE_ID_TPC5:
2794                 db_reg_offset = mmTPC5_QM_PQ_PI;
2795                 break;
2796
2797         case GOYA_QUEUE_ID_TPC6:
2798                 db_reg_offset = mmTPC6_QM_PQ_PI;
2799                 break;
2800
2801         case GOYA_QUEUE_ID_TPC7:
2802                 db_reg_offset = mmTPC7_QM_PQ_PI;
2803                 break;
2804
2805         default:
2806                 /* Should never get here */
2807                 dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n",
2808                         hw_queue_id);
2809                 return;
2810         }
2811
2812         db_value = pi;
2813
2814         /* ring the doorbell */
2815         WREG32(db_reg_offset, db_value);
2816
2817         if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ)
2818                 WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
2819                                 GOYA_ASYNC_EVENT_ID_PI_UPDATE);
2820 }
2821
2822 void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
2823 {
2824         /* The QMANs are on the SRAM so need to copy to IO space */
2825         memcpy_toio((void __iomem *) pqe, bd, sizeof(struct hl_bd));
2826 }
2827
2828 static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
2829                                         dma_addr_t *dma_handle, gfp_t flags)
2830 {
2831         void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
2832                                                 dma_handle, flags);
2833
2834         /* Shift to the device's base physical address of host memory */
2835         if (kernel_addr)
2836                 *dma_handle += HOST_PHYS_BASE;
2837
2838         return kernel_addr;
2839 }
2840
2841 static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
2842                                         void *cpu_addr, dma_addr_t dma_handle)
2843 {
2844         /* Cancel the device's base physical address of host memory */
2845         dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
2846
2847         dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
2848 }
2849
2850 void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
2851                                 dma_addr_t *dma_handle, u16 *queue_len)
2852 {
2853         void *base;
2854         u32 offset;
2855
2856         *dma_handle = hdev->asic_prop.sram_base_address;
2857
2858         base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
2859
2860         switch (queue_id) {
2861         case GOYA_QUEUE_ID_MME:
2862                 offset = MME_QMAN_BASE_OFFSET;
2863                 *queue_len = MME_QMAN_LENGTH;
2864                 break;
2865         case GOYA_QUEUE_ID_TPC0:
2866                 offset = TPC0_QMAN_BASE_OFFSET;
2867                 *queue_len = TPC_QMAN_LENGTH;
2868                 break;
2869         case GOYA_QUEUE_ID_TPC1:
2870                 offset = TPC1_QMAN_BASE_OFFSET;
2871                 *queue_len = TPC_QMAN_LENGTH;
2872                 break;
2873         case GOYA_QUEUE_ID_TPC2:
2874                 offset = TPC2_QMAN_BASE_OFFSET;
2875                 *queue_len = TPC_QMAN_LENGTH;
2876                 break;
2877         case GOYA_QUEUE_ID_TPC3:
2878                 offset = TPC3_QMAN_BASE_OFFSET;
2879                 *queue_len = TPC_QMAN_LENGTH;
2880                 break;
2881         case GOYA_QUEUE_ID_TPC4:
2882                 offset = TPC4_QMAN_BASE_OFFSET;
2883                 *queue_len = TPC_QMAN_LENGTH;
2884                 break;
2885         case GOYA_QUEUE_ID_TPC5:
2886                 offset = TPC5_QMAN_BASE_OFFSET;
2887                 *queue_len = TPC_QMAN_LENGTH;
2888                 break;
2889         case GOYA_QUEUE_ID_TPC6:
2890                 offset = TPC6_QMAN_BASE_OFFSET;
2891                 *queue_len = TPC_QMAN_LENGTH;
2892                 break;
2893         case GOYA_QUEUE_ID_TPC7:
2894                 offset = TPC7_QMAN_BASE_OFFSET;
2895                 *queue_len = TPC_QMAN_LENGTH;
2896                 break;
2897         default:
2898                 dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
2899                 return NULL;
2900         }
2901
2902         base += offset;
2903         *dma_handle += offset;
2904
2905         return base;
2906 }
2907
2908 static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
2909 {
2910         struct packet_msg_prot *fence_pkt;
2911         u32 *fence_ptr;
2912         dma_addr_t fence_dma_addr;
2913         struct hl_cb *cb;
2914         u32 tmp, timeout;
2915         int rc;
2916
2917         if (hdev->pldm)
2918                 timeout = GOYA_PLDM_QMAN0_TIMEOUT_USEC;
2919         else
2920                 timeout = HL_DEVICE_TIMEOUT_USEC;
2921
2922         if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
2923                 dev_err_ratelimited(hdev->dev,
2924                         "Can't send driver job on QMAN0 because the device is not idle\n");
2925                 return -EBUSY;
2926         }
2927
2928         fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
2929                                                         &fence_dma_addr);
2930         if (!fence_ptr) {
2931                 dev_err(hdev->dev,
2932                         "Failed to allocate fence memory for QMAN0\n");
2933                 return -ENOMEM;
2934         }
2935
2936         goya_qman0_set_security(hdev, true);
2937
2938         cb = job->patched_cb;
2939
2940         fence_pkt = (struct packet_msg_prot *) (uintptr_t) (cb->kernel_address +
2941                         job->job_cb_size - sizeof(struct packet_msg_prot));
2942
2943         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
2944                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
2945                         (1 << GOYA_PKT_CTL_MB_SHIFT);
2946         fence_pkt->ctl = cpu_to_le32(tmp);
2947         fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL);
2948         fence_pkt->addr = cpu_to_le64(fence_dma_addr);
2949
2950         rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
2951                                         job->job_cb_size, cb->bus_address);
2952         if (rc) {
2953                 dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
2954                 goto free_fence_ptr;
2955         }
2956
2957         rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
2958                                 (tmp == GOYA_QMAN0_FENCE_VAL), 1000,
2959                                 timeout, true);
2960
2961         hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0);
2962
2963         if (rc == -ETIMEDOUT) {
2964                 dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
2965                 goto free_fence_ptr;
2966         }
2967
2968 free_fence_ptr:
2969         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
2970                                         fence_dma_addr);
2971
2972         goya_qman0_set_security(hdev, false);
2973
2974         return rc;
2975 }
2976
2977 int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
2978                                 u32 timeout, long *result)
2979 {
2980         struct goya_device *goya = hdev->asic_specific;
2981
2982         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
2983                 if (result)
2984                         *result = 0;
2985                 return 0;
2986         }
2987
2988         return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
2989                                         timeout, result);
2990 }
2991
2992 int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
2993 {
2994         struct packet_msg_prot *fence_pkt;
2995         dma_addr_t pkt_dma_addr;
2996         u32 fence_val, tmp;
2997         dma_addr_t fence_dma_addr;
2998         u32 *fence_ptr;
2999         int rc;
3000
3001         fence_val = GOYA_QMAN0_FENCE_VAL;
3002
3003         fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
3004                                                         &fence_dma_addr);
3005         if (!fence_ptr) {
3006                 dev_err(hdev->dev,
3007                         "Failed to allocate memory for queue testing\n");
3008                 return -ENOMEM;
3009         }
3010
3011         *fence_ptr = 0;
3012
3013         fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
3014                                         sizeof(struct packet_msg_prot),
3015                                         GFP_KERNEL, &pkt_dma_addr);
3016         if (!fence_pkt) {
3017                 dev_err(hdev->dev,
3018                         "Failed to allocate packet for queue testing\n");
3019                 rc = -ENOMEM;
3020                 goto free_fence_ptr;
3021         }
3022
3023         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
3024                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
3025                         (1 << GOYA_PKT_CTL_MB_SHIFT);
3026         fence_pkt->ctl = cpu_to_le32(tmp);
3027         fence_pkt->value = cpu_to_le32(fence_val);
3028         fence_pkt->addr = cpu_to_le64(fence_dma_addr);
3029
3030         rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
3031                                         sizeof(struct packet_msg_prot),
3032                                         pkt_dma_addr);
3033         if (rc) {
3034                 dev_err(hdev->dev,
3035                         "Failed to send fence packet\n");
3036                 goto free_pkt;
3037         }
3038
3039         rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
3040                                         1000, GOYA_TEST_QUEUE_WAIT_USEC, true);
3041
3042         hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
3043
3044         if (rc == -ETIMEDOUT) {
3045                 dev_err(hdev->dev,
3046                         "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
3047                         hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
3048                 rc = -EIO;
3049         }
3050
3051 free_pkt:
3052         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
3053                                         pkt_dma_addr);
3054 free_fence_ptr:
3055         hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
3056                                         fence_dma_addr);
3057         return rc;
3058 }
3059
3060 int goya_test_cpu_queue(struct hl_device *hdev)
3061 {
3062         struct goya_device *goya = hdev->asic_specific;
3063
3064         /*
3065          * check capability here as send_cpu_message() won't update the result
3066          * value if no capability
3067          */
3068         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
3069                 return 0;
3070
3071         return hl_fw_test_cpu_queue(hdev);
3072 }
3073
3074 int goya_test_queues(struct hl_device *hdev)
3075 {
3076         int i, rc, ret_val = 0;
3077
3078         for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
3079                 rc = goya_test_queue(hdev, i);
3080                 if (rc)
3081                         ret_val = -EINVAL;
3082         }
3083
3084         return ret_val;
3085 }
3086
3087 static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
3088                                         gfp_t mem_flags, dma_addr_t *dma_handle)
3089 {
3090         void *kernel_addr;
3091
3092         if (size > GOYA_DMA_POOL_BLK_SIZE)
3093                 return NULL;
3094
3095         kernel_addr =  dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
3096
3097         /* Shift to the device's base physical address of host memory */
3098         if (kernel_addr)
3099                 *dma_handle += HOST_PHYS_BASE;
3100
3101         return kernel_addr;
3102 }
3103
3104 static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
3105                                 dma_addr_t dma_addr)
3106 {
3107         /* Cancel the device's base physical address of host memory */
3108         dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
3109
3110         dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
3111 }
3112
3113 void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
3114                                         dma_addr_t *dma_handle)
3115 {
3116         void *vaddr;
3117
3118         vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
3119         *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address +
3120                         VA_CPU_ACCESSIBLE_MEM_ADDR;
3121
3122         return vaddr;
3123 }
3124
3125 void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
3126                                         void *vaddr)
3127 {
3128         hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
3129 }
3130
3131 static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
3132                                 int nents, enum dma_data_direction dir)
3133 {
3134         struct scatterlist *sg;
3135         int i;
3136
3137         if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
3138                 return -ENOMEM;
3139
3140         /* Shift to the device's base physical address of host memory */
3141         for_each_sg(sgl, sg, nents, i)
3142                 sg->dma_address += HOST_PHYS_BASE;
3143
3144         return 0;
3145 }
3146
3147 static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
3148                                 int nents, enum dma_data_direction dir)
3149 {
3150         struct scatterlist *sg;
3151         int i;
3152
3153         /* Cancel the device's base physical address of host memory */
3154         for_each_sg(sgl, sg, nents, i)
3155                 sg->dma_address -= HOST_PHYS_BASE;
3156
3157         dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
3158 }
3159
3160 u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
3161 {
3162         struct scatterlist *sg, *sg_next_iter;
3163         u32 count, dma_desc_cnt;
3164         u64 len, len_next;
3165         dma_addr_t addr, addr_next;
3166
3167         dma_desc_cnt = 0;
3168
3169         for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3170
3171                 len = sg_dma_len(sg);
3172                 addr = sg_dma_address(sg);
3173
3174                 if (len == 0)
3175                         break;
3176
3177                 while ((count + 1) < sgt->nents) {
3178                         sg_next_iter = sg_next(sg);
3179                         len_next = sg_dma_len(sg_next_iter);
3180                         addr_next = sg_dma_address(sg_next_iter);
3181
3182                         if (len_next == 0)
3183                                 break;
3184
3185                         if ((addr + len == addr_next) &&
3186                                 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3187                                 len += len_next;
3188                                 count++;
3189                                 sg = sg_next_iter;
3190                         } else {
3191                                 break;
3192                         }
3193                 }
3194
3195                 dma_desc_cnt++;
3196         }
3197
3198         return dma_desc_cnt * sizeof(struct packet_lin_dma);
3199 }
3200
3201 static int goya_pin_memory_before_cs(struct hl_device *hdev,
3202                                 struct hl_cs_parser *parser,
3203                                 struct packet_lin_dma *user_dma_pkt,
3204                                 u64 addr, enum dma_data_direction dir)
3205 {
3206         struct hl_userptr *userptr;
3207         int rc;
3208
3209         if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3210                         parser->job_userptr_list, &userptr))
3211                 goto already_pinned;
3212
3213         userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
3214         if (!userptr)
3215                 return -ENOMEM;
3216
3217         rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
3218                                 userptr);
3219         if (rc)
3220                 goto free_userptr;
3221
3222         list_add_tail(&userptr->job_node, parser->job_userptr_list);
3223
3224         rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
3225                                         userptr->sgt->nents, dir);
3226         if (rc) {
3227                 dev_err(hdev->dev, "failed to map sgt with DMA region\n");
3228                 goto unpin_memory;
3229         }
3230
3231         userptr->dma_mapped = true;
3232         userptr->dir = dir;
3233
3234 already_pinned:
3235         parser->patched_cb_size +=
3236                         goya_get_dma_desc_list_size(hdev, userptr->sgt);
3237
3238         return 0;
3239
3240 unpin_memory:
3241         hl_unpin_host_memory(hdev, userptr);
3242 free_userptr:
3243         kfree(userptr);
3244         return rc;
3245 }
3246
3247 static int goya_validate_dma_pkt_host(struct hl_device *hdev,
3248                                 struct hl_cs_parser *parser,
3249                                 struct packet_lin_dma *user_dma_pkt)
3250 {
3251         u64 device_memory_addr, addr;
3252         enum dma_data_direction dir;
3253         enum goya_dma_direction user_dir;
3254         bool sram_addr = true;
3255         bool skip_host_mem_pin = false;
3256         bool user_memset;
3257         u32 ctl;
3258         int rc = 0;
3259
3260         ctl = le32_to_cpu(user_dma_pkt->ctl);
3261
3262         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3263                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3264
3265         user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3266                         GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3267
3268         switch (user_dir) {
3269         case DMA_HOST_TO_DRAM:
3270                 dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n");
3271                 dir = DMA_TO_DEVICE;
3272                 sram_addr = false;
3273                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3274                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3275                 if (user_memset)
3276                         skip_host_mem_pin = true;
3277                 break;
3278
3279         case DMA_DRAM_TO_HOST:
3280                 dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n");
3281                 dir = DMA_FROM_DEVICE;
3282                 sram_addr = false;
3283                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3284                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3285                 break;
3286
3287         case DMA_HOST_TO_SRAM:
3288                 dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n");
3289                 dir = DMA_TO_DEVICE;
3290                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3291                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3292                 if (user_memset)
3293                         skip_host_mem_pin = true;
3294                 break;
3295
3296         case DMA_SRAM_TO_HOST:
3297                 dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n");
3298                 dir = DMA_FROM_DEVICE;
3299                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3300                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3301                 break;
3302         default:
3303                 dev_err(hdev->dev, "DMA direction is undefined\n");
3304                 return -EFAULT;
3305         }
3306
3307         if (sram_addr) {
3308                 if (!hl_mem_area_inside_range(device_memory_addr,
3309                                 le32_to_cpu(user_dma_pkt->tsize),
3310                                 hdev->asic_prop.sram_user_base_address,
3311                                 hdev->asic_prop.sram_end_address)) {
3312
3313                         dev_err(hdev->dev,
3314                                 "SRAM address 0x%llx + 0x%x is invalid\n",
3315                                 device_memory_addr,
3316                                 user_dma_pkt->tsize);
3317                         return -EFAULT;
3318                 }
3319         } else {
3320                 if (!hl_mem_area_inside_range(device_memory_addr,
3321                                 le32_to_cpu(user_dma_pkt->tsize),
3322                                 hdev->asic_prop.dram_user_base_address,
3323                                 hdev->asic_prop.dram_end_address)) {
3324
3325                         dev_err(hdev->dev,
3326                                 "DRAM address 0x%llx + 0x%x is invalid\n",
3327                                 device_memory_addr,
3328                                 user_dma_pkt->tsize);
3329                         return -EFAULT;
3330                 }
3331         }
3332
3333         if (skip_host_mem_pin)
3334                 parser->patched_cb_size += sizeof(*user_dma_pkt);
3335         else {
3336                 if ((dir == DMA_TO_DEVICE) &&
3337                                 (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) {
3338                         dev_err(hdev->dev,
3339                                 "Can't DMA from host on queue other then 1\n");
3340                         return -EFAULT;
3341                 }
3342
3343                 rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt,
3344                                                 addr, dir);
3345         }
3346
3347         return rc;
3348 }
3349
3350 static int goya_validate_dma_pkt_no_host(struct hl_device *hdev,
3351                                 struct hl_cs_parser *parser,
3352                                 struct packet_lin_dma *user_dma_pkt)
3353 {
3354         u64 sram_memory_addr, dram_memory_addr;
3355         enum goya_dma_direction user_dir;
3356         u32 ctl;
3357
3358         ctl = le32_to_cpu(user_dma_pkt->ctl);
3359         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3360                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3361
3362         if (user_dir == DMA_DRAM_TO_SRAM) {
3363                 dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n");
3364                 dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3365                 sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3366         } else {
3367                 dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n");
3368                 sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3369                 dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3370         }
3371
3372         if (!hl_mem_area_inside_range(sram_memory_addr,
3373                                 le32_to_cpu(user_dma_pkt->tsize),
3374                                 hdev->asic_prop.sram_user_base_address,
3375                                 hdev->asic_prop.sram_end_address)) {
3376                 dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n",
3377                         sram_memory_addr, user_dma_pkt->tsize);
3378                 return -EFAULT;
3379         }
3380
3381         if (!hl_mem_area_inside_range(dram_memory_addr,
3382                                 le32_to_cpu(user_dma_pkt->tsize),
3383                                 hdev->asic_prop.dram_user_base_address,
3384                                 hdev->asic_prop.dram_end_address)) {
3385                 dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n",
3386                         dram_memory_addr, user_dma_pkt->tsize);
3387                 return -EFAULT;
3388         }
3389
3390         parser->patched_cb_size += sizeof(*user_dma_pkt);
3391
3392         return 0;
3393 }
3394
3395 static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev,
3396                                 struct hl_cs_parser *parser,
3397                                 struct packet_lin_dma *user_dma_pkt)
3398 {
3399         enum goya_dma_direction user_dir;
3400         u32 ctl;
3401         int rc;
3402
3403         dev_dbg(hdev->dev, "DMA packet details:\n");
3404         dev_dbg(hdev->dev, "source == 0x%llx\n",
3405                 le64_to_cpu(user_dma_pkt->src_addr));
3406         dev_dbg(hdev->dev, "destination == 0x%llx\n",
3407                 le64_to_cpu(user_dma_pkt->dst_addr));
3408         dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3409
3410         ctl = le32_to_cpu(user_dma_pkt->ctl);
3411         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3412                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3413
3414         /*
3415          * Special handling for DMA with size 0. The H/W has a bug where
3416          * this can cause the QMAN DMA to get stuck, so block it here.
3417          */
3418         if (user_dma_pkt->tsize == 0) {
3419                 dev_err(hdev->dev,
3420                         "Got DMA with size 0, might reset the device\n");
3421                 return -EINVAL;
3422         }
3423
3424         if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM))
3425                 rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt);
3426         else
3427                 rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt);
3428
3429         return rc;
3430 }
3431
3432 static int goya_validate_dma_pkt_mmu(struct hl_device *hdev,
3433                                 struct hl_cs_parser *parser,
3434                                 struct packet_lin_dma *user_dma_pkt)
3435 {
3436         dev_dbg(hdev->dev, "DMA packet details:\n");
3437         dev_dbg(hdev->dev, "source == 0x%llx\n",
3438                 le64_to_cpu(user_dma_pkt->src_addr));
3439         dev_dbg(hdev->dev, "destination == 0x%llx\n",
3440                 le64_to_cpu(user_dma_pkt->dst_addr));
3441         dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
3442
3443         /*
3444          * WA for HW-23.
3445          * We can't allow user to read from Host using QMANs other than 1.
3446          */
3447         if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 &&
3448                 hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr),
3449                                 le32_to_cpu(user_dma_pkt->tsize),
3450                                 hdev->asic_prop.va_space_host_start_address,
3451                                 hdev->asic_prop.va_space_host_end_address)) {
3452                 dev_err(hdev->dev,
3453                         "Can't DMA from host on queue other then 1\n");
3454                 return -EFAULT;
3455         }
3456
3457         if (user_dma_pkt->tsize == 0) {
3458                 dev_err(hdev->dev,
3459                         "Got DMA with size 0, might reset the device\n");
3460                 return -EINVAL;
3461         }
3462
3463         parser->patched_cb_size += sizeof(*user_dma_pkt);
3464
3465         return 0;
3466 }
3467
3468 static int goya_validate_wreg32(struct hl_device *hdev,
3469                                 struct hl_cs_parser *parser,
3470                                 struct packet_wreg32 *wreg_pkt)
3471 {
3472         struct goya_device *goya = hdev->asic_specific;
3473         u32 sob_start_addr, sob_end_addr;
3474         u16 reg_offset;
3475
3476         reg_offset = le32_to_cpu(wreg_pkt->ctl) &
3477                         GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK;
3478
3479         dev_dbg(hdev->dev, "WREG32 packet details:\n");
3480         dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset);
3481         dev_dbg(hdev->dev, "value      == 0x%x\n",
3482                 le32_to_cpu(wreg_pkt->value));
3483
3484         if (reg_offset != (mmDMA_CH_0_WR_COMP_ADDR_LO & 0x1FFF)) {
3485                 dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n",
3486                         reg_offset);
3487                 return -EPERM;
3488         }
3489
3490         /*
3491          * With MMU, DMA channels are not secured, so it doesn't matter where
3492          * the WR COMP will be written to because it will go out with
3493          * non-secured property
3494          */
3495         if (goya->hw_cap_initialized & HW_CAP_MMU)
3496                 return 0;
3497
3498         sob_start_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
3499         sob_end_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1023);
3500
3501         if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) ||
3502                         (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) {
3503
3504                 dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n",
3505                         wreg_pkt->value);
3506                 return -EPERM;
3507         }
3508
3509         return 0;
3510 }
3511
3512 static int goya_validate_cb(struct hl_device *hdev,
3513                         struct hl_cs_parser *parser, bool is_mmu)
3514 {
3515         u32 cb_parsed_length = 0;
3516         int rc = 0;
3517
3518         parser->patched_cb_size = 0;
3519
3520         /* cb_user_size is more than 0 so loop will always be executed */
3521         while (cb_parsed_length < parser->user_cb_size) {
3522                 enum packet_id pkt_id;
3523                 u16 pkt_size;
3524                 struct goya_packet *user_pkt;
3525
3526                 user_pkt = (struct goya_packet *) (uintptr_t)
3527                         (parser->user_cb->kernel_address + cb_parsed_length);
3528
3529                 pkt_id = (enum packet_id) (
3530                                 (le64_to_cpu(user_pkt->header) &
3531                                 PACKET_HEADER_PACKET_ID_MASK) >>
3532                                         PACKET_HEADER_PACKET_ID_SHIFT);
3533
3534                 pkt_size = goya_packet_sizes[pkt_id];
3535                 cb_parsed_length += pkt_size;
3536                 if (cb_parsed_length > parser->user_cb_size) {
3537                         dev_err(hdev->dev,
3538                                 "packet 0x%x is out of CB boundary\n", pkt_id);
3539                         rc = -EINVAL;
3540                         break;
3541                 }
3542
3543                 switch (pkt_id) {
3544                 case PACKET_WREG_32:
3545                         /*
3546                          * Although it is validated after copy in patch_cb(),
3547                          * need to validate here as well because patch_cb() is
3548                          * not called in MMU path while this function is called
3549                          */
3550                         rc = goya_validate_wreg32(hdev,
3551                                 parser, (struct packet_wreg32 *) user_pkt);
3552                         break;
3553
3554                 case PACKET_WREG_BULK:
3555                         dev_err(hdev->dev,
3556                                 "User not allowed to use WREG_BULK\n");
3557                         rc = -EPERM;
3558                         break;
3559
3560                 case PACKET_MSG_PROT:
3561                         dev_err(hdev->dev,
3562                                 "User not allowed to use MSG_PROT\n");
3563                         rc = -EPERM;
3564                         break;
3565
3566                 case PACKET_CP_DMA:
3567                         dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3568                         rc = -EPERM;
3569                         break;
3570
3571                 case PACKET_STOP:
3572                         dev_err(hdev->dev, "User not allowed to use STOP\n");
3573                         rc = -EPERM;
3574                         break;
3575
3576                 case PACKET_LIN_DMA:
3577                         if (is_mmu)
3578                                 rc = goya_validate_dma_pkt_mmu(hdev, parser,
3579                                         (struct packet_lin_dma *) user_pkt);
3580                         else
3581                                 rc = goya_validate_dma_pkt_no_mmu(hdev, parser,
3582                                         (struct packet_lin_dma *) user_pkt);
3583                         break;
3584
3585                 case PACKET_MSG_LONG:
3586                 case PACKET_MSG_SHORT:
3587                 case PACKET_FENCE:
3588                 case PACKET_NOP:
3589                         parser->patched_cb_size += pkt_size;
3590                         break;
3591
3592                 default:
3593                         dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3594                                 pkt_id);
3595                         rc = -EINVAL;
3596                         break;
3597                 }
3598
3599                 if (rc)
3600                         break;
3601         }
3602
3603         /*
3604          * The new CB should have space at the end for two MSG_PROT packets:
3605          * 1. A packet that will act as a completion packet
3606          * 2. A packet that will generate MSI-X interrupt
3607          */
3608         parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
3609
3610         return rc;
3611 }
3612
3613 static int goya_patch_dma_packet(struct hl_device *hdev,
3614                                 struct hl_cs_parser *parser,
3615                                 struct packet_lin_dma *user_dma_pkt,
3616                                 struct packet_lin_dma *new_dma_pkt,
3617                                 u32 *new_dma_pkt_size)
3618 {
3619         struct hl_userptr *userptr;
3620         struct scatterlist *sg, *sg_next_iter;
3621         u32 count, dma_desc_cnt;
3622         u64 len, len_next;
3623         dma_addr_t dma_addr, dma_addr_next;
3624         enum goya_dma_direction user_dir;
3625         u64 device_memory_addr, addr;
3626         enum dma_data_direction dir;
3627         struct sg_table *sgt;
3628         bool skip_host_mem_pin = false;
3629         bool user_memset;
3630         u32 user_rdcomp_mask, user_wrcomp_mask, ctl;
3631
3632         ctl = le32_to_cpu(user_dma_pkt->ctl);
3633
3634         user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
3635                         GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
3636
3637         user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
3638                         GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
3639
3640         if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM) ||
3641                         (user_dma_pkt->tsize == 0)) {
3642                 memcpy(new_dma_pkt, user_dma_pkt, sizeof(*new_dma_pkt));
3643                 *new_dma_pkt_size = sizeof(*new_dma_pkt);
3644                 return 0;
3645         }
3646
3647         if ((user_dir == DMA_HOST_TO_DRAM) || (user_dir == DMA_HOST_TO_SRAM)) {
3648                 addr = le64_to_cpu(user_dma_pkt->src_addr);
3649                 device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
3650                 dir = DMA_TO_DEVICE;
3651                 if (user_memset)
3652                         skip_host_mem_pin = true;
3653         } else {
3654                 addr = le64_to_cpu(user_dma_pkt->dst_addr);
3655                 device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
3656                 dir = DMA_FROM_DEVICE;
3657         }
3658
3659         if ((!skip_host_mem_pin) &&
3660                 (hl_userptr_is_pinned(hdev, addr,
3661                         le32_to_cpu(user_dma_pkt->tsize),
3662                         parser->job_userptr_list, &userptr) == false)) {
3663                 dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
3664                                 addr, user_dma_pkt->tsize);
3665                 return -EFAULT;
3666         }
3667
3668         if ((user_memset) && (dir == DMA_TO_DEVICE)) {
3669                 memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
3670                 *new_dma_pkt_size = sizeof(*user_dma_pkt);
3671                 return 0;
3672         }
3673
3674         user_rdcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK;
3675
3676         user_wrcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK;
3677
3678         sgt = userptr->sgt;
3679         dma_desc_cnt = 0;
3680
3681         for_each_sg(sgt->sgl, sg, sgt->nents, count) {
3682                 len = sg_dma_len(sg);
3683                 dma_addr = sg_dma_address(sg);
3684
3685                 if (len == 0)
3686                         break;
3687
3688                 while ((count + 1) < sgt->nents) {
3689                         sg_next_iter = sg_next(sg);
3690                         len_next = sg_dma_len(sg_next_iter);
3691                         dma_addr_next = sg_dma_address(sg_next_iter);
3692
3693                         if (len_next == 0)
3694                                 break;
3695
3696                         if ((dma_addr + len == dma_addr_next) &&
3697                                 (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
3698                                 len += len_next;
3699                                 count++;
3700                                 sg = sg_next_iter;
3701                         } else {
3702                                 break;
3703                         }
3704                 }
3705
3706                 ctl = le32_to_cpu(user_dma_pkt->ctl);
3707                 if (likely(dma_desc_cnt))
3708                         ctl &= ~GOYA_PKT_CTL_EB_MASK;
3709                 ctl &= ~(GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK |
3710                                 GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK);
3711                 new_dma_pkt->ctl = cpu_to_le32(ctl);
3712                 new_dma_pkt->tsize = cpu_to_le32((u32) len);
3713
3714                 if (dir == DMA_TO_DEVICE) {
3715                         new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
3716                         new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
3717                 } else {
3718                         new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
3719                         new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
3720                 }
3721
3722                 if (!user_memset)
3723                         device_memory_addr += len;
3724                 dma_desc_cnt++;
3725                 new_dma_pkt++;
3726         }
3727
3728         if (!dma_desc_cnt) {
3729                 dev_err(hdev->dev,
3730                         "Error of 0 SG entries when patching DMA packet\n");
3731                 return -EFAULT;
3732         }
3733
3734         /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */
3735         new_dma_pkt--;
3736         new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask);
3737
3738         *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
3739
3740         return 0;
3741 }
3742
3743 static int goya_patch_cb(struct hl_device *hdev,
3744                                 struct hl_cs_parser *parser)
3745 {
3746         u32 cb_parsed_length = 0;
3747         u32 cb_patched_cur_length = 0;
3748         int rc = 0;
3749
3750         /* cb_user_size is more than 0 so loop will always be executed */
3751         while (cb_parsed_length < parser->user_cb_size) {
3752                 enum packet_id pkt_id;
3753                 u16 pkt_size;
3754                 u32 new_pkt_size = 0;
3755                 struct goya_packet *user_pkt, *kernel_pkt;
3756
3757                 user_pkt = (struct goya_packet *) (uintptr_t)
3758                         (parser->user_cb->kernel_address + cb_parsed_length);
3759                 kernel_pkt = (struct goya_packet *) (uintptr_t)
3760                         (parser->patched_cb->kernel_address +
3761                                         cb_patched_cur_length);
3762
3763                 pkt_id = (enum packet_id) (
3764                                 (le64_to_cpu(user_pkt->header) &
3765                                 PACKET_HEADER_PACKET_ID_MASK) >>
3766                                         PACKET_HEADER_PACKET_ID_SHIFT);
3767
3768                 pkt_size = goya_packet_sizes[pkt_id];
3769                 cb_parsed_length += pkt_size;
3770                 if (cb_parsed_length > parser->user_cb_size) {
3771                         dev_err(hdev->dev,
3772                                 "packet 0x%x is out of CB boundary\n", pkt_id);
3773                         rc = -EINVAL;
3774                         break;
3775                 }
3776
3777                 switch (pkt_id) {
3778                 case PACKET_LIN_DMA:
3779                         rc = goya_patch_dma_packet(hdev, parser,
3780                                         (struct packet_lin_dma *) user_pkt,
3781                                         (struct packet_lin_dma *) kernel_pkt,
3782                                         &new_pkt_size);
3783                         cb_patched_cur_length += new_pkt_size;
3784                         break;
3785
3786                 case PACKET_WREG_32:
3787                         memcpy(kernel_pkt, user_pkt, pkt_size);
3788                         cb_patched_cur_length += pkt_size;
3789                         rc = goya_validate_wreg32(hdev, parser,
3790                                         (struct packet_wreg32 *) kernel_pkt);
3791                         break;
3792
3793                 case PACKET_WREG_BULK:
3794                         dev_err(hdev->dev,
3795                                 "User not allowed to use WREG_BULK\n");
3796                         rc = -EPERM;
3797                         break;
3798
3799                 case PACKET_MSG_PROT:
3800                         dev_err(hdev->dev,
3801                                 "User not allowed to use MSG_PROT\n");
3802                         rc = -EPERM;
3803                         break;
3804
3805                 case PACKET_CP_DMA:
3806                         dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
3807                         rc = -EPERM;
3808                         break;
3809
3810                 case PACKET_STOP:
3811                         dev_err(hdev->dev, "User not allowed to use STOP\n");
3812                         rc = -EPERM;
3813                         break;
3814
3815                 case PACKET_MSG_LONG:
3816                 case PACKET_MSG_SHORT:
3817                 case PACKET_FENCE:
3818                 case PACKET_NOP:
3819                         memcpy(kernel_pkt, user_pkt, pkt_size);
3820                         cb_patched_cur_length += pkt_size;
3821                         break;
3822
3823                 default:
3824                         dev_err(hdev->dev, "Invalid packet header 0x%x\n",
3825                                 pkt_id);
3826                         rc = -EINVAL;
3827                         break;
3828                 }
3829
3830                 if (rc)
3831                         break;
3832         }
3833
3834         return rc;
3835 }
3836
3837 static int goya_parse_cb_mmu(struct hl_device *hdev,
3838                 struct hl_cs_parser *parser)
3839 {
3840         u64 patched_cb_handle;
3841         u32 patched_cb_size;
3842         struct hl_cb *user_cb;
3843         int rc;
3844
3845         /*
3846          * The new CB should have space at the end for two MSG_PROT pkt:
3847          * 1. A packet that will act as a completion packet
3848          * 2. A packet that will generate MSI-X interrupt
3849          */
3850         parser->patched_cb_size = parser->user_cb_size +
3851                         sizeof(struct packet_msg_prot) * 2;
3852
3853         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
3854                                 parser->patched_cb_size,
3855                                 &patched_cb_handle, HL_KERNEL_ASID_ID);
3856
3857         if (rc) {
3858                 dev_err(hdev->dev,
3859                         "Failed to allocate patched CB for DMA CS %d\n",
3860                         rc);
3861                 return rc;
3862         }
3863
3864         patched_cb_handle >>= PAGE_SHIFT;
3865         parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3866                                 (u32) patched_cb_handle);
3867         /* hl_cb_get should never fail here so use kernel WARN */
3868         WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
3869                         (u32) patched_cb_handle);
3870         if (!parser->patched_cb) {
3871                 rc = -EFAULT;
3872                 goto out;
3873         }
3874
3875         /*
3876          * The check that parser->user_cb_size <= parser->user_cb->size was done
3877          * in validate_queue_index().
3878          */
3879         memcpy((void *) (uintptr_t) parser->patched_cb->kernel_address,
3880                 (void *) (uintptr_t) parser->user_cb->kernel_address,
3881                 parser->user_cb_size);
3882
3883         patched_cb_size = parser->patched_cb_size;
3884
3885         /* validate patched CB instead of user CB */
3886         user_cb = parser->user_cb;
3887         parser->user_cb = parser->patched_cb;
3888         rc = goya_validate_cb(hdev, parser, true);
3889         parser->user_cb = user_cb;
3890
3891         if (rc) {
3892                 hl_cb_put(parser->patched_cb);
3893                 goto out;
3894         }
3895
3896         if (patched_cb_size != parser->patched_cb_size) {
3897                 dev_err(hdev->dev, "user CB size mismatch\n");
3898                 hl_cb_put(parser->patched_cb);
3899                 rc = -EINVAL;
3900                 goto out;
3901         }
3902
3903 out:
3904         /*
3905          * Always call cb destroy here because we still have 1 reference
3906          * to it by calling cb_get earlier. After the job will be completed,
3907          * cb_put will release it, but here we want to remove it from the
3908          * idr
3909          */
3910         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3911                                         patched_cb_handle << PAGE_SHIFT);
3912
3913         return rc;
3914 }
3915
3916 static int goya_parse_cb_no_mmu(struct hl_device *hdev,
3917                                 struct hl_cs_parser *parser)
3918 {
3919         u64 patched_cb_handle;
3920         int rc;
3921
3922         rc = goya_validate_cb(hdev, parser, false);
3923
3924         if (rc)
3925                 goto free_userptr;
3926
3927         rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr,
3928                                 parser->patched_cb_size,
3929                                 &patched_cb_handle, HL_KERNEL_ASID_ID);
3930         if (rc) {
3931                 dev_err(hdev->dev,
3932                         "Failed to allocate patched CB for DMA CS %d\n", rc);
3933                 goto free_userptr;
3934         }
3935
3936         patched_cb_handle >>= PAGE_SHIFT;
3937         parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
3938                                 (u32) patched_cb_handle);
3939         /* hl_cb_get should never fail here so use kernel WARN */
3940         WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
3941                         (u32) patched_cb_handle);
3942         if (!parser->patched_cb) {
3943                 rc = -EFAULT;
3944                 goto out;
3945         }
3946
3947         rc = goya_patch_cb(hdev, parser);
3948
3949         if (rc)
3950                 hl_cb_put(parser->patched_cb);
3951
3952 out:
3953         /*
3954          * Always call cb destroy here because we still have 1 reference
3955          * to it by calling cb_get earlier. After the job will be completed,
3956          * cb_put will release it, but here we want to remove it from the
3957          * idr
3958          */
3959         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
3960                                 patched_cb_handle << PAGE_SHIFT);
3961
3962 free_userptr:
3963         if (rc)
3964                 hl_userptr_delete_list(hdev, parser->job_userptr_list);
3965         return rc;
3966 }
3967
3968 static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
3969                                         struct hl_cs_parser *parser)
3970 {
3971         struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
3972         struct goya_device *goya = hdev->asic_specific;
3973
3974         if (goya->hw_cap_initialized & HW_CAP_MMU)
3975                 return 0;
3976
3977         /* For internal queue jobs, just check if CB address is valid */
3978         if (hl_mem_area_inside_range(
3979                         (u64) (uintptr_t) parser->user_cb,
3980                         parser->user_cb_size,
3981                         asic_prop->sram_user_base_address,
3982                         asic_prop->sram_end_address))
3983                 return 0;
3984
3985         if (hl_mem_area_inside_range(
3986                         (u64) (uintptr_t) parser->user_cb,
3987                         parser->user_cb_size,
3988                         asic_prop->dram_user_base_address,
3989                         asic_prop->dram_end_address))
3990                 return 0;
3991
3992         dev_err(hdev->dev,
3993                 "Internal CB address 0x%px + 0x%x is not in SRAM nor in DRAM\n",
3994                 parser->user_cb, parser->user_cb_size);
3995
3996         return -EFAULT;
3997 }
3998
3999 int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
4000 {
4001         struct goya_device *goya = hdev->asic_specific;
4002
4003         if (parser->queue_type == QUEUE_TYPE_INT)
4004                 return goya_parse_cb_no_ext_queue(hdev, parser);
4005
4006         if (goya->hw_cap_initialized & HW_CAP_MMU)
4007                 return goya_parse_cb_mmu(hdev, parser);
4008         else
4009                 return goya_parse_cb_no_mmu(hdev, parser);
4010 }
4011
4012 void goya_add_end_of_cb_packets(struct hl_device *hdev, u64 kernel_address,
4013                                 u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec)
4014 {
4015         struct packet_msg_prot *cq_pkt;
4016         u32 tmp;
4017
4018         cq_pkt = (struct packet_msg_prot *) (uintptr_t)
4019                 (kernel_address + len - (sizeof(struct packet_msg_prot) * 2));
4020
4021         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
4022                         (1 << GOYA_PKT_CTL_EB_SHIFT) |
4023                         (1 << GOYA_PKT_CTL_MB_SHIFT);
4024         cq_pkt->ctl = cpu_to_le32(tmp);
4025         cq_pkt->value = cpu_to_le32(cq_val);
4026         cq_pkt->addr = cpu_to_le64(cq_addr);
4027
4028         cq_pkt++;
4029
4030         tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
4031                         (1 << GOYA_PKT_CTL_MB_SHIFT);
4032         cq_pkt->ctl = cpu_to_le32(tmp);
4033         cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF);
4034         cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF);
4035 }
4036
4037 void goya_update_eq_ci(struct hl_device *hdev, u32 val)
4038 {
4039         WREG32(mmCPU_EQ_CI, val);
4040 }
4041
4042 void goya_restore_phase_topology(struct hl_device *hdev)
4043 {
4044
4045 }
4046
4047 static void goya_clear_sm_regs(struct hl_device *hdev)
4048 {
4049         int i, num_of_sob_in_longs, num_of_mon_in_longs;
4050
4051         num_of_sob_in_longs =
4052                 ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4);
4053
4054         num_of_mon_in_longs =
4055                 ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4);
4056
4057         for (i = 0 ; i < num_of_sob_in_longs ; i += 4)
4058                 WREG32(mmSYNC_MNGR_SOB_OBJ_0 + i, 0);
4059
4060         for (i = 0 ; i < num_of_mon_in_longs ; i += 4)
4061                 WREG32(mmSYNC_MNGR_MON_STATUS_0 + i, 0);
4062
4063         /* Flush all WREG to prevent race */
4064         i = RREG32(mmSYNC_MNGR_SOB_OBJ_0);
4065 }
4066
4067 /*
4068  * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped
4069  *                       address.
4070  *
4071  * @hdev:       pointer to hl_device structure
4072  * @addr:       device or host mapped address
4073  * @val:        returned value
4074  *
4075  * In case of DDR address that is not mapped into the default aperture that
4076  * the DDR bar exposes, the function will configure the iATU so that the DDR
4077  * bar will be positioned at a base address that allows reading from the
4078  * required address. Configuring the iATU during normal operation can
4079  * lead to undefined behavior and therefore, should be done with extreme care
4080  *
4081  */
4082 static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
4083 {
4084         struct asic_fixed_properties *prop = &hdev->asic_prop;
4085         u64 ddr_bar_addr;
4086         int rc = 0;
4087
4088         if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
4089                 *val = RREG32(addr - CFG_BASE);
4090
4091         } else if ((addr >= SRAM_BASE_ADDR) &&
4092                         (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
4093
4094                 *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4095                                 (addr - SRAM_BASE_ADDR));
4096
4097         } else if ((addr >= DRAM_PHYS_BASE) &&
4098                         (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) {
4099
4100                 u64 bar_base_addr = DRAM_PHYS_BASE +
4101                                 (addr & ~(prop->dram_pci_bar_size - 0x1ull));
4102
4103                 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4104                 if (ddr_bar_addr != U64_MAX) {
4105                         *val = readl(hdev->pcie_bar[DDR_BAR_ID] +
4106                                                 (addr - bar_base_addr));
4107
4108                         ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4109                                                         ddr_bar_addr);
4110                 }
4111                 if (ddr_bar_addr == U64_MAX)
4112                         rc = -EIO;
4113
4114         } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
4115                 *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
4116
4117         } else {
4118                 rc = -EFAULT;
4119         }
4120
4121         return rc;
4122 }
4123
4124 /*
4125  * goya_debugfs_write32 - write a 32bit value to a given device or a host mapped
4126  *                        address.
4127  *
4128  * @hdev:       pointer to hl_device structure
4129  * @addr:       device or host mapped address
4130  * @val:        returned value
4131  *
4132  * In case of DDR address that is not mapped into the default aperture that
4133  * the DDR bar exposes, the function will configure the iATU so that the DDR
4134  * bar will be positioned at a base address that allows writing to the
4135  * required address. Configuring the iATU during normal operation can
4136  * lead to undefined behavior and therefore, should be done with extreme care
4137  *
4138  */
4139 static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
4140 {
4141         struct asic_fixed_properties *prop = &hdev->asic_prop;
4142         u64 ddr_bar_addr;
4143         int rc = 0;
4144
4145         if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
4146                 WREG32(addr - CFG_BASE, val);
4147
4148         } else if ((addr >= SRAM_BASE_ADDR) &&
4149                         (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
4150
4151                 writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
4152                                         (addr - SRAM_BASE_ADDR));
4153
4154         } else if ((addr >= DRAM_PHYS_BASE) &&
4155                         (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size)) {
4156
4157                 u64 bar_base_addr = DRAM_PHYS_BASE +
4158                                 (addr & ~(prop->dram_pci_bar_size - 0x1ull));
4159
4160                 ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
4161                 if (ddr_bar_addr != U64_MAX) {
4162                         writel(val, hdev->pcie_bar[DDR_BAR_ID] +
4163                                                 (addr - bar_base_addr));
4164
4165                         ddr_bar_addr = goya_set_ddr_bar_base(hdev,
4166                                                         ddr_bar_addr);
4167                 }
4168                 if (ddr_bar_addr == U64_MAX)
4169                         rc = -EIO;
4170
4171         } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
4172                 *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
4173
4174         } else {
4175                 rc = -EFAULT;
4176         }
4177
4178         return rc;
4179 }
4180
4181 static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
4182 {
4183         struct goya_device *goya = hdev->asic_specific;
4184
4185         if (hdev->hard_reset_pending)
4186                 return U64_MAX;
4187
4188         return readq(hdev->pcie_bar[DDR_BAR_ID] +
4189                         (addr - goya->ddr_bar_cur_addr));
4190 }
4191
4192 static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val)
4193 {
4194         struct goya_device *goya = hdev->asic_specific;
4195
4196         if (hdev->hard_reset_pending)
4197                 return;
4198
4199         writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
4200                         (addr - goya->ddr_bar_cur_addr));
4201 }
4202
4203 static const char *_goya_get_event_desc(u16 event_type)
4204 {
4205         switch (event_type) {
4206         case GOYA_ASYNC_EVENT_ID_PCIE_IF:
4207                 return "PCIe_if";
4208         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4209         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4210         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4211         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4212         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4213         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4214         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4215         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4216                 return "TPC%d_ecc";
4217         case GOYA_ASYNC_EVENT_ID_MME_ECC:
4218                 return "MME_ecc";
4219         case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
4220                 return "MME_ecc_ext";
4221         case GOYA_ASYNC_EVENT_ID_MMU_ECC:
4222                 return "MMU_ecc";
4223         case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
4224                 return "DMA_macro";
4225         case GOYA_ASYNC_EVENT_ID_DMA_ECC:
4226                 return "DMA_ecc";
4227         case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
4228                 return "CPU_if_ecc";
4229         case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
4230                 return "PSOC_mem";
4231         case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
4232                 return "PSOC_coresight";
4233         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4234                 return "SRAM%d";
4235         case GOYA_ASYNC_EVENT_ID_GIC500:
4236                 return "GIC500";
4237         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4238                 return "PLL%d";
4239         case GOYA_ASYNC_EVENT_ID_AXI_ECC:
4240                 return "AXI_ecc";
4241         case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
4242                 return "L2_ram_ecc";
4243         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
4244                 return "PSOC_gpio_05_sw_reset";
4245         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
4246                 return "PSOC_gpio_10_vrhot_icrit";
4247         case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
4248                 return "PCIe_dec";
4249         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4250         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4251         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4252         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4253         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4254         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4255         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4256         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4257                 return "TPC%d_dec";
4258         case GOYA_ASYNC_EVENT_ID_MME_WACS:
4259                 return "MME_wacs";
4260         case GOYA_ASYNC_EVENT_ID_MME_WACSD:
4261                 return "MME_wacsd";
4262         case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
4263                 return "CPU_axi_splitter";
4264         case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
4265                 return "PSOC_axi_dec";
4266         case GOYA_ASYNC_EVENT_ID_PSOC:
4267                 return "PSOC";
4268         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4269         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4270         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4271         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4272         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4273         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4274         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4275         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4276                 return "TPC%d_krn_err";
4277         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
4278                 return "TPC%d_cq";
4279         case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4280                 return "TPC%d_qm";
4281         case GOYA_ASYNC_EVENT_ID_MME_QM:
4282                 return "MME_qm";
4283         case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
4284                 return "MME_cq";
4285         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4286                 return "DMA%d_qm";
4287         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4288                 return "DMA%d_ch";
4289         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4290         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4291         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4292         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4293         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4294         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4295         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4296         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4297                 return "TPC%d_bmon_spmu";
4298         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4299                 return "DMA_bm_ch%d";
4300         default:
4301                 return "N/A";
4302         }
4303 }
4304
4305 static void goya_get_event_desc(u16 event_type, char *desc, size_t size)
4306 {
4307         u8 index;
4308
4309         switch (event_type) {
4310         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4311         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4312         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4313         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4314         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4315         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4316         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4317         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4318                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3;
4319                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4320                 break;
4321         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4322                 index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0;
4323                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4324                 break;
4325         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4326                 index = event_type - GOYA_ASYNC_EVENT_ID_PLL0;
4327                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4328                 break;
4329         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4330         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4331         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4332         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4333         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4334         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4335         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4336         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4337                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3;
4338                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4339                 break;
4340         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4341         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4342         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4343         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4344         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4345         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4346         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4347         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4348                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10;
4349                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4350                 break;
4351         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
4352                 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ;
4353                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4354                 break;
4355         case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4356                 index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM;
4357                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4358                 break;
4359         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4360                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM;
4361                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4362                 break;
4363         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4364                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH;
4365                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4366                 break;
4367         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4368         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4369         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4370         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4371         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4372         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4373         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4374         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4375                 index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10;
4376                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4377                 break;
4378         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4379                 index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0;
4380                 snprintf(desc, size, _goya_get_event_desc(event_type), index);
4381                 break;
4382         default:
4383                 snprintf(desc, size, _goya_get_event_desc(event_type));
4384                 break;
4385         }
4386 }
4387
4388 static void goya_print_razwi_info(struct hl_device *hdev)
4389 {
4390         if (RREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD)) {
4391                 dev_err(hdev->dev, "Illegal write to LBW\n");
4392                 WREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD, 0);
4393         }
4394
4395         if (RREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD)) {
4396                 dev_err(hdev->dev, "Illegal read from LBW\n");
4397                 WREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD, 0);
4398         }
4399
4400         if (RREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD)) {
4401                 dev_err(hdev->dev, "Illegal write to HBW\n");
4402                 WREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD, 0);
4403         }
4404
4405         if (RREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD)) {
4406                 dev_err(hdev->dev, "Illegal read from HBW\n");
4407                 WREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD, 0);
4408         }
4409 }
4410
4411 static void goya_print_mmu_error_info(struct hl_device *hdev)
4412 {
4413         struct goya_device *goya = hdev->asic_specific;
4414         u64 addr;
4415         u32 val;
4416
4417         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4418                 return;
4419
4420         val = RREG32(mmMMU_PAGE_ERROR_CAPTURE);
4421         if (val & MMU_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {
4422                 addr = val & MMU_PAGE_ERROR_CAPTURE_VA_49_32_MASK;
4423                 addr <<= 32;
4424                 addr |= RREG32(mmMMU_PAGE_ERROR_CAPTURE_VA);
4425
4426                 dev_err(hdev->dev, "MMU page fault on va 0x%llx\n", addr);
4427
4428                 WREG32(mmMMU_PAGE_ERROR_CAPTURE, 0);
4429         }
4430 }
4431
4432 static void goya_print_irq_info(struct hl_device *hdev, u16 event_type,
4433                                 bool razwi)
4434 {
4435         char desc[20] = "";
4436
4437         goya_get_event_desc(event_type, desc, sizeof(desc));
4438         dev_err(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
4439                 event_type, desc);
4440
4441         if (razwi) {
4442                 goya_print_razwi_info(hdev);
4443                 goya_print_mmu_error_info(hdev);
4444         }
4445 }
4446
4447 static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr,
4448                 size_t irq_arr_size)
4449 {
4450         struct armcp_unmask_irq_arr_packet *pkt;
4451         size_t total_pkt_size;
4452         long result;
4453         int rc;
4454         int irq_num_entries, irq_arr_index;
4455         __le32 *goya_irq_arr;
4456
4457         total_pkt_size = sizeof(struct armcp_unmask_irq_arr_packet) +
4458                         irq_arr_size;
4459
4460         /* data should be aligned to 8 bytes in order to ArmCP to copy it */
4461         total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
4462
4463         /* total_pkt_size is casted to u16 later on */
4464         if (total_pkt_size > USHRT_MAX) {
4465                 dev_err(hdev->dev, "too many elements in IRQ array\n");
4466                 return -EINVAL;
4467         }
4468
4469         pkt = kzalloc(total_pkt_size, GFP_KERNEL);
4470         if (!pkt)
4471                 return -ENOMEM;
4472
4473         irq_num_entries = irq_arr_size / sizeof(irq_arr[0]);
4474         pkt->length = cpu_to_le32(irq_num_entries);
4475
4476         /* We must perform any necessary endianness conversation on the irq
4477          * array being passed to the goya hardware
4478          */
4479         for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs;
4480                         irq_arr_index < irq_num_entries ; irq_arr_index++)
4481                 goya_irq_arr[irq_arr_index] =
4482                                 cpu_to_le32(irq_arr[irq_arr_index]);
4483
4484         pkt->armcp_pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
4485                                                 ARMCP_PKT_CTL_OPCODE_SHIFT);
4486
4487         rc = goya_send_cpu_message(hdev, (u32 *) pkt, total_pkt_size,
4488                         HL_DEVICE_TIMEOUT_USEC, &result);
4489
4490         if (rc)
4491                 dev_err(hdev->dev, "failed to unmask IRQ array\n");
4492
4493         kfree(pkt);
4494
4495         return rc;
4496 }
4497
4498 static int goya_soft_reset_late_init(struct hl_device *hdev)
4499 {
4500         /*
4501          * Unmask all IRQs since some could have been received
4502          * during the soft reset
4503          */
4504         return goya_unmask_irq_arr(hdev, goya_all_events,
4505                                         sizeof(goya_all_events));
4506 }
4507
4508 static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
4509 {
4510         struct armcp_packet pkt;
4511         long result;
4512         int rc;
4513
4514         memset(&pkt, 0, sizeof(pkt));
4515
4516         pkt.ctl = cpu_to_le32(ARMCP_PACKET_UNMASK_RAZWI_IRQ <<
4517                                 ARMCP_PKT_CTL_OPCODE_SHIFT);
4518         pkt.value = cpu_to_le64(event_type);
4519
4520         rc = goya_send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
4521                         HL_DEVICE_TIMEOUT_USEC, &result);
4522
4523         if (rc)
4524                 dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
4525
4526         return rc;
4527 }
4528
4529 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
4530 {
4531         u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
4532         u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
4533                                 >> EQ_CTL_EVENT_TYPE_SHIFT);
4534         struct goya_device *goya = hdev->asic_specific;
4535
4536         goya->events_stat[event_type]++;
4537         goya->events_stat_aggregate[event_type]++;
4538
4539         switch (event_type) {
4540         case GOYA_ASYNC_EVENT_ID_PCIE_IF:
4541         case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
4542         case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
4543         case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
4544         case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
4545         case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
4546         case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
4547         case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
4548         case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
4549         case GOYA_ASYNC_EVENT_ID_MME_ECC:
4550         case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
4551         case GOYA_ASYNC_EVENT_ID_MMU_ECC:
4552         case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
4553         case GOYA_ASYNC_EVENT_ID_DMA_ECC:
4554         case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
4555         case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
4556         case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
4557         case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
4558         case GOYA_ASYNC_EVENT_ID_GIC500:
4559         case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
4560         case GOYA_ASYNC_EVENT_ID_AXI_ECC:
4561         case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
4562         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
4563                 goya_print_irq_info(hdev, event_type, false);
4564                 hl_device_reset(hdev, true, false);
4565                 break;
4566
4567         case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
4568         case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
4569         case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
4570         case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
4571         case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
4572         case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
4573         case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
4574         case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
4575         case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
4576         case GOYA_ASYNC_EVENT_ID_MME_WACS:
4577         case GOYA_ASYNC_EVENT_ID_MME_WACSD:
4578         case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
4579         case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
4580         case GOYA_ASYNC_EVENT_ID_PSOC:
4581         case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
4582         case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
4583         case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
4584         case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
4585         case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
4586         case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
4587         case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
4588         case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
4589         case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
4590         case GOYA_ASYNC_EVENT_ID_MME_QM:
4591         case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
4592         case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
4593         case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
4594                 goya_print_irq_info(hdev, event_type, true);
4595                 goya_unmask_irq(hdev, event_type);
4596                 break;
4597
4598         case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
4599         case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
4600         case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
4601         case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
4602         case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
4603         case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
4604         case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
4605         case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
4606         case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
4607         case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
4608                 goya_print_irq_info(hdev, event_type, false);
4609                 goya_unmask_irq(hdev, event_type);
4610                 break;
4611
4612         default:
4613                 dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
4614                                 event_type);
4615                 break;
4616         }
4617 }
4618
4619 void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
4620 {
4621         struct goya_device *goya = hdev->asic_specific;
4622
4623         if (aggregate) {
4624                 *size = (u32) sizeof(goya->events_stat_aggregate);
4625                 return goya->events_stat_aggregate;
4626         }
4627
4628         *size = (u32) sizeof(goya->events_stat);
4629         return goya->events_stat;
4630 }
4631
4632 static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size,
4633                                 u64 val, bool is_dram)
4634 {
4635         struct packet_lin_dma *lin_dma_pkt;
4636         struct hl_cs_job *job;
4637         u32 cb_size, ctl;
4638         struct hl_cb *cb;
4639         int rc, lin_dma_pkts_cnt;
4640
4641         lin_dma_pkts_cnt = DIV_ROUND_UP_ULL(size, SZ_2G);
4642         cb_size = lin_dma_pkts_cnt * sizeof(struct packet_lin_dma) +
4643                                                 sizeof(struct packet_msg_prot);
4644         cb = hl_cb_kernel_create(hdev, cb_size);
4645         if (!cb)
4646                 return -ENOMEM;
4647
4648         lin_dma_pkt = (struct packet_lin_dma *) (uintptr_t) cb->kernel_address;
4649
4650         do {
4651                 memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));
4652
4653                 ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) |
4654                                 (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) |
4655                                 (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) |
4656                                 (1 << GOYA_PKT_CTL_RB_SHIFT) |
4657                                 (1 << GOYA_PKT_CTL_MB_SHIFT));
4658                 ctl |= (is_dram ? DMA_HOST_TO_DRAM : DMA_HOST_TO_SRAM) <<
4659                                 GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
4660                 lin_dma_pkt->ctl = cpu_to_le32(ctl);
4661
4662                 lin_dma_pkt->src_addr = cpu_to_le64(val);
4663                 lin_dma_pkt->dst_addr = cpu_to_le64(addr);
4664                 if (lin_dma_pkts_cnt > 1)
4665                         lin_dma_pkt->tsize = cpu_to_le32(SZ_2G);
4666                 else
4667                         lin_dma_pkt->tsize = cpu_to_le32(size);
4668
4669                 size -= SZ_2G;
4670                 addr += SZ_2G;
4671                 lin_dma_pkt++;
4672         } while (--lin_dma_pkts_cnt);
4673
4674         job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
4675         if (!job) {
4676                 dev_err(hdev->dev, "Failed to allocate a new job\n");
4677                 rc = -ENOMEM;
4678                 goto release_cb;
4679         }
4680
4681         job->id = 0;
4682         job->user_cb = cb;
4683         job->user_cb->cs_cnt++;
4684         job->user_cb_size = cb_size;
4685         job->hw_queue_id = GOYA_QUEUE_ID_DMA_0;
4686         job->patched_cb = job->user_cb;
4687         job->job_cb_size = job->user_cb_size;
4688
4689         hl_debugfs_add_job(hdev, job);
4690
4691         rc = goya_send_job_on_qman0(hdev, job);
4692
4693         hl_debugfs_remove_job(hdev, job);
4694         kfree(job);
4695         cb->cs_cnt--;
4696
4697 release_cb:
4698         hl_cb_put(cb);
4699         hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
4700
4701         return rc;
4702 }
4703
4704 int goya_context_switch(struct hl_device *hdev, u32 asid)
4705 {
4706         struct asic_fixed_properties *prop = &hdev->asic_prop;
4707         u64 addr = prop->sram_base_address, sob_addr;
4708         u32 size = hdev->pldm ? 0x10000 : prop->sram_size;
4709         u64 val = 0x7777777777777777ull;
4710         int rc, dma_id;
4711         u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO -
4712                                         mmDMA_CH_0_WR_COMP_ADDR_LO;
4713
4714         rc = goya_memset_device_memory(hdev, addr, size, val, false);
4715         if (rc) {
4716                 dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
4717                 return rc;
4718         }
4719
4720         /* we need to reset registers that the user is allowed to change */
4721         sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
4722         WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO, lower_32_bits(sob_addr));
4723
4724         for (dma_id = 1 ; dma_id < NUMBER_OF_EXT_HW_QUEUES ; dma_id++) {
4725                 sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
4726                                                         (dma_id - 1) * 4;
4727                 WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + channel_off * dma_id,
4728                                                 lower_32_bits(sob_addr));
4729         }
4730
4731         WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
4732
4733         goya_mmu_prepare(hdev, asid);
4734
4735         goya_clear_sm_regs(hdev);
4736
4737         return 0;
4738 }
4739
4740 static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
4741 {
4742         struct asic_fixed_properties *prop = &hdev->asic_prop;
4743         struct goya_device *goya = hdev->asic_specific;
4744         u64 addr = prop->mmu_pgt_addr;
4745         u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE +
4746                         MMU_CACHE_MNG_SIZE;
4747
4748         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4749                 return 0;
4750
4751         return goya_memset_device_memory(hdev, addr, size, 0, true);
4752 }
4753
4754 static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
4755 {
4756         struct goya_device *goya = hdev->asic_specific;
4757         u64 addr = hdev->asic_prop.mmu_dram_default_page_addr;
4758         u32 size = MMU_DRAM_DEFAULT_PAGE_SIZE;
4759         u64 val = 0x9999999999999999ull;
4760
4761         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4762                 return 0;
4763
4764         return goya_memset_device_memory(hdev, addr, size, val, true);
4765 }
4766
4767 static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev)
4768 {
4769         struct asic_fixed_properties *prop = &hdev->asic_prop;
4770         struct goya_device *goya = hdev->asic_specific;
4771         s64 off, cpu_off;
4772         int rc;
4773
4774         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4775                 return 0;
4776
4777         for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) {
4778                 rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off,
4779                                 prop->dram_base_address + off, PAGE_SIZE_2MB);
4780                 if (rc) {
4781                         dev_err(hdev->dev, "Map failed for address 0x%llx\n",
4782                                 prop->dram_base_address + off);
4783                         goto unmap;
4784                 }
4785         }
4786
4787         if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4788                 rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4789                         hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB);
4790
4791                 if (rc) {
4792                         dev_err(hdev->dev,
4793                                 "Map failed for CPU accessible memory\n");
4794                         off -= PAGE_SIZE_2MB;
4795                         goto unmap;
4796                 }
4797         } else {
4798                 for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) {
4799                         rc = hl_mmu_map(hdev->kernel_ctx,
4800                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4801                                 hdev->cpu_accessible_dma_address + cpu_off,
4802                                 PAGE_SIZE_4KB);
4803                         if (rc) {
4804                                 dev_err(hdev->dev,
4805                                         "Map failed for CPU accessible memory\n");
4806                                 cpu_off -= PAGE_SIZE_4KB;
4807                                 goto unmap_cpu;
4808                         }
4809                 }
4810         }
4811
4812         goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID);
4813         goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID);
4814         WREG32(mmCPU_IF_ARUSER_OVR_EN, 0x7FF);
4815         WREG32(mmCPU_IF_AWUSER_OVR_EN, 0x7FF);
4816
4817         /* Make sure configuration is flushed to device */
4818         RREG32(mmCPU_IF_AWUSER_OVR_EN);
4819
4820         goya->device_cpu_mmu_mappings_done = true;
4821
4822         return 0;
4823
4824 unmap_cpu:
4825         for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB)
4826                 if (hl_mmu_unmap(hdev->kernel_ctx,
4827                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4828                                 PAGE_SIZE_4KB))
4829                         dev_warn_ratelimited(hdev->dev,
4830                                 "failed to unmap address 0x%llx\n",
4831                                 VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
4832 unmap:
4833         for (; off >= 0 ; off -= PAGE_SIZE_2MB)
4834                 if (hl_mmu_unmap(hdev->kernel_ctx,
4835                                 prop->dram_base_address + off, PAGE_SIZE_2MB))
4836                         dev_warn_ratelimited(hdev->dev,
4837                                 "failed to unmap address 0x%llx\n",
4838                                 prop->dram_base_address + off);
4839
4840         return rc;
4841 }
4842
4843 void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev)
4844 {
4845         struct asic_fixed_properties *prop = &hdev->asic_prop;
4846         struct goya_device *goya = hdev->asic_specific;
4847         u32 off, cpu_off;
4848
4849         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4850                 return;
4851
4852         if (!goya->device_cpu_mmu_mappings_done)
4853                 return;
4854
4855         WREG32(mmCPU_IF_ARUSER_OVR_EN, 0);
4856         WREG32(mmCPU_IF_AWUSER_OVR_EN, 0);
4857
4858         if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
4859                 if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
4860                                 PAGE_SIZE_2MB))
4861                         dev_warn(hdev->dev,
4862                                 "Failed to unmap CPU accessible memory\n");
4863         } else {
4864                 for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB)
4865                         if (hl_mmu_unmap(hdev->kernel_ctx,
4866                                         VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
4867                                         PAGE_SIZE_4KB))
4868                                 dev_warn_ratelimited(hdev->dev,
4869                                         "failed to unmap address 0x%llx\n",
4870                                         VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
4871         }
4872
4873         for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB)
4874                 if (hl_mmu_unmap(hdev->kernel_ctx,
4875                                 prop->dram_base_address + off, PAGE_SIZE_2MB))
4876                         dev_warn_ratelimited(hdev->dev,
4877                                         "Failed to unmap address 0x%llx\n",
4878                                         prop->dram_base_address + off);
4879
4880         goya->device_cpu_mmu_mappings_done = false;
4881 }
4882
4883 static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
4884 {
4885         struct goya_device *goya = hdev->asic_specific;
4886         int i;
4887
4888         if (!(goya->hw_cap_initialized & HW_CAP_MMU))
4889                 return;
4890
4891         if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) {
4892                 WARN(1, "asid %u is too big\n", asid);
4893                 return;
4894         }
4895
4896         /* zero the MMBP and ASID bits and then set the ASID */
4897         for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++)
4898                 goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
4899 }
4900
4901 static void goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
4902                                         u32 flags)
4903 {
4904         struct goya_device *goya = hdev->asic_specific;
4905         u32 status, timeout_usec;
4906         int rc;
4907
4908         if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
4909                 hdev->hard_reset_pending)
4910                 return;
4911
4912         /* no need in L1 only invalidation in Goya */
4913         if (!is_hard)
4914                 return;
4915
4916         if (hdev->pldm)
4917                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
4918         else
4919                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
4920
4921         mutex_lock(&hdev->mmu_cache_lock);
4922
4923         /* L0 & L1 invalidation */
4924         WREG32(mmSTLB_INV_ALL_START, 1);
4925
4926         rc = hl_poll_timeout(
4927                 hdev,
4928                 mmSTLB_INV_ALL_START,
4929                 status,
4930                 !status,
4931                 1000,
4932                 timeout_usec);
4933
4934         mutex_unlock(&hdev->mmu_cache_lock);
4935
4936         if (rc)
4937                 dev_notice_ratelimited(hdev->dev,
4938                         "Timeout when waiting for MMU cache invalidation\n");
4939 }
4940
4941 static void goya_mmu_invalidate_cache_range(struct hl_device *hdev,
4942                 bool is_hard, u32 asid, u64 va, u64 size)
4943 {
4944         struct goya_device *goya = hdev->asic_specific;
4945         u32 status, timeout_usec, inv_data, pi;
4946         int rc;
4947
4948         if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
4949                 hdev->hard_reset_pending)
4950                 return;
4951
4952         /* no need in L1 only invalidation in Goya */
4953         if (!is_hard)
4954                 return;
4955
4956         if (hdev->pldm)
4957                 timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
4958         else
4959                 timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
4960
4961         mutex_lock(&hdev->mmu_cache_lock);
4962
4963         /*
4964          * TODO: currently invalidate entire L0 & L1 as in regular hard
4965          * invalidation. Need to apply invalidation of specific cache lines with
4966          * mask of ASID & VA & size.
4967          * Note that L1 with be flushed entirely in any case.
4968          */
4969
4970         /* L0 & L1 invalidation */
4971         inv_data = RREG32(mmSTLB_CACHE_INV);
4972         /* PI is 8 bit */
4973         pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
4974         WREG32(mmSTLB_CACHE_INV,
4975                         (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
4976
4977         rc = hl_poll_timeout(
4978                 hdev,
4979                 mmSTLB_INV_CONSUMER_INDEX,
4980                 status,
4981                 status == pi,
4982                 1000,
4983                 timeout_usec);
4984
4985         mutex_unlock(&hdev->mmu_cache_lock);
4986
4987         if (rc)
4988                 dev_notice_ratelimited(hdev->dev,
4989                         "Timeout when waiting for MMU cache invalidation\n");
4990 }
4991
4992 int goya_send_heartbeat(struct hl_device *hdev)
4993 {
4994         struct goya_device *goya = hdev->asic_specific;
4995
4996         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
4997                 return 0;
4998
4999         return hl_fw_send_heartbeat(hdev);
5000 }
5001
5002 int goya_armcp_info_get(struct hl_device *hdev)
5003 {
5004         struct goya_device *goya = hdev->asic_specific;
5005         struct asic_fixed_properties *prop = &hdev->asic_prop;
5006         u64 dram_size;
5007         int rc;
5008
5009         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
5010                 return 0;
5011
5012         rc = hl_fw_armcp_info_get(hdev);
5013         if (rc)
5014                 return rc;
5015
5016         dram_size = le64_to_cpu(prop->armcp_info.dram_size);
5017         if (dram_size) {
5018                 if ((!is_power_of_2(dram_size)) ||
5019                                 (dram_size < DRAM_PHYS_DEFAULT_SIZE)) {
5020                         dev_err(hdev->dev,
5021                                 "F/W reported invalid DRAM size %llu. Trying to use default size\n",
5022                                 dram_size);
5023                         dram_size = DRAM_PHYS_DEFAULT_SIZE;
5024                 }
5025
5026                 prop->dram_size = dram_size;
5027                 prop->dram_end_address = prop->dram_base_address + dram_size;
5028         }
5029
5030         if (!strlen(prop->armcp_info.card_name))
5031                 strncpy(prop->armcp_info.card_name, GOYA_DEFAULT_CARD_NAME,
5032                                 CARD_NAME_MAX_LEN);
5033
5034         return 0;
5035 }
5036
5037 static bool goya_is_device_idle(struct hl_device *hdev, u32 *mask,
5038                                 struct seq_file *s)
5039 {
5040         const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n";
5041         const char *dma_fmt = "%-5d%-9s%#-14x%#x\n";
5042         u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts,
5043                 mme_arch_sts;
5044         bool is_idle = true, is_eng_idle;
5045         u64 offset;
5046         int i;
5047
5048         if (s)
5049                 seq_puts(s, "\nDMA  is_idle  QM_GLBL_STS0  DMA_CORE_STS0\n"
5050                                 "---  -------  ------------  -------------\n");
5051
5052         offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0;
5053
5054         for (i = 0 ; i < DMA_MAX_NUM ; i++) {
5055                 qm_glbl_sts0 = RREG32(mmDMA_QM_0_GLBL_STS0 + i * offset);
5056                 dma_core_sts0 = RREG32(mmDMA_CH_0_STS0 + i * offset);
5057                 is_eng_idle = IS_DMA_QM_IDLE(qm_glbl_sts0) &&
5058                                 IS_DMA_IDLE(dma_core_sts0);
5059                 is_idle &= is_eng_idle;
5060
5061                 if (mask)
5062                         *mask |= !is_eng_idle << (GOYA_ENGINE_ID_DMA_0 + i);
5063                 if (s)
5064                         seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N",
5065                                         qm_glbl_sts0, dma_core_sts0);
5066         }
5067
5068         if (s)
5069                 seq_puts(s,
5070                         "\nTPC  is_idle  QM_GLBL_STS0  CMDQ_GLBL_STS0  CFG_STATUS\n"
5071                         "---  -------  ------------  --------------  ----------\n");
5072
5073         offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0;
5074
5075         for (i = 0 ; i < TPC_MAX_NUM ; i++) {
5076                 qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + i * offset);
5077                 cmdq_glbl_sts0 = RREG32(mmTPC0_CMDQ_GLBL_STS0 + i * offset);
5078                 tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + i * offset);
5079                 is_eng_idle = IS_TPC_QM_IDLE(qm_glbl_sts0) &&
5080                                 IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) &&
5081                                 IS_TPC_IDLE(tpc_cfg_sts);
5082                 is_idle &= is_eng_idle;
5083
5084                 if (mask)
5085                         *mask |= !is_eng_idle << (GOYA_ENGINE_ID_TPC_0 + i);
5086                 if (s)
5087                         seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N",
5088                                 qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts);
5089         }
5090
5091         if (s)
5092                 seq_puts(s,
5093                         "\nMME  is_idle  QM_GLBL_STS0  CMDQ_GLBL_STS0  ARCH_STATUS\n"
5094                         "---  -------  ------------  --------------  -----------\n");
5095
5096         qm_glbl_sts0 = RREG32(mmMME_QM_GLBL_STS0);
5097         cmdq_glbl_sts0 = RREG32(mmMME_CMDQ_GLBL_STS0);
5098         mme_arch_sts = RREG32(mmMME_ARCH_STATUS);
5099         is_eng_idle = IS_MME_QM_IDLE(qm_glbl_sts0) &&
5100                         IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) &&
5101                         IS_MME_IDLE(mme_arch_sts);
5102         is_idle &= is_eng_idle;
5103
5104         if (mask)
5105                 *mask |= !is_eng_idle << GOYA_ENGINE_ID_MME_0;
5106         if (s) {
5107                 seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0,
5108                                 cmdq_glbl_sts0, mme_arch_sts);
5109                 seq_puts(s, "\n");
5110         }
5111
5112         return is_idle;
5113 }
5114
5115 static void goya_hw_queues_lock(struct hl_device *hdev)
5116 {
5117         struct goya_device *goya = hdev->asic_specific;
5118
5119         spin_lock(&goya->hw_queues_lock);
5120 }
5121
5122 static void goya_hw_queues_unlock(struct hl_device *hdev)
5123 {
5124         struct goya_device *goya = hdev->asic_specific;
5125
5126         spin_unlock(&goya->hw_queues_lock);
5127 }
5128
5129 static u32 goya_get_pci_id(struct hl_device *hdev)
5130 {
5131         return hdev->pdev->device;
5132 }
5133
5134 static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
5135                                 size_t max_size)
5136 {
5137         struct goya_device *goya = hdev->asic_specific;
5138
5139         if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
5140                 return 0;
5141
5142         return hl_fw_get_eeprom_data(hdev, data, max_size);
5143 }
5144
5145 static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
5146 {
5147         return RREG32(mmHW_STATE);
5148 }
5149
5150 static const struct hl_asic_funcs goya_funcs = {
5151         .early_init = goya_early_init,
5152         .early_fini = goya_early_fini,
5153         .late_init = goya_late_init,
5154         .late_fini = goya_late_fini,
5155         .sw_init = goya_sw_init,
5156         .sw_fini = goya_sw_fini,
5157         .hw_init = goya_hw_init,
5158         .hw_fini = goya_hw_fini,
5159         .halt_engines = goya_halt_engines,
5160         .suspend = goya_suspend,
5161         .resume = goya_resume,
5162         .cb_mmap = goya_cb_mmap,
5163         .ring_doorbell = goya_ring_doorbell,
5164         .pqe_write = goya_pqe_write,
5165         .asic_dma_alloc_coherent = goya_dma_alloc_coherent,
5166         .asic_dma_free_coherent = goya_dma_free_coherent,
5167         .get_int_queue_base = goya_get_int_queue_base,
5168         .test_queues = goya_test_queues,
5169         .asic_dma_pool_zalloc = goya_dma_pool_zalloc,
5170         .asic_dma_pool_free = goya_dma_pool_free,
5171         .cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
5172         .cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
5173         .hl_dma_unmap_sg = goya_dma_unmap_sg,
5174         .cs_parser = goya_cs_parser,
5175         .asic_dma_map_sg = goya_dma_map_sg,
5176         .get_dma_desc_list_size = goya_get_dma_desc_list_size,
5177         .add_end_of_cb_packets = goya_add_end_of_cb_packets,
5178         .update_eq_ci = goya_update_eq_ci,
5179         .context_switch = goya_context_switch,
5180         .restore_phase_topology = goya_restore_phase_topology,
5181         .debugfs_read32 = goya_debugfs_read32,
5182         .debugfs_write32 = goya_debugfs_write32,
5183         .add_device_attr = goya_add_device_attr,
5184         .handle_eqe = goya_handle_eqe,
5185         .set_pll_profile = goya_set_pll_profile,
5186         .get_events_stat = goya_get_events_stat,
5187         .read_pte = goya_read_pte,
5188         .write_pte = goya_write_pte,
5189         .mmu_invalidate_cache = goya_mmu_invalidate_cache,
5190         .mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range,
5191         .send_heartbeat = goya_send_heartbeat,
5192         .debug_coresight = goya_debug_coresight,
5193         .is_device_idle = goya_is_device_idle,
5194         .soft_reset_late_init = goya_soft_reset_late_init,
5195         .hw_queues_lock = goya_hw_queues_lock,
5196         .hw_queues_unlock = goya_hw_queues_unlock,
5197         .get_pci_id = goya_get_pci_id,
5198         .get_eeprom_data = goya_get_eeprom_data,
5199         .send_cpu_message = goya_send_cpu_message,
5200         .get_hw_state = goya_get_hw_state,
5201         .pci_bars_map = goya_pci_bars_map,
5202         .set_dram_bar_base = goya_set_ddr_bar_base,
5203         .init_iatu = goya_init_iatu,
5204         .rreg = hl_rreg,
5205         .wreg = hl_wreg,
5206         .halt_coresight = goya_halt_coresight,
5207         .get_clk_rate = goya_get_clk_rate
5208 };
5209
5210 /*
5211  * goya_set_asic_funcs - set Goya function pointers
5212  *
5213  * @*hdev: pointer to hl_device structure
5214  *
5215  */
5216 void goya_set_asic_funcs(struct hl_device *hdev)
5217 {
5218         hdev->asic_funcs = &goya_funcs;
5219 }