drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 static void psp_set_funcs(struct amdgpu_device *adev);
  38
  39 static int psp_early_init(void *handle)
  40 {
  41         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  42         struct psp_context *psp = &adev->psp;
  43
  44         psp_set_funcs(adev);
  45
  46         switch (adev->asic_type) {
  47         case CHIP_VEGA10:
  48         case CHIP_VEGA12:
  49                 psp_v3_1_set_psp_funcs(psp);
  50                 psp->autoload_supported = false;
  51                 break;
  52         case CHIP_RAVEN:
  53                 psp_v10_0_set_psp_funcs(psp);
  54                 psp->autoload_supported = false;
  55                 break;
  56         case CHIP_VEGA20:
  57         case CHIP_ARCTURUS:
  58                 psp_v11_0_set_psp_funcs(psp);
  59                 psp->autoload_supported = false;
  60                 break;
  61         case CHIP_NAVI10:
  62         case CHIP_NAVI14:
  63         case CHIP_NAVI12:
  64                 psp_v11_0_set_psp_funcs(psp);
  65                 psp->autoload_supported = true;
  66                 break;
  67         case CHIP_RENOIR:
  68                 psp_v12_0_set_psp_funcs(psp);
  69                 break;
  70         default:
  71                 return -EINVAL;
  72         }
  73
  74         psp->adev = adev;
  75
  76         return 0;
  77 }
  78
  79 static int psp_sw_init(void *handle)
  80 {
  81         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  82         struct psp_context *psp = &adev->psp;
  83         int ret;
  84
  85         ret = psp_init_microcode(psp);
  86         if (ret) {
  87                 DRM_ERROR("Failed to load psp firmware!\n");
  88                 return ret;
  89         }
  90
  91         return 0;
  92 }
  93
  94 static int psp_sw_fini(void *handle)
  95 {
  96         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  97
  98         release_firmware(adev->psp.sos_fw);
  99         adev->psp.sos_fw = NULL;
 100         release_firmware(adev->psp.asd_fw);
 101         adev->psp.asd_fw = NULL;
 102         if (adev->psp.ta_fw) {
 103                 release_firmware(adev->psp.ta_fw);
 104                 adev->psp.ta_fw = NULL;
 105         }
 106         return 0;
 107 }
 108
 109 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 110                  uint32_t reg_val, uint32_t mask, bool check_changed)
 111 {
 112         uint32_t val;
 113         int i;
 114         struct amdgpu_device *adev = psp->adev;
 115
 116         for (i = 0; i < adev->usec_timeout; i++) {
 117                 val = RREG32(reg_index);
 118                 if (check_changed) {
 119                         if (val != reg_val)
 120                                 return 0;
 121                 } else {
 122                         if ((val & mask) == reg_val)
 123                                 return 0;
 124                 }
 125                 udelay(1);
 126         }
 127
 128         return -ETIME;
 129 }
 130
 131 static int
 132 psp_cmd_submit_buf(struct psp_context *psp,
 133                    struct amdgpu_firmware_info *ucode,
 134                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 135 {
 136         int ret;
 137         int index;
 138         int timeout = 2000;
 139
 140         mutex_lock(&psp->mutex);
 141
 142         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 143
 144         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 145
 146         index = atomic_inc_return(&psp->fence_value);
 147         ret = psp_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 148         if (ret) {
 149                 atomic_dec(&psp->fence_value);
 150                 mutex_unlock(&psp->mutex);
 151                 return ret;
 152         }
 153
 154         while (*((unsigned int *)psp->fence_buf) != index) {
 155                 if (--timeout == 0)
 156                         break;
 157                 msleep(1);
 158         }
 159
 160         /* In some cases, psp response status is not 0 even there is no
 161          * problem while the command is submitted. Some version of PSP FW
 162          * doesn't write 0 to that field.
 163          * So here we would like to only print a warning instead of an error
 164          * during psp initialization to avoid breaking hw_init and it doesn't
 165          * return -EINVAL.
 166          */
 167         if (psp->cmd_buf_mem->resp.status || !timeout) {
 168                 if (ucode)
 169                         DRM_WARN("failed to load ucode id (%d) ",
 170                                   ucode->ucode_id);
 171                 DRM_WARN("psp command failed and response status is (0x%X)\n",
 172                           psp->cmd_buf_mem->resp.status & GFX_CMD_STATUS_MASK);
 173                 if (!timeout) {
 174                         mutex_unlock(&psp->mutex);
 175                         return -EINVAL;
 176                 }
 177         }
 178
 179         /* get xGMI session id from response buffer */
 180         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 181
 182         if (ucode) {
 183                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 184                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 185         }
 186         mutex_unlock(&psp->mutex);
 187
 188         return ret;
 189 }
 190
 191 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 192                                  struct psp_gfx_cmd_resp *cmd,
 193                                  uint64_t tmr_mc, uint32_t size)
 194 {
 195         if (psp_support_vmr_ring(psp))
 196                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 197         else
 198                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 199         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 200         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 201         cmd->cmd.cmd_setup_tmr.buf_size = size;
 202 }
 203
 204 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 205                                       uint64_t pri_buf_mc, uint32_t size)
 206 {
 207         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 208         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 209         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 210         cmd->cmd.cmd_load_toc.toc_size = size;
 211 }
 212
 213 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 214 static int psp_load_toc(struct psp_context *psp,
 215                         uint32_t *tmr_size)
 216 {
 217         int ret;
 218         struct psp_gfx_cmd_resp *cmd;
 219
 220         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 221         if (!cmd)
 222                 return -ENOMEM;
 223         /* Copy toc to psp firmware private buffer */
 224         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 225         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 226
 227         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 228
 229         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 230                                  psp->fence_buf_mc_addr);
 231         if (!ret)
 232                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 233         kfree(cmd);
 234         return ret;
 235 }
 236
 237 /* Set up Trusted Memory Region */
 238 static int psp_tmr_init(struct psp_context *psp)
 239 {
 240         int ret;
 241         int tmr_size;
 242
 243         /*
 244          * According to HW engineer, they prefer the TMR address be "naturally
 245          * aligned" , e.g. the start address be an integer divide of TMR size.
 246          *
 247          * Note: this memory need be reserved till the driver
 248          * uninitializes.
 249          */
 250         tmr_size = PSP_TMR_SIZE;
 251
 252         /* For ASICs support RLC autoload, psp will parse the toc
 253          * and calculate the total size of TMR needed */
 254         if (psp->toc_start_addr &&
 255             psp->toc_bin_size &&
 256             psp->fw_pri_buf) {
 257                 ret = psp_load_toc(psp, &tmr_size);
 258                 if (ret) {
 259                         DRM_ERROR("Failed to load toc\n");
 260                         return ret;
 261                 }
 262         }
 263
 264         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 265                                       AMDGPU_GEM_DOMAIN_VRAM,
 266                                       &psp->tmr_bo, &psp->tmr_mc_addr, NULL);
 267
 268         return ret;
 269 }
 270
 271 static int psp_tmr_load(struct psp_context *psp)
 272 {
 273         int ret;
 274         struct psp_gfx_cmd_resp *cmd;
 275
 276         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 277         if (!cmd)
 278                 return -ENOMEM;
 279
 280         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 281                              amdgpu_bo_size(psp->tmr_bo));
 282         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 283                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 284
 285         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 286                                  psp->fence_buf_mc_addr);
 287         if (ret)
 288                 goto failed;
 289
 290         kfree(cmd);
 291
 292         return 0;
 293
 294 failed:
 295         kfree(cmd);
 296         return ret;
 297 }
 298
 299 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 300                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 301                                  uint32_t size, uint32_t shared_size)
 302 {
 303         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 304         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 305         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 306         cmd->cmd.cmd_load_ta.app_len = size;
 307
 308         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 309         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 310         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 311 }
 312
 313 static int psp_asd_init(struct psp_context *psp)
 314 {
 315         int ret;
 316
 317         /*
 318          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 319          * physical) for shared ASD <-> Driver
 320          */
 321         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 322                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 323                                       &psp->asd_shared_bo,
 324                                       &psp->asd_shared_mc_addr,
 325                                       &psp->asd_shared_buf);
 326
 327         return ret;
 328 }
 329
 330 static int psp_asd_load(struct psp_context *psp)
 331 {
 332         int ret;
 333         struct psp_gfx_cmd_resp *cmd;
 334
 335         /* If PSP version doesn't match ASD version, asd loading will be failed.
 336          * add workaround to bypass it for sriov now.
 337          * TODO: add version check to make it common
 338          */
 339         if (amdgpu_sriov_vf(psp->adev))
 340                 return 0;
 341
 342         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 343         if (!cmd)
 344                 return -ENOMEM;
 345
 346         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 347         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 348
 349         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 350                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 351
 352         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 353                                  psp->fence_buf_mc_addr);
 354
 355         kfree(cmd);
 356
 357         return ret;
 358 }
 359
 360 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 361                 uint32_t id, uint32_t value)
 362 {
 363         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 364         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 365         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 366 }
 367
 368 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 369                 uint32_t value)
 370 {
 371         struct psp_gfx_cmd_resp *cmd = NULL;
 372         int ret = 0;
 373
 374         if (reg >= PSP_REG_LAST)
 375                 return -EINVAL;
 376
 377         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 378         if (!cmd)
 379                 return -ENOMEM;
 380
 381         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 382         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 383
 384         kfree(cmd);
 385         return ret;
 386 }
 387
 388 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 389                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 390                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 391 {
 392         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 393         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 394         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 395         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 396
 397         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 398         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 399         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 400 }
 401
 402 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 403 {
 404         int ret;
 405
 406         /*
 407          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 408          * physical) for xgmi ta <-> Driver
 409          */
 410         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 411                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 412                                       &psp->xgmi_context.xgmi_shared_bo,
 413                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 414                                       &psp->xgmi_context.xgmi_shared_buf);
 415
 416         return ret;
 417 }
 418
 419 static int psp_xgmi_load(struct psp_context *psp)
 420 {
 421         int ret;
 422         struct psp_gfx_cmd_resp *cmd;
 423
 424         /*
 425          * TODO: bypass the loading in sriov for now
 426          */
 427         if (amdgpu_sriov_vf(psp->adev))
 428                 return 0;
 429
 430         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 431         if (!cmd)
 432                 return -ENOMEM;
 433
 434         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 435         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 436
 437         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 438                                       psp->xgmi_context.xgmi_shared_mc_addr,
 439                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 440
 441         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 442                                  psp->fence_buf_mc_addr);
 443
 444         if (!ret) {
 445                 psp->xgmi_context.initialized = 1;
 446                 psp->xgmi_context.session_id = cmd->resp.session_id;
 447         }
 448
 449         kfree(cmd);
 450
 451         return ret;
 452 }
 453
 454 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 455                                             uint32_t xgmi_session_id)
 456 {
 457         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 458         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 459 }
 460
 461 static int psp_xgmi_unload(struct psp_context *psp)
 462 {
 463         int ret;
 464         struct psp_gfx_cmd_resp *cmd;
 465
 466         /*
 467          * TODO: bypass the unloading in sriov for now
 468          */
 469         if (amdgpu_sriov_vf(psp->adev))
 470                 return 0;
 471
 472         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 473         if (!cmd)
 474                 return -ENOMEM;
 475
 476         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 477
 478         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 479                                  psp->fence_buf_mc_addr);
 480
 481         kfree(cmd);
 482
 483         return ret;
 484 }
 485
 486 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 487                                             uint32_t ta_cmd_id,
 488                                             uint32_t xgmi_session_id)
 489 {
 490         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 491         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 492         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 493         /* Note: cmd_invoke_cmd.buf is not used for now */
 494 }
 495
 496 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 497 {
 498         int ret;
 499         struct psp_gfx_cmd_resp *cmd;
 500
 501         /*
 502          * TODO: bypass the loading in sriov for now
 503         */
 504         if (amdgpu_sriov_vf(psp->adev))
 505                 return 0;
 506
 507         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 508         if (!cmd)
 509                 return -ENOMEM;
 510
 511         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 512                                         psp->xgmi_context.session_id);
 513
 514         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 515                                  psp->fence_buf_mc_addr);
 516
 517         kfree(cmd);
 518
 519         return ret;
 520 }
 521
 522 static int psp_xgmi_terminate(struct psp_context *psp)
 523 {
 524         int ret;
 525
 526         if (!psp->xgmi_context.initialized)
 527                 return 0;
 528
 529         ret = psp_xgmi_unload(psp);
 530         if (ret)
 531                 return ret;
 532
 533         psp->xgmi_context.initialized = 0;
 534
 535         /* free xgmi shared memory */
 536         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 537                         &psp->xgmi_context.xgmi_shared_mc_addr,
 538                         &psp->xgmi_context.xgmi_shared_buf);
 539
 540         return 0;
 541 }
 542
 543 static int psp_xgmi_initialize(struct psp_context *psp)
 544 {
 545         struct ta_xgmi_shared_memory *xgmi_cmd;
 546         int ret;
 547
 548         if (!psp->adev->psp.ta_fw)
 549                 return -ENOENT;
 550
 551         if (!psp->xgmi_context.initialized) {
 552                 ret = psp_xgmi_init_shared_buf(psp);
 553                 if (ret)
 554                         return ret;
 555         }
 556
 557         /* Load XGMI TA */
 558         ret = psp_xgmi_load(psp);
 559         if (ret)
 560                 return ret;
 561
 562         /* Initialize XGMI session */
 563         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 564         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 565         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 566
 567         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 568
 569         return ret;
 570 }
 571
 572 // ras begin
 573 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 574                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 575                 uint32_t ras_ta_size, uint32_t shared_size)
 576 {
 577         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 578         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 579         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 580         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 581
 582         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 583         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 584         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 585 }
 586
 587 static int psp_ras_init_shared_buf(struct psp_context *psp)
 588 {
 589         int ret;
 590
 591         /*
 592          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 593          * physical) for ras ta <-> Driver
 594          */
 595         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 596                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 597                         &psp->ras.ras_shared_bo,
 598                         &psp->ras.ras_shared_mc_addr,
 599                         &psp->ras.ras_shared_buf);
 600
 601         return ret;
 602 }
 603
 604 static int psp_ras_load(struct psp_context *psp)
 605 {
 606         int ret;
 607         struct psp_gfx_cmd_resp *cmd;
 608
 609         /*
 610          * TODO: bypass the loading in sriov for now
 611          */
 612         if (amdgpu_sriov_vf(psp->adev))
 613                 return 0;
 614
 615         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 616         if (!cmd)
 617                 return -ENOMEM;
 618
 619         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 620         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 621
 622         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 623                         psp->ras.ras_shared_mc_addr,
 624                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 625
 626         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 627                         psp->fence_buf_mc_addr);
 628
 629         if (!ret) {
 630                 psp->ras.ras_initialized = 1;
 631                 psp->ras.session_id = cmd->resp.session_id;
 632         }
 633
 634         kfree(cmd);
 635
 636         return ret;
 637 }
 638
 639 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 640                                                 uint32_t ras_session_id)
 641 {
 642         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 643         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 644 }
 645
 646 static int psp_ras_unload(struct psp_context *psp)
 647 {
 648         int ret;
 649         struct psp_gfx_cmd_resp *cmd;
 650
 651         /*
 652          * TODO: bypass the unloading in sriov for now
 653          */
 654         if (amdgpu_sriov_vf(psp->adev))
 655                 return 0;
 656
 657         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 658         if (!cmd)
 659                 return -ENOMEM;
 660
 661         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 662
 663         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 664                         psp->fence_buf_mc_addr);
 665
 666         kfree(cmd);
 667
 668         return ret;
 669 }
 670
 671 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 672                 uint32_t ta_cmd_id,
 673                 uint32_t ras_session_id)
 674 {
 675         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 676         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 677         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 678         /* Note: cmd_invoke_cmd.buf is not used for now */
 679 }
 680
 681 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 682 {
 683         int ret;
 684         struct psp_gfx_cmd_resp *cmd;
 685
 686         /*
 687          * TODO: bypass the loading in sriov for now
 688          */
 689         if (amdgpu_sriov_vf(psp->adev))
 690                 return 0;
 691
 692         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 693         if (!cmd)
 694                 return -ENOMEM;
 695
 696         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 697                         psp->ras.session_id);
 698
 699         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 700                         psp->fence_buf_mc_addr);
 701
 702         kfree(cmd);
 703
 704         return ret;
 705 }
 706
 707 int psp_ras_enable_features(struct psp_context *psp,
 708                 union ta_ras_cmd_input *info, bool enable)
 709 {
 710         struct ta_ras_shared_memory *ras_cmd;
 711         int ret;
 712
 713         if (!psp->ras.ras_initialized)
 714                 return -EINVAL;
 715
 716         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 717         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 718
 719         if (enable)
 720                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 721         else
 722                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 723
 724         ras_cmd->ras_in_message = *info;
 725
 726         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 727         if (ret)
 728                 return -EINVAL;
 729
 730         return ras_cmd->ras_status;
 731 }
 732
 733 static int psp_ras_terminate(struct psp_context *psp)
 734 {
 735         int ret;
 736
 737         if (!psp->ras.ras_initialized)
 738                 return 0;
 739
 740         ret = psp_ras_unload(psp);
 741         if (ret)
 742                 return ret;
 743
 744         psp->ras.ras_initialized = 0;
 745
 746         /* free ras shared memory */
 747         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 748                         &psp->ras.ras_shared_mc_addr,
 749                         &psp->ras.ras_shared_buf);
 750
 751         return 0;
 752 }
 753
 754 static int psp_ras_initialize(struct psp_context *psp)
 755 {
 756         int ret;
 757
 758         if (!psp->ras.ras_initialized) {
 759                 ret = psp_ras_init_shared_buf(psp);
 760                 if (ret)
 761                         return ret;
 762         }
 763
 764         ret = psp_ras_load(psp);
 765         if (ret)
 766                 return ret;
 767
 768         return 0;
 769 }
 770 // ras end
 771
 772 static int psp_hw_start(struct psp_context *psp)
 773 {
 774         struct amdgpu_device *adev = psp->adev;
 775         int ret;
 776
 777         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 778                 if (psp->kdb_bin_size &&
 779                     (psp->funcs->bootloader_load_kdb != NULL)) {
 780                         ret = psp_bootloader_load_kdb(psp);
 781                         if (ret) {
 782                                 DRM_ERROR("PSP load kdb failed!\n");
 783                                 return ret;
 784                         }
 785                 }
 786
 787                 ret = psp_bootloader_load_sysdrv(psp);
 788                 if (ret) {
 789                         DRM_ERROR("PSP load sysdrv failed!\n");
 790                         return ret;
 791                 }
 792
 793                 ret = psp_bootloader_load_sos(psp);
 794                 if (ret) {
 795                         DRM_ERROR("PSP load sos failed!\n");
 796                         return ret;
 797                 }
 798         }
 799
 800         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 801         if (ret) {
 802                 DRM_ERROR("PSP create ring failed!\n");
 803                 return ret;
 804         }
 805
 806         ret = psp_tmr_init(psp);
 807         if (ret) {
 808                 DRM_ERROR("PSP tmr init failed!\n");
 809                 return ret;
 810         }
 811
 812         ret = psp_tmr_load(psp);
 813         if (ret) {
 814                 DRM_ERROR("PSP load tmr failed!\n");
 815                 return ret;
 816         }
 817
 818         ret = psp_asd_init(psp);
 819         if (ret) {
 820                 DRM_ERROR("PSP asd init failed!\n");
 821                 return ret;
 822         }
 823
 824         ret = psp_asd_load(psp);
 825         if (ret) {
 826                 DRM_ERROR("PSP load asd failed!\n");
 827                 return ret;
 828         }
 829
 830         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 831                 ret = psp_xgmi_initialize(psp);
 832                 /* Warning the XGMI seesion initialize failure
 833                  * Instead of stop driver initialization
 834                  */
 835                 if (ret)
 836                         dev_err(psp->adev->dev,
 837                                 "XGMI: Failed to initialize XGMI session\n");
 838         }
 839
 840         if (psp->adev->psp.ta_fw) {
 841                 ret = psp_ras_initialize(psp);
 842                 if (ret)
 843                         dev_err(psp->adev->dev,
 844                                         "RAS: Failed to initialize RAS\n");
 845         }
 846
 847         return 0;
 848 }
 849
 850 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 851                            enum psp_gfx_fw_type *type)
 852 {
 853         switch (ucode->ucode_id) {
 854         case AMDGPU_UCODE_ID_SDMA0:
 855                 *type = GFX_FW_TYPE_SDMA0;
 856                 break;
 857         case AMDGPU_UCODE_ID_SDMA1:
 858                 *type = GFX_FW_TYPE_SDMA1;
 859                 break;
 860         case AMDGPU_UCODE_ID_SDMA2:
 861                 *type = GFX_FW_TYPE_SDMA2;
 862                 break;
 863         case AMDGPU_UCODE_ID_SDMA3:
 864                 *type = GFX_FW_TYPE_SDMA3;
 865                 break;
 866         case AMDGPU_UCODE_ID_SDMA4:
 867                 *type = GFX_FW_TYPE_SDMA4;
 868                 break;
 869         case AMDGPU_UCODE_ID_SDMA5:
 870                 *type = GFX_FW_TYPE_SDMA5;
 871                 break;
 872         case AMDGPU_UCODE_ID_SDMA6:
 873                 *type = GFX_FW_TYPE_SDMA6;
 874                 break;
 875         case AMDGPU_UCODE_ID_SDMA7:
 876                 *type = GFX_FW_TYPE_SDMA7;
 877                 break;
 878         case AMDGPU_UCODE_ID_CP_CE:
 879                 *type = GFX_FW_TYPE_CP_CE;
 880                 break;
 881         case AMDGPU_UCODE_ID_CP_PFP:
 882                 *type = GFX_FW_TYPE_CP_PFP;
 883                 break;
 884         case AMDGPU_UCODE_ID_CP_ME:
 885                 *type = GFX_FW_TYPE_CP_ME;
 886                 break;
 887         case AMDGPU_UCODE_ID_CP_MEC1:
 888                 *type = GFX_FW_TYPE_CP_MEC;
 889                 break;
 890         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 891                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 892                 break;
 893         case AMDGPU_UCODE_ID_CP_MEC2:
 894                 *type = GFX_FW_TYPE_CP_MEC;
 895                 break;
 896         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 897                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 898                 break;
 899         case AMDGPU_UCODE_ID_RLC_G:
 900                 *type = GFX_FW_TYPE_RLC_G;
 901                 break;
 902         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 903                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 904                 break;
 905         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 906                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 907                 break;
 908         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 909                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 910                 break;
 911         case AMDGPU_UCODE_ID_SMC:
 912                 *type = GFX_FW_TYPE_SMU;
 913                 break;
 914         case AMDGPU_UCODE_ID_UVD:
 915                 *type = GFX_FW_TYPE_UVD;
 916                 break;
 917         case AMDGPU_UCODE_ID_UVD1:
 918                 *type = GFX_FW_TYPE_UVD1;
 919                 break;
 920         case AMDGPU_UCODE_ID_VCE:
 921                 *type = GFX_FW_TYPE_VCE;
 922                 break;
 923         case AMDGPU_UCODE_ID_VCN:
 924                 *type = GFX_FW_TYPE_VCN;
 925                 break;
 926         case AMDGPU_UCODE_ID_DMCU_ERAM:
 927                 *type = GFX_FW_TYPE_DMCU_ERAM;
 928                 break;
 929         case AMDGPU_UCODE_ID_DMCU_INTV:
 930                 *type = GFX_FW_TYPE_DMCU_ISR;
 931                 break;
 932         case AMDGPU_UCODE_ID_VCN0_RAM:
 933                 *type = GFX_FW_TYPE_VCN0_RAM;
 934                 break;
 935         case AMDGPU_UCODE_ID_VCN1_RAM:
 936                 *type = GFX_FW_TYPE_VCN1_RAM;
 937                 break;
 938         case AMDGPU_UCODE_ID_MAXIMUM:
 939         default:
 940                 return -EINVAL;
 941         }
 942
 943         return 0;
 944 }
 945
 946 static void psp_print_fw_hdr(struct psp_context *psp,
 947                              struct amdgpu_firmware_info *ucode)
 948 {
 949         struct amdgpu_device *adev = psp->adev;
 950         const struct sdma_firmware_header_v1_0 *sdma_hdr =
 951                 (const struct sdma_firmware_header_v1_0 *)
 952                 adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
 953         const struct gfx_firmware_header_v1_0 *ce_hdr =
 954                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 955         const struct gfx_firmware_header_v1_0 *pfp_hdr =
 956                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 957         const struct gfx_firmware_header_v1_0 *me_hdr =
 958                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 959         const struct gfx_firmware_header_v1_0 *mec_hdr =
 960                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 961         const struct rlc_firmware_header_v2_0 *rlc_hdr =
 962                 (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 963         const struct smc_firmware_header_v1_0 *smc_hdr =
 964                 (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
 965
 966         switch (ucode->ucode_id) {
 967         case AMDGPU_UCODE_ID_SDMA0:
 968         case AMDGPU_UCODE_ID_SDMA1:
 969         case AMDGPU_UCODE_ID_SDMA2:
 970         case AMDGPU_UCODE_ID_SDMA3:
 971         case AMDGPU_UCODE_ID_SDMA4:
 972         case AMDGPU_UCODE_ID_SDMA5:
 973         case AMDGPU_UCODE_ID_SDMA6:
 974         case AMDGPU_UCODE_ID_SDMA7:
 975                 amdgpu_ucode_print_sdma_hdr(&sdma_hdr->header);
 976                 break;
 977         case AMDGPU_UCODE_ID_CP_CE:
 978                 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
 979                 break;
 980         case AMDGPU_UCODE_ID_CP_PFP:
 981                 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
 982                 break;
 983         case AMDGPU_UCODE_ID_CP_ME:
 984                 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
 985                 break;
 986         case AMDGPU_UCODE_ID_CP_MEC1:
 987                 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
 988                 break;
 989         case AMDGPU_UCODE_ID_RLC_G:
 990                 amdgpu_ucode_print_rlc_hdr(&rlc_hdr->header);
 991                 break;
 992         case AMDGPU_UCODE_ID_SMC:
 993                 amdgpu_ucode_print_smc_hdr(&smc_hdr->header);
 994                 break;
 995         default:
 996                 break;
 997         }
 998 }
 999
1000 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1001                                        struct psp_gfx_cmd_resp *cmd)
1002 {
1003         int ret;
1004         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1005
1006         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1007
1008         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1009         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1010         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1011         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1012
1013         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1014         if (ret)
1015                 DRM_ERROR("Unknown firmware type\n");
1016
1017         return ret;
1018 }
1019
1020 static int psp_execute_np_fw_load(struct psp_context *psp,
1021                                struct amdgpu_firmware_info *ucode)
1022 {
1023         int ret = 0;
1024
1025         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1026         if (ret)
1027                 return ret;
1028
1029         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1030                                  psp->fence_buf_mc_addr);
1031
1032         return ret;
1033 }
1034
1035 static int psp_np_fw_load(struct psp_context *psp)
1036 {
1037         int i, ret;
1038         struct amdgpu_firmware_info *ucode;
1039         struct amdgpu_device* adev = psp->adev;
1040
1041         if (psp->autoload_supported) {
1042                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1043                 if (!ucode->fw)
1044                         goto out;
1045
1046                 ret = psp_execute_np_fw_load(psp, ucode);
1047                 if (ret)
1048                         return ret;
1049         }
1050
1051 out:
1052         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1053                 ucode = &adev->firmware.ucode[i];
1054                 if (!ucode->fw)
1055                         continue;
1056
1057                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1058                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1059                         continue;
1060
1061                 if (amdgpu_sriov_vf(adev) &&
1062                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1063                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1064                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1065                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1066                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1067                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1068                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1069                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1070                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
1071                         /*skip ucode loading in SRIOV VF */
1072                         continue;
1073
1074                 if (psp->autoload_supported &&
1075                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1076                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1077                         /* skip mec JT when autoload is enabled */
1078                         continue;
1079                 /* Renoir only needs to load mec jump table one time */
1080                 if (adev->asic_type == CHIP_RENOIR &&
1081                     ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)
1082                         continue;
1083
1084                 psp_print_fw_hdr(psp, ucode);
1085
1086                 ret = psp_execute_np_fw_load(psp, ucode);
1087                 if (ret)
1088                         return ret;
1089
1090                 /* Start rlc autoload after psp recieved all the gfx firmware */
1091                 if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
1092                         ret = psp_rlc_autoload(psp);
1093                         if (ret) {
1094                                 DRM_ERROR("Failed to start rlc autoload\n");
1095                                 return ret;
1096                         }
1097                 }
1098 #if 0
1099                 /* check if firmware loaded sucessfully */
1100                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1101                         return -EINVAL;
1102 #endif
1103         }
1104
1105         return 0;
1106 }
1107
1108 static int psp_load_fw(struct amdgpu_device *adev)
1109 {
1110         int ret;
1111         struct psp_context *psp = &adev->psp;
1112
1113         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1114                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1115                 goto skip_memalloc;
1116         }
1117
1118         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1119         if (!psp->cmd)
1120                 return -ENOMEM;
1121
1122         /* this fw pri bo is not used under SRIOV */
1123         if (!amdgpu_sriov_vf(psp->adev)) {
1124                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1125                                               AMDGPU_GEM_DOMAIN_GTT,
1126                                               &psp->fw_pri_bo,
1127                                               &psp->fw_pri_mc_addr,
1128                                               &psp->fw_pri_buf);
1129                 if (ret)
1130                         goto failed;
1131         }
1132
1133         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1134                                         AMDGPU_GEM_DOMAIN_VRAM,
1135                                         &psp->fence_buf_bo,
1136                                         &psp->fence_buf_mc_addr,
1137                                         &psp->fence_buf);
1138         if (ret)
1139                 goto failed;
1140
1141         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1142                                       AMDGPU_GEM_DOMAIN_VRAM,
1143                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1144                                       (void **)&psp->cmd_buf_mem);
1145         if (ret)
1146                 goto failed;
1147
1148         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1149
1150         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1151         if (ret) {
1152                 DRM_ERROR("PSP ring init failed!\n");
1153                 goto failed;
1154         }
1155
1156 skip_memalloc:
1157         ret = psp_hw_start(psp);
1158         if (ret)
1159                 goto failed;
1160
1161         ret = psp_np_fw_load(psp);
1162         if (ret)
1163                 goto failed;
1164
1165         return 0;
1166
1167 failed:
1168         /*
1169          * all cleanup jobs (xgmi terminate, ras terminate,
1170          * ring destroy, cmd/fence/fw buffers destory,
1171          * psp->cmd destory) are delayed to psp_hw_fini
1172          */
1173         return ret;
1174 }
1175
1176 static int psp_hw_init(void *handle)
1177 {
1178         int ret;
1179         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1180
1181         mutex_lock(&adev->firmware.mutex);
1182         /*
1183          * This sequence is just used on hw_init only once, no need on
1184          * resume.
1185          */
1186         ret = amdgpu_ucode_init_bo(adev);
1187         if (ret)
1188                 goto failed;
1189
1190         ret = psp_load_fw(adev);
1191         if (ret) {
1192                 DRM_ERROR("PSP firmware loading failed\n");
1193                 goto failed;
1194         }
1195
1196         mutex_unlock(&adev->firmware.mutex);
1197         return 0;
1198
1199 failed:
1200         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1201         mutex_unlock(&adev->firmware.mutex);
1202         return -EINVAL;
1203 }
1204
1205 static int psp_hw_fini(void *handle)
1206 {
1207         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1208         struct psp_context *psp = &adev->psp;
1209
1210         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1211             psp->xgmi_context.initialized == 1)
1212                 psp_xgmi_terminate(psp);
1213
1214         if (psp->adev->psp.ta_fw)
1215                 psp_ras_terminate(psp);
1216
1217         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1218
1219         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, NULL);
1220         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1221                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1222         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1223                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1224         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1225                               &psp->asd_shared_buf);
1226         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1227                               (void **)&psp->cmd_buf_mem);
1228
1229         kfree(psp->cmd);
1230         psp->cmd = NULL;
1231
1232         return 0;
1233 }
1234
1235 static int psp_suspend(void *handle)
1236 {
1237         int ret;
1238         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1239         struct psp_context *psp = &adev->psp;
1240
1241         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1242             psp->xgmi_context.initialized == 1) {
1243                 ret = psp_xgmi_terminate(psp);
1244                 if (ret) {
1245                         DRM_ERROR("Failed to terminate xgmi ta\n");
1246                         return ret;
1247                 }
1248         }
1249
1250         if (psp->adev->psp.ta_fw) {
1251                 ret = psp_ras_terminate(psp);
1252                 if (ret) {
1253                         DRM_ERROR("Failed to terminate ras ta\n");
1254                         return ret;
1255                 }
1256         }
1257
1258         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1259         if (ret) {
1260                 DRM_ERROR("PSP ring stop failed\n");
1261                 return ret;
1262         }
1263
1264         return 0;
1265 }
1266
1267 static int psp_resume(void *handle)
1268 {
1269         int ret;
1270         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1271         struct psp_context *psp = &adev->psp;
1272
1273         DRM_INFO("PSP is resuming...\n");
1274
1275         mutex_lock(&adev->firmware.mutex);
1276
1277         ret = psp_hw_start(psp);
1278         if (ret)
1279                 goto failed;
1280
1281         ret = psp_np_fw_load(psp);
1282         if (ret)
1283                 goto failed;
1284
1285         mutex_unlock(&adev->firmware.mutex);
1286
1287         return 0;
1288
1289 failed:
1290         DRM_ERROR("PSP resume failed\n");
1291         mutex_unlock(&adev->firmware.mutex);
1292         return ret;
1293 }
1294
1295 int psp_gpu_reset(struct amdgpu_device *adev)
1296 {
1297         int ret;
1298
1299         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1300                 return 0;
1301
1302         mutex_lock(&adev->psp.mutex);
1303         ret = psp_mode1_reset(&adev->psp);
1304         mutex_unlock(&adev->psp.mutex);
1305
1306         return ret;
1307 }
1308
1309 int psp_rlc_autoload_start(struct psp_context *psp)
1310 {
1311         int ret;
1312         struct psp_gfx_cmd_resp *cmd;
1313
1314         if (amdgpu_sriov_vf(psp->adev))
1315                 return 0;
1316
1317         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1318         if (!cmd)
1319                 return -ENOMEM;
1320
1321         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1322
1323         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1324                                  psp->fence_buf_mc_addr);
1325         kfree(cmd);
1326         return ret;
1327 }
1328
1329 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1330                         uint64_t cmd_gpu_addr, int cmd_size)
1331 {
1332         struct amdgpu_firmware_info ucode = {0};
1333
1334         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1335                 AMDGPU_UCODE_ID_VCN0_RAM;
1336         ucode.mc_addr = cmd_gpu_addr;
1337         ucode.ucode_size = cmd_size;
1338
1339         return psp_execute_np_fw_load(&adev->psp, &ucode);
1340 }
1341
1342 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1343                                         enum AMDGPU_UCODE_ID ucode_type)
1344 {
1345         struct amdgpu_firmware_info *ucode = NULL;
1346
1347         if (!adev->firmware.fw_size)
1348                 return false;
1349
1350         ucode = &adev->firmware.ucode[ucode_type];
1351         if (!ucode->fw || !ucode->ucode_size)
1352                 return false;
1353
1354         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1355 }
1356
1357 static int psp_set_clockgating_state(void *handle,
1358                                      enum amd_clockgating_state state)
1359 {
1360         return 0;
1361 }
1362
1363 static int psp_set_powergating_state(void *handle,
1364                                      enum amd_powergating_state state)
1365 {
1366         return 0;
1367 }
1368
1369 const struct amd_ip_funcs psp_ip_funcs = {
1370         .name = "psp",
1371         .early_init = psp_early_init,
1372         .late_init = NULL,
1373         .sw_init = psp_sw_init,
1374         .sw_fini = psp_sw_fini,
1375         .hw_init = psp_hw_init,
1376         .hw_fini = psp_hw_fini,
1377         .suspend = psp_suspend,
1378         .resume = psp_resume,
1379         .is_idle = NULL,
1380         .check_soft_reset = NULL,
1381         .wait_for_idle = NULL,
1382         .soft_reset = NULL,
1383         .set_clockgating_state = psp_set_clockgating_state,
1384         .set_powergating_state = psp_set_powergating_state,
1385 };
1386
1387 static const struct amdgpu_psp_funcs psp_funcs = {
1388         .check_fw_loading_status = psp_check_fw_loading_status,
1389 };
1390
1391 static void psp_set_funcs(struct amdgpu_device *adev)
1392 {
1393         if (NULL == adev->firmware.funcs)
1394                 adev->firmware.funcs = &psp_funcs;
1395 }
1396
1397 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1398 {
1399         .type = AMD_IP_BLOCK_TYPE_PSP,
1400         .major = 3,
1401         .minor = 1,
1402         .rev = 0,
1403         .funcs = &psp_ip_funcs,
1404 };
1405
1406 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1407 {
1408         .type = AMD_IP_BLOCK_TYPE_PSP,
1409         .major = 10,
1410         .minor = 0,
1411         .rev = 0,
1412         .funcs = &psp_ip_funcs,
1413 };
1414
1415 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1416 {
1417         .type = AMD_IP_BLOCK_TYPE_PSP,
1418         .major = 11,
1419         .minor = 0,
1420         .rev = 0,
1421         .funcs = &psp_ip_funcs,
1422 };
1423
1424 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1425 {
1426         .type = AMD_IP_BLOCK_TYPE_PSP,
1427         .major = 12,
1428         .minor = 0,
1429         .rev = 0,
1430         .funcs = &psp_ip_funcs,
1431 };