drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 static void psp_set_funcs(struct amdgpu_device *adev);
  38
  39 static int psp_early_init(void *handle)
  40 {
  41         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  42         struct psp_context *psp = &adev->psp;
  43
  44         psp_set_funcs(adev);
  45
  46         switch (adev->asic_type) {
  47         case CHIP_VEGA10:
  48         case CHIP_VEGA12:
  49                 psp_v3_1_set_psp_funcs(psp);
  50                 psp->autoload_supported = false;
  51                 break;
  52         case CHIP_RAVEN:
  53                 psp_v10_0_set_psp_funcs(psp);
  54                 psp->autoload_supported = false;
  55                 break;
  56         case CHIP_VEGA20:
  57         case CHIP_ARCTURUS:
  58                 psp_v11_0_set_psp_funcs(psp);
  59                 psp->autoload_supported = false;
  60                 break;
  61         case CHIP_NAVI10:
  62         case CHIP_NAVI14:
  63         case CHIP_NAVI12:
  64                 psp_v11_0_set_psp_funcs(psp);
  65                 psp->autoload_supported = true;
  66                 break;
  67         case CHIP_RENOIR:
  68                 psp_v12_0_set_psp_funcs(psp);
  69                 break;
  70         default:
  71                 return -EINVAL;
  72         }
  73
  74         psp->adev = adev;
  75
  76         return 0;
  77 }
  78
  79 static int psp_sw_init(void *handle)
  80 {
  81         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  82         struct psp_context *psp = &adev->psp;
  83         int ret;
  84
  85         ret = psp_init_microcode(psp);
  86         if (ret) {
  87                 DRM_ERROR("Failed to load psp firmware!\n");
  88                 return ret;
  89         }
  90
  91         return 0;
  92 }
  93
  94 static int psp_sw_fini(void *handle)
  95 {
  96         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  97
  98         release_firmware(adev->psp.sos_fw);
  99         adev->psp.sos_fw = NULL;
 100         release_firmware(adev->psp.asd_fw);
 101         adev->psp.asd_fw = NULL;
 102         if (adev->psp.ta_fw) {
 103                 release_firmware(adev->psp.ta_fw);
 104                 adev->psp.ta_fw = NULL;
 105         }
 106         return 0;
 107 }
 108
 109 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 110                  uint32_t reg_val, uint32_t mask, bool check_changed)
 111 {
 112         uint32_t val;
 113         int i;
 114         struct amdgpu_device *adev = psp->adev;
 115
 116         for (i = 0; i < adev->usec_timeout; i++) {
 117                 val = RREG32(reg_index);
 118                 if (check_changed) {
 119                         if (val != reg_val)
 120                                 return 0;
 121                 } else {
 122                         if ((val & mask) == reg_val)
 123                                 return 0;
 124                 }
 125                 udelay(1);
 126         }
 127
 128         return -ETIME;
 129 }
 130
 131 static int
 132 psp_cmd_submit_buf(struct psp_context *psp,
 133                    struct amdgpu_firmware_info *ucode,
 134                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 135 {
 136         int ret;
 137         int index;
 138         int timeout = 2000;
 139
 140         mutex_lock(&psp->mutex);
 141
 142         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 143
 144         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 145
 146         index = atomic_inc_return(&psp->fence_value);
 147         ret = psp_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 148         if (ret) {
 149                 atomic_dec(&psp->fence_value);
 150                 mutex_unlock(&psp->mutex);
 151                 return ret;
 152         }
 153
 154         while (*((unsigned int *)psp->fence_buf) != index) {
 155                 if (--timeout == 0)
 156                         break;
 157                 msleep(1);
 158         }
 159
 160         /* In some cases, psp response status is not 0 even there is no
 161          * problem while the command is submitted. Some version of PSP FW
 162          * doesn't write 0 to that field.
 163          * So here we would like to only print a warning instead of an error
 164          * during psp initialization to avoid breaking hw_init and it doesn't
 165          * return -EINVAL.
 166          */
 167         if (psp->cmd_buf_mem->resp.status || !timeout) {
 168                 if (ucode)
 169                         DRM_WARN("failed to load ucode id (%d) ",
 170                                   ucode->ucode_id);
 171                 DRM_WARN("psp command failed and response status is (0x%X)\n",
 172                           psp->cmd_buf_mem->resp.status & GFX_CMD_STATUS_MASK);
 173                 if (!timeout) {
 174                         mutex_unlock(&psp->mutex);
 175                         return -EINVAL;
 176                 }
 177         }
 178
 179         /* get xGMI session id from response buffer */
 180         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 181
 182         if (ucode) {
 183                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 184                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 185         }
 186         mutex_unlock(&psp->mutex);
 187
 188         return ret;
 189 }
 190
 191 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 192                                  struct psp_gfx_cmd_resp *cmd,
 193                                  uint64_t tmr_mc, uint32_t size)
 194 {
 195         if (psp_support_vmr_ring(psp))
 196                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 197         else
 198                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 199         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 200         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 201         cmd->cmd.cmd_setup_tmr.buf_size = size;
 202 }
 203
 204 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 205                                       uint64_t pri_buf_mc, uint32_t size)
 206 {
 207         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 208         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 209         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 210         cmd->cmd.cmd_load_toc.toc_size = size;
 211 }
 212
 213 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 214 static int psp_load_toc(struct psp_context *psp,
 215                         uint32_t *tmr_size)
 216 {
 217         int ret;
 218         struct psp_gfx_cmd_resp *cmd;
 219
 220         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 221         if (!cmd)
 222                 return -ENOMEM;
 223         /* Copy toc to psp firmware private buffer */
 224         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 225         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 226
 227         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 228
 229         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 230                                  psp->fence_buf_mc_addr);
 231         if (!ret)
 232                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 233         kfree(cmd);
 234         return ret;
 235 }
 236
 237 /* Set up Trusted Memory Region */
 238 static int psp_tmr_init(struct psp_context *psp)
 239 {
 240         int ret;
 241         int tmr_size;
 242         void *tmr_buf;
 243         void **pptr;
 244
 245         /*
 246          * According to HW engineer, they prefer the TMR address be "naturally
 247          * aligned" , e.g. the start address be an integer divide of TMR size.
 248          *
 249          * Note: this memory need be reserved till the driver
 250          * uninitializes.
 251          */
 252         tmr_size = PSP_TMR_SIZE;
 253
 254         /* For ASICs support RLC autoload, psp will parse the toc
 255          * and calculate the total size of TMR needed */
 256         if (!amdgpu_sriov_vf(psp->adev) &&
 257             psp->toc_start_addr &&
 258             psp->toc_bin_size &&
 259             psp->fw_pri_buf) {
 260                 ret = psp_load_toc(psp, &tmr_size);
 261                 if (ret) {
 262                         DRM_ERROR("Failed to load toc\n");
 263                         return ret;
 264                 }
 265         }
 266
 267         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 268         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 269                                       AMDGPU_GEM_DOMAIN_VRAM,
 270                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 271
 272         return ret;
 273 }
 274
 275 static int psp_tmr_load(struct psp_context *psp)
 276 {
 277         int ret;
 278         struct psp_gfx_cmd_resp *cmd;
 279
 280         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 281         if (!cmd)
 282                 return -ENOMEM;
 283
 284         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 285                              amdgpu_bo_size(psp->tmr_bo));
 286         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 287                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 288
 289         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 290                                  psp->fence_buf_mc_addr);
 291
 292         kfree(cmd);
 293
 294         return ret;
 295 }
 296
 297 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 298                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 299                                  uint32_t size, uint32_t shared_size)
 300 {
 301         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 302         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 303         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 304         cmd->cmd.cmd_load_ta.app_len = size;
 305
 306         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 307         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 308         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 309 }
 310
 311 static int psp_asd_init(struct psp_context *psp)
 312 {
 313         int ret;
 314
 315         /*
 316          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 317          * physical) for shared ASD <-> Driver
 318          */
 319         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 320                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 321                                       &psp->asd_shared_bo,
 322                                       &psp->asd_shared_mc_addr,
 323                                       &psp->asd_shared_buf);
 324
 325         return ret;
 326 }
 327
 328 static int psp_asd_load(struct psp_context *psp)
 329 {
 330         int ret;
 331         struct psp_gfx_cmd_resp *cmd;
 332
 333         /* If PSP version doesn't match ASD version, asd loading will be failed.
 334          * add workaround to bypass it for sriov now.
 335          * TODO: add version check to make it common
 336          */
 337         if (amdgpu_sriov_vf(psp->adev))
 338                 return 0;
 339
 340         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 341         if (!cmd)
 342                 return -ENOMEM;
 343
 344         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 345         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 346
 347         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 348                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 349
 350         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 351                                  psp->fence_buf_mc_addr);
 352
 353         kfree(cmd);
 354
 355         return ret;
 356 }
 357
 358 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 359                 uint32_t id, uint32_t value)
 360 {
 361         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 362         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 363         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 364 }
 365
 366 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 367                 uint32_t value)
 368 {
 369         struct psp_gfx_cmd_resp *cmd = NULL;
 370         int ret = 0;
 371
 372         if (reg >= PSP_REG_LAST)
 373                 return -EINVAL;
 374
 375         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 376         if (!cmd)
 377                 return -ENOMEM;
 378
 379         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 380         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 381
 382         kfree(cmd);
 383         return ret;
 384 }
 385
 386 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 387                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 388                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 389 {
 390         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 391         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 392         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 393         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 394
 395         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 396         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 397         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 398 }
 399
 400 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 401 {
 402         int ret;
 403
 404         /*
 405          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 406          * physical) for xgmi ta <-> Driver
 407          */
 408         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 409                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 410                                       &psp->xgmi_context.xgmi_shared_bo,
 411                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 412                                       &psp->xgmi_context.xgmi_shared_buf);
 413
 414         return ret;
 415 }
 416
 417 static int psp_xgmi_load(struct psp_context *psp)
 418 {
 419         int ret;
 420         struct psp_gfx_cmd_resp *cmd;
 421
 422         /*
 423          * TODO: bypass the loading in sriov for now
 424          */
 425         if (amdgpu_sriov_vf(psp->adev))
 426                 return 0;
 427
 428         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 429         if (!cmd)
 430                 return -ENOMEM;
 431
 432         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 433         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 434
 435         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 436                                       psp->xgmi_context.xgmi_shared_mc_addr,
 437                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 438
 439         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 440                                  psp->fence_buf_mc_addr);
 441
 442         if (!ret) {
 443                 psp->xgmi_context.initialized = 1;
 444                 psp->xgmi_context.session_id = cmd->resp.session_id;
 445         }
 446
 447         kfree(cmd);
 448
 449         return ret;
 450 }
 451
 452 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 453                                             uint32_t xgmi_session_id)
 454 {
 455         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 456         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 457 }
 458
 459 static int psp_xgmi_unload(struct psp_context *psp)
 460 {
 461         int ret;
 462         struct psp_gfx_cmd_resp *cmd;
 463
 464         /*
 465          * TODO: bypass the unloading in sriov for now
 466          */
 467         if (amdgpu_sriov_vf(psp->adev))
 468                 return 0;
 469
 470         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 471         if (!cmd)
 472                 return -ENOMEM;
 473
 474         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 475
 476         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 477                                  psp->fence_buf_mc_addr);
 478
 479         kfree(cmd);
 480
 481         return ret;
 482 }
 483
 484 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 485                                             uint32_t ta_cmd_id,
 486                                             uint32_t xgmi_session_id)
 487 {
 488         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 489         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 490         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 491         /* Note: cmd_invoke_cmd.buf is not used for now */
 492 }
 493
 494 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 495 {
 496         int ret;
 497         struct psp_gfx_cmd_resp *cmd;
 498
 499         /*
 500          * TODO: bypass the loading in sriov for now
 501         */
 502         if (amdgpu_sriov_vf(psp->adev))
 503                 return 0;
 504
 505         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 506         if (!cmd)
 507                 return -ENOMEM;
 508
 509         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 510                                         psp->xgmi_context.session_id);
 511
 512         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 513                                  psp->fence_buf_mc_addr);
 514
 515         kfree(cmd);
 516
 517         return ret;
 518 }
 519
 520 static int psp_xgmi_terminate(struct psp_context *psp)
 521 {
 522         int ret;
 523
 524         if (!psp->xgmi_context.initialized)
 525                 return 0;
 526
 527         ret = psp_xgmi_unload(psp);
 528         if (ret)
 529                 return ret;
 530
 531         psp->xgmi_context.initialized = 0;
 532
 533         /* free xgmi shared memory */
 534         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 535                         &psp->xgmi_context.xgmi_shared_mc_addr,
 536                         &psp->xgmi_context.xgmi_shared_buf);
 537
 538         return 0;
 539 }
 540
 541 static int psp_xgmi_initialize(struct psp_context *psp)
 542 {
 543         struct ta_xgmi_shared_memory *xgmi_cmd;
 544         int ret;
 545
 546         if (!psp->adev->psp.ta_fw)
 547                 return -ENOENT;
 548
 549         if (!psp->xgmi_context.initialized) {
 550                 ret = psp_xgmi_init_shared_buf(psp);
 551                 if (ret)
 552                         return ret;
 553         }
 554
 555         /* Load XGMI TA */
 556         ret = psp_xgmi_load(psp);
 557         if (ret)
 558                 return ret;
 559
 560         /* Initialize XGMI session */
 561         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 562         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 563         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 564
 565         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 566
 567         return ret;
 568 }
 569
 570 // ras begin
 571 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 572                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 573                 uint32_t ras_ta_size, uint32_t shared_size)
 574 {
 575         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 576         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 577         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 578         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 579
 580         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 581         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 582         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 583 }
 584
 585 static int psp_ras_init_shared_buf(struct psp_context *psp)
 586 {
 587         int ret;
 588
 589         /*
 590          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 591          * physical) for ras ta <-> Driver
 592          */
 593         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 594                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 595                         &psp->ras.ras_shared_bo,
 596                         &psp->ras.ras_shared_mc_addr,
 597                         &psp->ras.ras_shared_buf);
 598
 599         return ret;
 600 }
 601
 602 static int psp_ras_load(struct psp_context *psp)
 603 {
 604         int ret;
 605         struct psp_gfx_cmd_resp *cmd;
 606
 607         /*
 608          * TODO: bypass the loading in sriov for now
 609          */
 610         if (amdgpu_sriov_vf(psp->adev))
 611                 return 0;
 612
 613         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 614         if (!cmd)
 615                 return -ENOMEM;
 616
 617         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 618         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 619
 620         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 621                         psp->ras.ras_shared_mc_addr,
 622                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 623
 624         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 625                         psp->fence_buf_mc_addr);
 626
 627         if (!ret) {
 628                 psp->ras.ras_initialized = 1;
 629                 psp->ras.session_id = cmd->resp.session_id;
 630         }
 631
 632         kfree(cmd);
 633
 634         return ret;
 635 }
 636
 637 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 638                                                 uint32_t ras_session_id)
 639 {
 640         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 641         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 642 }
 643
 644 static int psp_ras_unload(struct psp_context *psp)
 645 {
 646         int ret;
 647         struct psp_gfx_cmd_resp *cmd;
 648
 649         /*
 650          * TODO: bypass the unloading in sriov for now
 651          */
 652         if (amdgpu_sriov_vf(psp->adev))
 653                 return 0;
 654
 655         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 656         if (!cmd)
 657                 return -ENOMEM;
 658
 659         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 660
 661         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 662                         psp->fence_buf_mc_addr);
 663
 664         kfree(cmd);
 665
 666         return ret;
 667 }
 668
 669 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 670                 uint32_t ta_cmd_id,
 671                 uint32_t ras_session_id)
 672 {
 673         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 674         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 675         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 676         /* Note: cmd_invoke_cmd.buf is not used for now */
 677 }
 678
 679 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 680 {
 681         int ret;
 682         struct psp_gfx_cmd_resp *cmd;
 683
 684         /*
 685          * TODO: bypass the loading in sriov for now
 686          */
 687         if (amdgpu_sriov_vf(psp->adev))
 688                 return 0;
 689
 690         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 691         if (!cmd)
 692                 return -ENOMEM;
 693
 694         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 695                         psp->ras.session_id);
 696
 697         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 698                         psp->fence_buf_mc_addr);
 699
 700         kfree(cmd);
 701
 702         return ret;
 703 }
 704
 705 int psp_ras_enable_features(struct psp_context *psp,
 706                 union ta_ras_cmd_input *info, bool enable)
 707 {
 708         struct ta_ras_shared_memory *ras_cmd;
 709         int ret;
 710
 711         if (!psp->ras.ras_initialized)
 712                 return -EINVAL;
 713
 714         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 715         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 716
 717         if (enable)
 718                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 719         else
 720                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 721
 722         ras_cmd->ras_in_message = *info;
 723
 724         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 725         if (ret)
 726                 return -EINVAL;
 727
 728         return ras_cmd->ras_status;
 729 }
 730
 731 static int psp_ras_terminate(struct psp_context *psp)
 732 {
 733         int ret;
 734
 735         if (!psp->ras.ras_initialized)
 736                 return 0;
 737
 738         ret = psp_ras_unload(psp);
 739         if (ret)
 740                 return ret;
 741
 742         psp->ras.ras_initialized = 0;
 743
 744         /* free ras shared memory */
 745         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 746                         &psp->ras.ras_shared_mc_addr,
 747                         &psp->ras.ras_shared_buf);
 748
 749         return 0;
 750 }
 751
 752 static int psp_ras_initialize(struct psp_context *psp)
 753 {
 754         int ret;
 755
 756         if (!psp->ras.ras_initialized) {
 757                 ret = psp_ras_init_shared_buf(psp);
 758                 if (ret)
 759                         return ret;
 760         }
 761
 762         ret = psp_ras_load(psp);
 763         if (ret)
 764                 return ret;
 765
 766         return 0;
 767 }
 768 // ras end
 769
 770 // HDCP start
 771 static void psp_prep_hdcp_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 772                                           uint64_t hdcp_ta_mc,
 773                                           uint64_t hdcp_mc_shared,
 774                                           uint32_t hdcp_ta_size,
 775                                           uint32_t shared_size)
 776 {
 777         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 778         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(hdcp_ta_mc);
 779         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(hdcp_ta_mc);
 780         cmd->cmd.cmd_load_ta.app_len = hdcp_ta_size;
 781
 782         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
 783                 lower_32_bits(hdcp_mc_shared);
 784         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
 785                 upper_32_bits(hdcp_mc_shared);
 786         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 787 }
 788
 789 static int psp_hdcp_init_shared_buf(struct psp_context *psp)
 790 {
 791         int ret;
 792
 793         /*
 794          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 795          * physical) for hdcp ta <-> Driver
 796          */
 797         ret = amdgpu_bo_create_kernel(psp->adev, PSP_HDCP_SHARED_MEM_SIZE,
 798                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 799                                       &psp->hdcp_context.hdcp_shared_bo,
 800                                       &psp->hdcp_context.hdcp_shared_mc_addr,
 801                                       &psp->hdcp_context.hdcp_shared_buf);
 802
 803         return ret;
 804 }
 805
 806 static int psp_hdcp_load(struct psp_context *psp)
 807 {
 808         int ret;
 809         struct psp_gfx_cmd_resp *cmd;
 810
 811         /*
 812          * TODO: bypass the loading in sriov for now
 813          */
 814         if (amdgpu_sriov_vf(psp->adev))
 815                 return 0;
 816
 817         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 818         if (!cmd)
 819                 return -ENOMEM;
 820
 821         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 822         memcpy(psp->fw_pri_buf, psp->ta_hdcp_start_addr,
 823                psp->ta_hdcp_ucode_size);
 824
 825         psp_prep_hdcp_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 826                                       psp->hdcp_context.hdcp_shared_mc_addr,
 827                                       psp->ta_hdcp_ucode_size,
 828                                       PSP_HDCP_SHARED_MEM_SIZE);
 829
 830         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 831
 832         if (!ret) {
 833                 psp->hdcp_context.hdcp_initialized = 1;
 834                 psp->hdcp_context.session_id = cmd->resp.session_id;
 835         }
 836
 837         kfree(cmd);
 838
 839         return ret;
 840 }
 841 static int psp_hdcp_initialize(struct psp_context *psp)
 842 {
 843         int ret;
 844
 845         if (!psp->hdcp_context.hdcp_initialized) {
 846                 ret = psp_hdcp_init_shared_buf(psp);
 847                 if (ret)
 848                         return ret;
 849         }
 850
 851         ret = psp_hdcp_load(psp);
 852         if (ret)
 853                 return ret;
 854
 855         return 0;
 856 }
 857 static void psp_prep_hdcp_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 858                                             uint32_t hdcp_session_id)
 859 {
 860         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 861         cmd->cmd.cmd_unload_ta.session_id = hdcp_session_id;
 862 }
 863
 864 static int psp_hdcp_unload(struct psp_context *psp)
 865 {
 866         int ret;
 867         struct psp_gfx_cmd_resp *cmd;
 868
 869         /*
 870          * TODO: bypass the unloading in sriov for now
 871          */
 872         if (amdgpu_sriov_vf(psp->adev))
 873                 return 0;
 874
 875         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 876         if (!cmd)
 877                 return -ENOMEM;
 878
 879         psp_prep_hdcp_ta_unload_cmd_buf(cmd, psp->hdcp_context.session_id);
 880
 881         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 882
 883         kfree(cmd);
 884
 885         return ret;
 886 }
 887
 888 static void psp_prep_hdcp_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 889                                             uint32_t ta_cmd_id,
 890                                             uint32_t hdcp_session_id)
 891 {
 892         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 893         cmd->cmd.cmd_invoke_cmd.session_id = hdcp_session_id;
 894         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 895         /* Note: cmd_invoke_cmd.buf is not used for now */
 896 }
 897
 898 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 899 {
 900         int ret;
 901         struct psp_gfx_cmd_resp *cmd;
 902
 903         /*
 904          * TODO: bypass the loading in sriov for now
 905          */
 906         if (amdgpu_sriov_vf(psp->adev))
 907                 return 0;
 908
 909         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 910         if (!cmd)
 911                 return -ENOMEM;
 912
 913         psp_prep_hdcp_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 914                                         psp->hdcp_context.session_id);
 915
 916         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 917
 918         kfree(cmd);
 919
 920         return ret;
 921 }
 922
 923 static int psp_hdcp_terminate(struct psp_context *psp)
 924 {
 925         int ret;
 926
 927         if (!psp->hdcp_context.hdcp_initialized)
 928                 return 0;
 929
 930         ret = psp_hdcp_unload(psp);
 931         if (ret)
 932                 return ret;
 933
 934         psp->hdcp_context.hdcp_initialized = 0;
 935
 936         /* free hdcp shared memory */
 937         amdgpu_bo_free_kernel(&psp->hdcp_context.hdcp_shared_bo,
 938                               &psp->hdcp_context.hdcp_shared_mc_addr,
 939                               &psp->hdcp_context.hdcp_shared_buf);
 940
 941         return 0;
 942 }
 943 // HDCP end
 944
 945 static int psp_hw_start(struct psp_context *psp)
 946 {
 947         struct amdgpu_device *adev = psp->adev;
 948         int ret;
 949
 950         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 951                 if (psp->kdb_bin_size &&
 952                     (psp->funcs->bootloader_load_kdb != NULL)) {
 953                         ret = psp_bootloader_load_kdb(psp);
 954                         if (ret) {
 955                                 DRM_ERROR("PSP load kdb failed!\n");
 956                                 return ret;
 957                         }
 958                 }
 959
 960                 ret = psp_bootloader_load_sysdrv(psp);
 961                 if (ret) {
 962                         DRM_ERROR("PSP load sysdrv failed!\n");
 963                         return ret;
 964                 }
 965
 966                 ret = psp_bootloader_load_sos(psp);
 967                 if (ret) {
 968                         DRM_ERROR("PSP load sos failed!\n");
 969                         return ret;
 970                 }
 971         }
 972
 973         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 974         if (ret) {
 975                 DRM_ERROR("PSP create ring failed!\n");
 976                 return ret;
 977         }
 978
 979         ret = psp_tmr_init(psp);
 980         if (ret) {
 981                 DRM_ERROR("PSP tmr init failed!\n");
 982                 return ret;
 983         }
 984
 985         ret = psp_tmr_load(psp);
 986         if (ret) {
 987                 DRM_ERROR("PSP load tmr failed!\n");
 988                 return ret;
 989         }
 990
 991         ret = psp_asd_init(psp);
 992         if (ret) {
 993                 DRM_ERROR("PSP asd init failed!\n");
 994                 return ret;
 995         }
 996
 997         ret = psp_asd_load(psp);
 998         if (ret) {
 999                 DRM_ERROR("PSP load asd failed!\n");
1000                 return ret;
1001         }
1002
1003         if (adev->gmc.xgmi.num_physical_nodes > 1) {
1004                 ret = psp_xgmi_initialize(psp);
1005                 /* Warning the XGMI seesion initialize failure
1006                  * Instead of stop driver initialization
1007                  */
1008                 if (ret)
1009                         dev_err(psp->adev->dev,
1010                                 "XGMI: Failed to initialize XGMI session\n");
1011         }
1012
1013         if (psp->adev->psp.ta_fw) {
1014                 ret = psp_ras_initialize(psp);
1015                 if (ret)
1016                         dev_err(psp->adev->dev,
1017                                         "RAS: Failed to initialize RAS\n");
1018
1019                 ret = psp_hdcp_initialize(psp);
1020                 if (ret)
1021                         dev_err(psp->adev->dev,
1022                                 "HDCP: Failed to initialize HDCP\n");
1023         }
1024
1025         return 0;
1026 }
1027
1028 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
1029                            enum psp_gfx_fw_type *type)
1030 {
1031         switch (ucode->ucode_id) {
1032         case AMDGPU_UCODE_ID_SDMA0:
1033                 *type = GFX_FW_TYPE_SDMA0;
1034                 break;
1035         case AMDGPU_UCODE_ID_SDMA1:
1036                 *type = GFX_FW_TYPE_SDMA1;
1037                 break;
1038         case AMDGPU_UCODE_ID_SDMA2:
1039                 *type = GFX_FW_TYPE_SDMA2;
1040                 break;
1041         case AMDGPU_UCODE_ID_SDMA3:
1042                 *type = GFX_FW_TYPE_SDMA3;
1043                 break;
1044         case AMDGPU_UCODE_ID_SDMA4:
1045                 *type = GFX_FW_TYPE_SDMA4;
1046                 break;
1047         case AMDGPU_UCODE_ID_SDMA5:
1048                 *type = GFX_FW_TYPE_SDMA5;
1049                 break;
1050         case AMDGPU_UCODE_ID_SDMA6:
1051                 *type = GFX_FW_TYPE_SDMA6;
1052                 break;
1053         case AMDGPU_UCODE_ID_SDMA7:
1054                 *type = GFX_FW_TYPE_SDMA7;
1055                 break;
1056         case AMDGPU_UCODE_ID_CP_CE:
1057                 *type = GFX_FW_TYPE_CP_CE;
1058                 break;
1059         case AMDGPU_UCODE_ID_CP_PFP:
1060                 *type = GFX_FW_TYPE_CP_PFP;
1061                 break;
1062         case AMDGPU_UCODE_ID_CP_ME:
1063                 *type = GFX_FW_TYPE_CP_ME;
1064                 break;
1065         case AMDGPU_UCODE_ID_CP_MEC1:
1066                 *type = GFX_FW_TYPE_CP_MEC;
1067                 break;
1068         case AMDGPU_UCODE_ID_CP_MEC1_JT:
1069                 *type = GFX_FW_TYPE_CP_MEC_ME1;
1070                 break;
1071         case AMDGPU_UCODE_ID_CP_MEC2:
1072                 *type = GFX_FW_TYPE_CP_MEC;
1073                 break;
1074         case AMDGPU_UCODE_ID_CP_MEC2_JT:
1075                 *type = GFX_FW_TYPE_CP_MEC_ME2;
1076                 break;
1077         case AMDGPU_UCODE_ID_RLC_G:
1078                 *type = GFX_FW_TYPE_RLC_G;
1079                 break;
1080         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
1081                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
1082                 break;
1083         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
1084                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
1085                 break;
1086         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
1087                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
1088                 break;
1089         case AMDGPU_UCODE_ID_SMC:
1090                 *type = GFX_FW_TYPE_SMU;
1091                 break;
1092         case AMDGPU_UCODE_ID_UVD:
1093                 *type = GFX_FW_TYPE_UVD;
1094                 break;
1095         case AMDGPU_UCODE_ID_UVD1:
1096                 *type = GFX_FW_TYPE_UVD1;
1097                 break;
1098         case AMDGPU_UCODE_ID_VCE:
1099                 *type = GFX_FW_TYPE_VCE;
1100                 break;
1101         case AMDGPU_UCODE_ID_VCN:
1102                 *type = GFX_FW_TYPE_VCN;
1103                 break;
1104         case AMDGPU_UCODE_ID_DMCU_ERAM:
1105                 *type = GFX_FW_TYPE_DMCU_ERAM;
1106                 break;
1107         case AMDGPU_UCODE_ID_DMCU_INTV:
1108                 *type = GFX_FW_TYPE_DMCU_ISR;
1109                 break;
1110         case AMDGPU_UCODE_ID_VCN0_RAM:
1111                 *type = GFX_FW_TYPE_VCN0_RAM;
1112                 break;
1113         case AMDGPU_UCODE_ID_VCN1_RAM:
1114                 *type = GFX_FW_TYPE_VCN1_RAM;
1115                 break;
1116         case AMDGPU_UCODE_ID_MAXIMUM:
1117         default:
1118                 return -EINVAL;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static void psp_print_fw_hdr(struct psp_context *psp,
1125                              struct amdgpu_firmware_info *ucode)
1126 {
1127         struct amdgpu_device *adev = psp->adev;
1128         struct common_firmware_header *hdr;
1129
1130         switch (ucode->ucode_id) {
1131         case AMDGPU_UCODE_ID_SDMA0:
1132         case AMDGPU_UCODE_ID_SDMA1:
1133         case AMDGPU_UCODE_ID_SDMA2:
1134         case AMDGPU_UCODE_ID_SDMA3:
1135         case AMDGPU_UCODE_ID_SDMA4:
1136         case AMDGPU_UCODE_ID_SDMA5:
1137         case AMDGPU_UCODE_ID_SDMA6:
1138         case AMDGPU_UCODE_ID_SDMA7:
1139                 hdr = (struct common_firmware_header *)
1140                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
1141                 amdgpu_ucode_print_sdma_hdr(hdr);
1142                 break;
1143         case AMDGPU_UCODE_ID_CP_CE:
1144                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
1145                 amdgpu_ucode_print_gfx_hdr(hdr);
1146                 break;
1147         case AMDGPU_UCODE_ID_CP_PFP:
1148                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
1149                 amdgpu_ucode_print_gfx_hdr(hdr);
1150                 break;
1151         case AMDGPU_UCODE_ID_CP_ME:
1152                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
1153                 amdgpu_ucode_print_gfx_hdr(hdr);
1154                 break;
1155         case AMDGPU_UCODE_ID_CP_MEC1:
1156                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
1157                 amdgpu_ucode_print_gfx_hdr(hdr);
1158                 break;
1159         case AMDGPU_UCODE_ID_RLC_G:
1160                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
1161                 amdgpu_ucode_print_rlc_hdr(hdr);
1162                 break;
1163         case AMDGPU_UCODE_ID_SMC:
1164                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
1165                 amdgpu_ucode_print_smc_hdr(hdr);
1166                 break;
1167         default:
1168                 break;
1169         }
1170 }
1171
1172 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
1173                                        struct psp_gfx_cmd_resp *cmd)
1174 {
1175         int ret;
1176         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1177
1178         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1179
1180         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1181         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1182         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1183         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1184
1185         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1186         if (ret)
1187                 DRM_ERROR("Unknown firmware type\n");
1188
1189         return ret;
1190 }
1191
1192 static int psp_execute_np_fw_load(struct psp_context *psp,
1193                                struct amdgpu_firmware_info *ucode)
1194 {
1195         int ret = 0;
1196
1197         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1198         if (ret)
1199                 return ret;
1200
1201         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1202                                  psp->fence_buf_mc_addr);
1203
1204         return ret;
1205 }
1206
1207 static int psp_np_fw_load(struct psp_context *psp)
1208 {
1209         int i, ret;
1210         struct amdgpu_firmware_info *ucode;
1211         struct amdgpu_device* adev = psp->adev;
1212
1213         if (psp->autoload_supported) {
1214                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1215                 if (!ucode->fw)
1216                         goto out;
1217
1218                 ret = psp_execute_np_fw_load(psp, ucode);
1219                 if (ret)
1220                         return ret;
1221         }
1222
1223 out:
1224         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1225                 ucode = &adev->firmware.ucode[i];
1226                 if (!ucode->fw)
1227                         continue;
1228
1229                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1230                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1231                         continue;
1232
1233                 if (amdgpu_sriov_vf(adev) &&
1234                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1235                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1236                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1237                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1238                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1239                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1240                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1241                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1242                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
1243                         /*skip ucode loading in SRIOV VF */
1244                         continue;
1245
1246                 if (psp->autoload_supported &&
1247                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1248                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1249                         /* skip mec JT when autoload is enabled */
1250                         continue;
1251                 /* Renoir only needs to load mec jump table one time */
1252                 if (adev->asic_type == CHIP_RENOIR &&
1253                     ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)
1254                         continue;
1255
1256                 psp_print_fw_hdr(psp, ucode);
1257
1258                 ret = psp_execute_np_fw_load(psp, ucode);
1259                 if (ret)
1260                         return ret;
1261
1262                 /* Start rlc autoload after psp recieved all the gfx firmware */
1263                 if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
1264                         ret = psp_rlc_autoload(psp);
1265                         if (ret) {
1266                                 DRM_ERROR("Failed to start rlc autoload\n");
1267                                 return ret;
1268                         }
1269                 }
1270 #if 0
1271                 /* check if firmware loaded sucessfully */
1272                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1273                         return -EINVAL;
1274 #endif
1275         }
1276
1277         return 0;
1278 }
1279
1280 static int psp_load_fw(struct amdgpu_device *adev)
1281 {
1282         int ret;
1283         struct psp_context *psp = &adev->psp;
1284
1285         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1286                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1287                 goto skip_memalloc;
1288         }
1289
1290         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1291         if (!psp->cmd)
1292                 return -ENOMEM;
1293
1294         /* this fw pri bo is not used under SRIOV */
1295         if (!amdgpu_sriov_vf(psp->adev)) {
1296                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1297                                               AMDGPU_GEM_DOMAIN_GTT,
1298                                               &psp->fw_pri_bo,
1299                                               &psp->fw_pri_mc_addr,
1300                                               &psp->fw_pri_buf);
1301                 if (ret)
1302                         goto failed;
1303         }
1304
1305         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1306                                         AMDGPU_GEM_DOMAIN_VRAM,
1307                                         &psp->fence_buf_bo,
1308                                         &psp->fence_buf_mc_addr,
1309                                         &psp->fence_buf);
1310         if (ret)
1311                 goto failed;
1312
1313         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1314                                       AMDGPU_GEM_DOMAIN_VRAM,
1315                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1316                                       (void **)&psp->cmd_buf_mem);
1317         if (ret)
1318                 goto failed;
1319
1320         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1321
1322         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1323         if (ret) {
1324                 DRM_ERROR("PSP ring init failed!\n");
1325                 goto failed;
1326         }
1327
1328 skip_memalloc:
1329         ret = psp_hw_start(psp);
1330         if (ret)
1331                 goto failed;
1332
1333         ret = psp_np_fw_load(psp);
1334         if (ret)
1335                 goto failed;
1336
1337         return 0;
1338
1339 failed:
1340         /*
1341          * all cleanup jobs (xgmi terminate, ras terminate,
1342          * ring destroy, cmd/fence/fw buffers destory,
1343          * psp->cmd destory) are delayed to psp_hw_fini
1344          */
1345         return ret;
1346 }
1347
1348 static int psp_hw_init(void *handle)
1349 {
1350         int ret;
1351         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1352
1353         mutex_lock(&adev->firmware.mutex);
1354         /*
1355          * This sequence is just used on hw_init only once, no need on
1356          * resume.
1357          */
1358         ret = amdgpu_ucode_init_bo(adev);
1359         if (ret)
1360                 goto failed;
1361
1362         ret = psp_load_fw(adev);
1363         if (ret) {
1364                 DRM_ERROR("PSP firmware loading failed\n");
1365                 goto failed;
1366         }
1367
1368         mutex_unlock(&adev->firmware.mutex);
1369         return 0;
1370
1371 failed:
1372         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1373         mutex_unlock(&adev->firmware.mutex);
1374         return -EINVAL;
1375 }
1376
1377 static int psp_hw_fini(void *handle)
1378 {
1379         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1380         struct psp_context *psp = &adev->psp;
1381         void *tmr_buf;
1382         void **pptr;
1383
1384         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1385             psp->xgmi_context.initialized == 1)
1386                 psp_xgmi_terminate(psp);
1387
1388         if (psp->adev->psp.ta_fw) {
1389                 psp_ras_terminate(psp);
1390                 psp_hdcp_terminate(psp);
1391         }
1392
1393         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1394
1395         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1396         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1397         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1398                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1399         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1400                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1401         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1402                               &psp->asd_shared_buf);
1403         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1404                               (void **)&psp->cmd_buf_mem);
1405
1406         kfree(psp->cmd);
1407         psp->cmd = NULL;
1408
1409         return 0;
1410 }
1411
1412 static int psp_suspend(void *handle)
1413 {
1414         int ret;
1415         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1416         struct psp_context *psp = &adev->psp;
1417
1418         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1419             psp->xgmi_context.initialized == 1) {
1420                 ret = psp_xgmi_terminate(psp);
1421                 if (ret) {
1422                         DRM_ERROR("Failed to terminate xgmi ta\n");
1423                         return ret;
1424                 }
1425         }
1426
1427         if (psp->adev->psp.ta_fw) {
1428                 ret = psp_ras_terminate(psp);
1429                 if (ret) {
1430                         DRM_ERROR("Failed to terminate ras ta\n");
1431                         return ret;
1432                 }
1433                 ret = psp_hdcp_terminate(psp);
1434                 if (ret) {
1435                         DRM_ERROR("Failed to terminate hdcp ta\n");
1436                         return ret;
1437                 }
1438         }
1439
1440         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1441         if (ret) {
1442                 DRM_ERROR("PSP ring stop failed\n");
1443                 return ret;
1444         }
1445
1446         return 0;
1447 }
1448
1449 static int psp_resume(void *handle)
1450 {
1451         int ret;
1452         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1453         struct psp_context *psp = &adev->psp;
1454
1455         DRM_INFO("PSP is resuming...\n");
1456
1457         mutex_lock(&adev->firmware.mutex);
1458
1459         ret = psp_hw_start(psp);
1460         if (ret)
1461                 goto failed;
1462
1463         ret = psp_np_fw_load(psp);
1464         if (ret)
1465                 goto failed;
1466
1467         mutex_unlock(&adev->firmware.mutex);
1468
1469         return 0;
1470
1471 failed:
1472         DRM_ERROR("PSP resume failed\n");
1473         mutex_unlock(&adev->firmware.mutex);
1474         return ret;
1475 }
1476
1477 int psp_gpu_reset(struct amdgpu_device *adev)
1478 {
1479         int ret;
1480
1481         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1482                 return 0;
1483
1484         mutex_lock(&adev->psp.mutex);
1485         ret = psp_mode1_reset(&adev->psp);
1486         mutex_unlock(&adev->psp.mutex);
1487
1488         return ret;
1489 }
1490
1491 int psp_rlc_autoload_start(struct psp_context *psp)
1492 {
1493         int ret;
1494         struct psp_gfx_cmd_resp *cmd;
1495
1496         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1497         if (!cmd)
1498                 return -ENOMEM;
1499
1500         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1501
1502         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1503                                  psp->fence_buf_mc_addr);
1504         kfree(cmd);
1505         return ret;
1506 }
1507
1508 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1509                         uint64_t cmd_gpu_addr, int cmd_size)
1510 {
1511         struct amdgpu_firmware_info ucode = {0};
1512
1513         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1514                 AMDGPU_UCODE_ID_VCN0_RAM;
1515         ucode.mc_addr = cmd_gpu_addr;
1516         ucode.ucode_size = cmd_size;
1517
1518         return psp_execute_np_fw_load(&adev->psp, &ucode);
1519 }
1520
1521 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1522                                         enum AMDGPU_UCODE_ID ucode_type)
1523 {
1524         struct amdgpu_firmware_info *ucode = NULL;
1525
1526         if (!adev->firmware.fw_size)
1527                 return false;
1528
1529         ucode = &adev->firmware.ucode[ucode_type];
1530         if (!ucode->fw || !ucode->ucode_size)
1531                 return false;
1532
1533         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1534 }
1535
1536 static int psp_set_clockgating_state(void *handle,
1537                                      enum amd_clockgating_state state)
1538 {
1539         return 0;
1540 }
1541
1542 static int psp_set_powergating_state(void *handle,
1543                                      enum amd_powergating_state state)
1544 {
1545         return 0;
1546 }
1547
1548 const struct amd_ip_funcs psp_ip_funcs = {
1549         .name = "psp",
1550         .early_init = psp_early_init,
1551         .late_init = NULL,
1552         .sw_init = psp_sw_init,
1553         .sw_fini = psp_sw_fini,
1554         .hw_init = psp_hw_init,
1555         .hw_fini = psp_hw_fini,
1556         .suspend = psp_suspend,
1557         .resume = psp_resume,
1558         .is_idle = NULL,
1559         .check_soft_reset = NULL,
1560         .wait_for_idle = NULL,
1561         .soft_reset = NULL,
1562         .set_clockgating_state = psp_set_clockgating_state,
1563         .set_powergating_state = psp_set_powergating_state,
1564 };
1565
1566 static const struct amdgpu_psp_funcs psp_funcs = {
1567         .check_fw_loading_status = psp_check_fw_loading_status,
1568 };
1569
1570 static void psp_set_funcs(struct amdgpu_device *adev)
1571 {
1572         if (NULL == adev->firmware.funcs)
1573                 adev->firmware.funcs = &psp_funcs;
1574 }
1575
1576 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1577 {
1578         .type = AMD_IP_BLOCK_TYPE_PSP,
1579         .major = 3,
1580         .minor = 1,
1581         .rev = 0,
1582         .funcs = &psp_ip_funcs,
1583 };
1584
1585 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1586 {
1587         .type = AMD_IP_BLOCK_TYPE_PSP,
1588         .major = 10,
1589         .minor = 0,
1590         .rev = 0,
1591         .funcs = &psp_ip_funcs,
1592 };
1593
1594 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1595 {
1596         .type = AMD_IP_BLOCK_TYPE_PSP,
1597         .major = 11,
1598         .minor = 0,
1599         .rev = 0,
1600         .funcs = &psp_ip_funcs,
1601 };
1602
1603 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1604 {
1605         .type = AMD_IP_BLOCK_TYPE_PSP,
1606         .major = 12,
1607         .minor = 0,
1608         .rev = 0,
1609         .funcs = &psp_ip_funcs,
1610 };