drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 static void psp_set_funcs(struct amdgpu_device *adev);
  38
  39 static int psp_early_init(void *handle)
  40 {
  41         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  42         struct psp_context *psp = &adev->psp;
  43
  44         psp_set_funcs(adev);
  45
  46         switch (adev->asic_type) {
  47         case CHIP_VEGA10:
  48         case CHIP_VEGA12:
  49                 psp_v3_1_set_psp_funcs(psp);
  50                 psp->autoload_supported = false;
  51                 break;
  52         case CHIP_RAVEN:
  53                 psp_v10_0_set_psp_funcs(psp);
  54                 psp->autoload_supported = false;
  55                 break;
  56         case CHIP_VEGA20:
  57         case CHIP_ARCTURUS:
  58                 psp_v11_0_set_psp_funcs(psp);
  59                 psp->autoload_supported = false;
  60                 break;
  61         case CHIP_NAVI10:
  62         case CHIP_NAVI14:
  63         case CHIP_NAVI12:
  64                 psp_v11_0_set_psp_funcs(psp);
  65                 psp->autoload_supported = true;
  66                 break;
  67         case CHIP_RENOIR:
  68                 psp_v12_0_set_psp_funcs(psp);
  69                 break;
  70         default:
  71                 return -EINVAL;
  72         }
  73
  74         psp->adev = adev;
  75
  76         return 0;
  77 }
  78
  79 static int psp_sw_init(void *handle)
  80 {
  81         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  82         struct psp_context *psp = &adev->psp;
  83         int ret;
  84
  85         ret = psp_init_microcode(psp);
  86         if (ret) {
  87                 DRM_ERROR("Failed to load psp firmware!\n");
  88                 return ret;
  89         }
  90
  91         return 0;
  92 }
  93
  94 static int psp_sw_fini(void *handle)
  95 {
  96         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  97
  98         release_firmware(adev->psp.sos_fw);
  99         adev->psp.sos_fw = NULL;
 100         release_firmware(adev->psp.asd_fw);
 101         adev->psp.asd_fw = NULL;
 102         if (adev->psp.ta_fw) {
 103                 release_firmware(adev->psp.ta_fw);
 104                 adev->psp.ta_fw = NULL;
 105         }
 106         return 0;
 107 }
 108
 109 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 110                  uint32_t reg_val, uint32_t mask, bool check_changed)
 111 {
 112         uint32_t val;
 113         int i;
 114         struct amdgpu_device *adev = psp->adev;
 115
 116         for (i = 0; i < adev->usec_timeout; i++) {
 117                 val = RREG32(reg_index);
 118                 if (check_changed) {
 119                         if (val != reg_val)
 120                                 return 0;
 121                 } else {
 122                         if ((val & mask) == reg_val)
 123                                 return 0;
 124                 }
 125                 udelay(1);
 126         }
 127
 128         return -ETIME;
 129 }
 130
 131 static int
 132 psp_cmd_submit_buf(struct psp_context *psp,
 133                    struct amdgpu_firmware_info *ucode,
 134                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 135 {
 136         int ret;
 137         int index;
 138         int timeout = 2000;
 139
 140         mutex_lock(&psp->mutex);
 141
 142         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 143
 144         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 145
 146         index = atomic_inc_return(&psp->fence_value);
 147         ret = psp_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 148         if (ret) {
 149                 atomic_dec(&psp->fence_value);
 150                 mutex_unlock(&psp->mutex);
 151                 return ret;
 152         }
 153
 154         while (*((unsigned int *)psp->fence_buf) != index) {
 155                 if (--timeout == 0)
 156                         break;
 157                 msleep(1);
 158         }
 159
 160         /* In some cases, psp response status is not 0 even there is no
 161          * problem while the command is submitted. Some version of PSP FW
 162          * doesn't write 0 to that field.
 163          * So here we would like to only print a warning instead of an error
 164          * during psp initialization to avoid breaking hw_init and it doesn't
 165          * return -EINVAL.
 166          */
 167         if (psp->cmd_buf_mem->resp.status || !timeout) {
 168                 if (ucode)
 169                         DRM_WARN("failed to load ucode id (%d) ",
 170                                   ucode->ucode_id);
 171                 DRM_WARN("psp command failed and response status is (0x%X)\n",
 172                           psp->cmd_buf_mem->resp.status & GFX_CMD_STATUS_MASK);
 173                 if (!timeout) {
 174                         mutex_unlock(&psp->mutex);
 175                         return -EINVAL;
 176                 }
 177         }
 178
 179         /* get xGMI session id from response buffer */
 180         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 181
 182         if (ucode) {
 183                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 184                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 185         }
 186         mutex_unlock(&psp->mutex);
 187
 188         return ret;
 189 }
 190
 191 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 192                                  struct psp_gfx_cmd_resp *cmd,
 193                                  uint64_t tmr_mc, uint32_t size)
 194 {
 195         if (psp_support_vmr_ring(psp))
 196                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 197         else
 198                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 199         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 200         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 201         cmd->cmd.cmd_setup_tmr.buf_size = size;
 202 }
 203
 204 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 205                                       uint64_t pri_buf_mc, uint32_t size)
 206 {
 207         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 208         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 209         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 210         cmd->cmd.cmd_load_toc.toc_size = size;
 211 }
 212
 213 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 214 static int psp_load_toc(struct psp_context *psp,
 215                         uint32_t *tmr_size)
 216 {
 217         int ret;
 218         struct psp_gfx_cmd_resp *cmd;
 219
 220         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 221         if (!cmd)
 222                 return -ENOMEM;
 223         /* Copy toc to psp firmware private buffer */
 224         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 225         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 226
 227         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 228
 229         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 230                                  psp->fence_buf_mc_addr);
 231         if (!ret)
 232                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 233         kfree(cmd);
 234         return ret;
 235 }
 236
 237 /* Set up Trusted Memory Region */
 238 static int psp_tmr_init(struct psp_context *psp)
 239 {
 240         int ret;
 241         int tmr_size;
 242         void *tmr_buf;
 243         void **pptr;
 244
 245         /*
 246          * According to HW engineer, they prefer the TMR address be "naturally
 247          * aligned" , e.g. the start address be an integer divide of TMR size.
 248          *
 249          * Note: this memory need be reserved till the driver
 250          * uninitializes.
 251          */
 252         tmr_size = PSP_TMR_SIZE;
 253
 254         /* For ASICs support RLC autoload, psp will parse the toc
 255          * and calculate the total size of TMR needed */
 256         if (psp->toc_start_addr &&
 257             psp->toc_bin_size &&
 258             psp->fw_pri_buf) {
 259                 ret = psp_load_toc(psp, &tmr_size);
 260                 if (ret) {
 261                         DRM_ERROR("Failed to load toc\n");
 262                         return ret;
 263                 }
 264         }
 265
 266         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 267         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 268                                       AMDGPU_GEM_DOMAIN_VRAM,
 269                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 270
 271         return ret;
 272 }
 273
 274 static int psp_tmr_load(struct psp_context *psp)
 275 {
 276         int ret;
 277         struct psp_gfx_cmd_resp *cmd;
 278
 279         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 280         if (!cmd)
 281                 return -ENOMEM;
 282
 283         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 284                              amdgpu_bo_size(psp->tmr_bo));
 285         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 286                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 287
 288         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 289                                  psp->fence_buf_mc_addr);
 290
 291         kfree(cmd);
 292
 293         return ret;
 294 }
 295
 296 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 297                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 298                                  uint32_t size, uint32_t shared_size)
 299 {
 300         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 301         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 302         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 303         cmd->cmd.cmd_load_ta.app_len = size;
 304
 305         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 306         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 307         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 308 }
 309
 310 static int psp_asd_init(struct psp_context *psp)
 311 {
 312         int ret;
 313
 314         /*
 315          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 316          * physical) for shared ASD <-> Driver
 317          */
 318         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 319                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 320                                       &psp->asd_shared_bo,
 321                                       &psp->asd_shared_mc_addr,
 322                                       &psp->asd_shared_buf);
 323
 324         return ret;
 325 }
 326
 327 static int psp_asd_load(struct psp_context *psp)
 328 {
 329         int ret;
 330         struct psp_gfx_cmd_resp *cmd;
 331
 332         /* If PSP version doesn't match ASD version, asd loading will be failed.
 333          * add workaround to bypass it for sriov now.
 334          * TODO: add version check to make it common
 335          */
 336         if (amdgpu_sriov_vf(psp->adev))
 337                 return 0;
 338
 339         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 340         if (!cmd)
 341                 return -ENOMEM;
 342
 343         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 344         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 345
 346         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 347                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 348
 349         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 350                                  psp->fence_buf_mc_addr);
 351
 352         kfree(cmd);
 353
 354         return ret;
 355 }
 356
 357 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 358                 uint32_t id, uint32_t value)
 359 {
 360         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 361         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 362         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 363 }
 364
 365 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 366                 uint32_t value)
 367 {
 368         struct psp_gfx_cmd_resp *cmd = NULL;
 369         int ret = 0;
 370
 371         if (reg >= PSP_REG_LAST)
 372                 return -EINVAL;
 373
 374         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 375         if (!cmd)
 376                 return -ENOMEM;
 377
 378         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 379         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 380
 381         kfree(cmd);
 382         return ret;
 383 }
 384
 385 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 386                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 387                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 388 {
 389         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 390         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 391         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 392         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 393
 394         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 395         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 396         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 397 }
 398
 399 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 400 {
 401         int ret;
 402
 403         /*
 404          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 405          * physical) for xgmi ta <-> Driver
 406          */
 407         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 408                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 409                                       &psp->xgmi_context.xgmi_shared_bo,
 410                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 411                                       &psp->xgmi_context.xgmi_shared_buf);
 412
 413         return ret;
 414 }
 415
 416 static int psp_xgmi_load(struct psp_context *psp)
 417 {
 418         int ret;
 419         struct psp_gfx_cmd_resp *cmd;
 420
 421         /*
 422          * TODO: bypass the loading in sriov for now
 423          */
 424         if (amdgpu_sriov_vf(psp->adev))
 425                 return 0;
 426
 427         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 428         if (!cmd)
 429                 return -ENOMEM;
 430
 431         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 432         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 433
 434         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 435                                       psp->xgmi_context.xgmi_shared_mc_addr,
 436                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 437
 438         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 439                                  psp->fence_buf_mc_addr);
 440
 441         if (!ret) {
 442                 psp->xgmi_context.initialized = 1;
 443                 psp->xgmi_context.session_id = cmd->resp.session_id;
 444         }
 445
 446         kfree(cmd);
 447
 448         return ret;
 449 }
 450
 451 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 452                                             uint32_t xgmi_session_id)
 453 {
 454         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 455         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 456 }
 457
 458 static int psp_xgmi_unload(struct psp_context *psp)
 459 {
 460         int ret;
 461         struct psp_gfx_cmd_resp *cmd;
 462
 463         /*
 464          * TODO: bypass the unloading in sriov for now
 465          */
 466         if (amdgpu_sriov_vf(psp->adev))
 467                 return 0;
 468
 469         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 470         if (!cmd)
 471                 return -ENOMEM;
 472
 473         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 474
 475         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 476                                  psp->fence_buf_mc_addr);
 477
 478         kfree(cmd);
 479
 480         return ret;
 481 }
 482
 483 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 484                                             uint32_t ta_cmd_id,
 485                                             uint32_t xgmi_session_id)
 486 {
 487         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 488         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 489         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 490         /* Note: cmd_invoke_cmd.buf is not used for now */
 491 }
 492
 493 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 494 {
 495         int ret;
 496         struct psp_gfx_cmd_resp *cmd;
 497
 498         /*
 499          * TODO: bypass the loading in sriov for now
 500         */
 501         if (amdgpu_sriov_vf(psp->adev))
 502                 return 0;
 503
 504         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 505         if (!cmd)
 506                 return -ENOMEM;
 507
 508         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 509                                         psp->xgmi_context.session_id);
 510
 511         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 512                                  psp->fence_buf_mc_addr);
 513
 514         kfree(cmd);
 515
 516         return ret;
 517 }
 518
 519 static int psp_xgmi_terminate(struct psp_context *psp)
 520 {
 521         int ret;
 522
 523         if (!psp->xgmi_context.initialized)
 524                 return 0;
 525
 526         ret = psp_xgmi_unload(psp);
 527         if (ret)
 528                 return ret;
 529
 530         psp->xgmi_context.initialized = 0;
 531
 532         /* free xgmi shared memory */
 533         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 534                         &psp->xgmi_context.xgmi_shared_mc_addr,
 535                         &psp->xgmi_context.xgmi_shared_buf);
 536
 537         return 0;
 538 }
 539
 540 static int psp_xgmi_initialize(struct psp_context *psp)
 541 {
 542         struct ta_xgmi_shared_memory *xgmi_cmd;
 543         int ret;
 544
 545         if (!psp->adev->psp.ta_fw)
 546                 return -ENOENT;
 547
 548         if (!psp->xgmi_context.initialized) {
 549                 ret = psp_xgmi_init_shared_buf(psp);
 550                 if (ret)
 551                         return ret;
 552         }
 553
 554         /* Load XGMI TA */
 555         ret = psp_xgmi_load(psp);
 556         if (ret)
 557                 return ret;
 558
 559         /* Initialize XGMI session */
 560         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 561         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 562         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 563
 564         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 565
 566         return ret;
 567 }
 568
 569 // ras begin
 570 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 571                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 572                 uint32_t ras_ta_size, uint32_t shared_size)
 573 {
 574         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 575         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 576         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 577         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 578
 579         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 580         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 581         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 582 }
 583
 584 static int psp_ras_init_shared_buf(struct psp_context *psp)
 585 {
 586         int ret;
 587
 588         /*
 589          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 590          * physical) for ras ta <-> Driver
 591          */
 592         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 593                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 594                         &psp->ras.ras_shared_bo,
 595                         &psp->ras.ras_shared_mc_addr,
 596                         &psp->ras.ras_shared_buf);
 597
 598         return ret;
 599 }
 600
 601 static int psp_ras_load(struct psp_context *psp)
 602 {
 603         int ret;
 604         struct psp_gfx_cmd_resp *cmd;
 605
 606         /*
 607          * TODO: bypass the loading in sriov for now
 608          */
 609         if (amdgpu_sriov_vf(psp->adev))
 610                 return 0;
 611
 612         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 613         if (!cmd)
 614                 return -ENOMEM;
 615
 616         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 617         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 618
 619         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 620                         psp->ras.ras_shared_mc_addr,
 621                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 622
 623         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 624                         psp->fence_buf_mc_addr);
 625
 626         if (!ret) {
 627                 psp->ras.ras_initialized = 1;
 628                 psp->ras.session_id = cmd->resp.session_id;
 629         }
 630
 631         kfree(cmd);
 632
 633         return ret;
 634 }
 635
 636 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 637                                                 uint32_t ras_session_id)
 638 {
 639         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 640         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 641 }
 642
 643 static int psp_ras_unload(struct psp_context *psp)
 644 {
 645         int ret;
 646         struct psp_gfx_cmd_resp *cmd;
 647
 648         /*
 649          * TODO: bypass the unloading in sriov for now
 650          */
 651         if (amdgpu_sriov_vf(psp->adev))
 652                 return 0;
 653
 654         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 655         if (!cmd)
 656                 return -ENOMEM;
 657
 658         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 659
 660         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 661                         psp->fence_buf_mc_addr);
 662
 663         kfree(cmd);
 664
 665         return ret;
 666 }
 667
 668 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 669                 uint32_t ta_cmd_id,
 670                 uint32_t ras_session_id)
 671 {
 672         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 673         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 674         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 675         /* Note: cmd_invoke_cmd.buf is not used for now */
 676 }
 677
 678 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 679 {
 680         int ret;
 681         struct psp_gfx_cmd_resp *cmd;
 682
 683         /*
 684          * TODO: bypass the loading in sriov for now
 685          */
 686         if (amdgpu_sriov_vf(psp->adev))
 687                 return 0;
 688
 689         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 690         if (!cmd)
 691                 return -ENOMEM;
 692
 693         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 694                         psp->ras.session_id);
 695
 696         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 697                         psp->fence_buf_mc_addr);
 698
 699         kfree(cmd);
 700
 701         return ret;
 702 }
 703
 704 int psp_ras_enable_features(struct psp_context *psp,
 705                 union ta_ras_cmd_input *info, bool enable)
 706 {
 707         struct ta_ras_shared_memory *ras_cmd;
 708         int ret;
 709
 710         if (!psp->ras.ras_initialized)
 711                 return -EINVAL;
 712
 713         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 714         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 715
 716         if (enable)
 717                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 718         else
 719                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 720
 721         ras_cmd->ras_in_message = *info;
 722
 723         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 724         if (ret)
 725                 return -EINVAL;
 726
 727         return ras_cmd->ras_status;
 728 }
 729
 730 static int psp_ras_terminate(struct psp_context *psp)
 731 {
 732         int ret;
 733
 734         if (!psp->ras.ras_initialized)
 735                 return 0;
 736
 737         ret = psp_ras_unload(psp);
 738         if (ret)
 739                 return ret;
 740
 741         psp->ras.ras_initialized = 0;
 742
 743         /* free ras shared memory */
 744         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 745                         &psp->ras.ras_shared_mc_addr,
 746                         &psp->ras.ras_shared_buf);
 747
 748         return 0;
 749 }
 750
 751 static int psp_ras_initialize(struct psp_context *psp)
 752 {
 753         int ret;
 754
 755         if (!psp->ras.ras_initialized) {
 756                 ret = psp_ras_init_shared_buf(psp);
 757                 if (ret)
 758                         return ret;
 759         }
 760
 761         ret = psp_ras_load(psp);
 762         if (ret)
 763                 return ret;
 764
 765         return 0;
 766 }
 767 // ras end
 768
 769 static int psp_hw_start(struct psp_context *psp)
 770 {
 771         struct amdgpu_device *adev = psp->adev;
 772         int ret;
 773
 774         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 775                 if (psp->kdb_bin_size &&
 776                     (psp->funcs->bootloader_load_kdb != NULL)) {
 777                         ret = psp_bootloader_load_kdb(psp);
 778                         if (ret) {
 779                                 DRM_ERROR("PSP load kdb failed!\n");
 780                                 return ret;
 781                         }
 782                 }
 783
 784                 ret = psp_bootloader_load_sysdrv(psp);
 785                 if (ret) {
 786                         DRM_ERROR("PSP load sysdrv failed!\n");
 787                         return ret;
 788                 }
 789
 790                 ret = psp_bootloader_load_sos(psp);
 791                 if (ret) {
 792                         DRM_ERROR("PSP load sos failed!\n");
 793                         return ret;
 794                 }
 795         }
 796
 797         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 798         if (ret) {
 799                 DRM_ERROR("PSP create ring failed!\n");
 800                 return ret;
 801         }
 802
 803         ret = psp_tmr_init(psp);
 804         if (ret) {
 805                 DRM_ERROR("PSP tmr init failed!\n");
 806                 return ret;
 807         }
 808
 809         ret = psp_tmr_load(psp);
 810         if (ret) {
 811                 DRM_ERROR("PSP load tmr failed!\n");
 812                 return ret;
 813         }
 814
 815         ret = psp_asd_init(psp);
 816         if (ret) {
 817                 DRM_ERROR("PSP asd init failed!\n");
 818                 return ret;
 819         }
 820
 821         ret = psp_asd_load(psp);
 822         if (ret) {
 823                 DRM_ERROR("PSP load asd failed!\n");
 824                 return ret;
 825         }
 826
 827         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 828                 ret = psp_xgmi_initialize(psp);
 829                 /* Warning the XGMI seesion initialize failure
 830                  * Instead of stop driver initialization
 831                  */
 832                 if (ret)
 833                         dev_err(psp->adev->dev,
 834                                 "XGMI: Failed to initialize XGMI session\n");
 835         }
 836
 837         if (psp->adev->psp.ta_fw) {
 838                 ret = psp_ras_initialize(psp);
 839                 if (ret)
 840                         dev_err(psp->adev->dev,
 841                                         "RAS: Failed to initialize RAS\n");
 842         }
 843
 844         return 0;
 845 }
 846
 847 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 848                            enum psp_gfx_fw_type *type)
 849 {
 850         switch (ucode->ucode_id) {
 851         case AMDGPU_UCODE_ID_SDMA0:
 852                 *type = GFX_FW_TYPE_SDMA0;
 853                 break;
 854         case AMDGPU_UCODE_ID_SDMA1:
 855                 *type = GFX_FW_TYPE_SDMA1;
 856                 break;
 857         case AMDGPU_UCODE_ID_SDMA2:
 858                 *type = GFX_FW_TYPE_SDMA2;
 859                 break;
 860         case AMDGPU_UCODE_ID_SDMA3:
 861                 *type = GFX_FW_TYPE_SDMA3;
 862                 break;
 863         case AMDGPU_UCODE_ID_SDMA4:
 864                 *type = GFX_FW_TYPE_SDMA4;
 865                 break;
 866         case AMDGPU_UCODE_ID_SDMA5:
 867                 *type = GFX_FW_TYPE_SDMA5;
 868                 break;
 869         case AMDGPU_UCODE_ID_SDMA6:
 870                 *type = GFX_FW_TYPE_SDMA6;
 871                 break;
 872         case AMDGPU_UCODE_ID_SDMA7:
 873                 *type = GFX_FW_TYPE_SDMA7;
 874                 break;
 875         case AMDGPU_UCODE_ID_CP_CE:
 876                 *type = GFX_FW_TYPE_CP_CE;
 877                 break;
 878         case AMDGPU_UCODE_ID_CP_PFP:
 879                 *type = GFX_FW_TYPE_CP_PFP;
 880                 break;
 881         case AMDGPU_UCODE_ID_CP_ME:
 882                 *type = GFX_FW_TYPE_CP_ME;
 883                 break;
 884         case AMDGPU_UCODE_ID_CP_MEC1:
 885                 *type = GFX_FW_TYPE_CP_MEC;
 886                 break;
 887         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 888                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 889                 break;
 890         case AMDGPU_UCODE_ID_CP_MEC2:
 891                 *type = GFX_FW_TYPE_CP_MEC;
 892                 break;
 893         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 894                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 895                 break;
 896         case AMDGPU_UCODE_ID_RLC_G:
 897                 *type = GFX_FW_TYPE_RLC_G;
 898                 break;
 899         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 900                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 901                 break;
 902         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 903                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 904                 break;
 905         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 906                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 907                 break;
 908         case AMDGPU_UCODE_ID_SMC:
 909                 *type = GFX_FW_TYPE_SMU;
 910                 break;
 911         case AMDGPU_UCODE_ID_UVD:
 912                 *type = GFX_FW_TYPE_UVD;
 913                 break;
 914         case AMDGPU_UCODE_ID_UVD1:
 915                 *type = GFX_FW_TYPE_UVD1;
 916                 break;
 917         case AMDGPU_UCODE_ID_VCE:
 918                 *type = GFX_FW_TYPE_VCE;
 919                 break;
 920         case AMDGPU_UCODE_ID_VCN:
 921                 *type = GFX_FW_TYPE_VCN;
 922                 break;
 923         case AMDGPU_UCODE_ID_DMCU_ERAM:
 924                 *type = GFX_FW_TYPE_DMCU_ERAM;
 925                 break;
 926         case AMDGPU_UCODE_ID_DMCU_INTV:
 927                 *type = GFX_FW_TYPE_DMCU_ISR;
 928                 break;
 929         case AMDGPU_UCODE_ID_VCN0_RAM:
 930                 *type = GFX_FW_TYPE_VCN0_RAM;
 931                 break;
 932         case AMDGPU_UCODE_ID_VCN1_RAM:
 933                 *type = GFX_FW_TYPE_VCN1_RAM;
 934                 break;
 935         case AMDGPU_UCODE_ID_MAXIMUM:
 936         default:
 937                 return -EINVAL;
 938         }
 939
 940         return 0;
 941 }
 942
 943 static void psp_print_fw_hdr(struct psp_context *psp,
 944                              struct amdgpu_firmware_info *ucode)
 945 {
 946         struct amdgpu_device *adev = psp->adev;
 947         const struct sdma_firmware_header_v1_0 *sdma_hdr =
 948                 (const struct sdma_firmware_header_v1_0 *)
 949                 adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
 950         const struct gfx_firmware_header_v1_0 *ce_hdr =
 951                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
 952         const struct gfx_firmware_header_v1_0 *pfp_hdr =
 953                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
 954         const struct gfx_firmware_header_v1_0 *me_hdr =
 955                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
 956         const struct gfx_firmware_header_v1_0 *mec_hdr =
 957                 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
 958         const struct rlc_firmware_header_v2_0 *rlc_hdr =
 959                 (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 960         const struct smc_firmware_header_v1_0 *smc_hdr =
 961                 (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data;
 962
 963         switch (ucode->ucode_id) {
 964         case AMDGPU_UCODE_ID_SDMA0:
 965         case AMDGPU_UCODE_ID_SDMA1:
 966         case AMDGPU_UCODE_ID_SDMA2:
 967         case AMDGPU_UCODE_ID_SDMA3:
 968         case AMDGPU_UCODE_ID_SDMA4:
 969         case AMDGPU_UCODE_ID_SDMA5:
 970         case AMDGPU_UCODE_ID_SDMA6:
 971         case AMDGPU_UCODE_ID_SDMA7:
 972                 amdgpu_ucode_print_sdma_hdr(&sdma_hdr->header);
 973                 break;
 974         case AMDGPU_UCODE_ID_CP_CE:
 975                 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
 976                 break;
 977         case AMDGPU_UCODE_ID_CP_PFP:
 978                 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
 979                 break;
 980         case AMDGPU_UCODE_ID_CP_ME:
 981                 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
 982                 break;
 983         case AMDGPU_UCODE_ID_CP_MEC1:
 984                 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
 985                 break;
 986         case AMDGPU_UCODE_ID_RLC_G:
 987                 amdgpu_ucode_print_rlc_hdr(&rlc_hdr->header);
 988                 break;
 989         case AMDGPU_UCODE_ID_SMC:
 990                 amdgpu_ucode_print_smc_hdr(&smc_hdr->header);
 991                 break;
 992         default:
 993                 break;
 994         }
 995 }
 996
 997 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
 998                                        struct psp_gfx_cmd_resp *cmd)
 999 {
1000         int ret;
1001         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1002
1003         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1004
1005         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1006         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1007         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1008         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1009
1010         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1011         if (ret)
1012                 DRM_ERROR("Unknown firmware type\n");
1013
1014         return ret;
1015 }
1016
1017 static int psp_execute_np_fw_load(struct psp_context *psp,
1018                                struct amdgpu_firmware_info *ucode)
1019 {
1020         int ret = 0;
1021
1022         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1023         if (ret)
1024                 return ret;
1025
1026         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1027                                  psp->fence_buf_mc_addr);
1028
1029         return ret;
1030 }
1031
1032 static int psp_np_fw_load(struct psp_context *psp)
1033 {
1034         int i, ret;
1035         struct amdgpu_firmware_info *ucode;
1036         struct amdgpu_device* adev = psp->adev;
1037
1038         if (psp->autoload_supported) {
1039                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1040                 if (!ucode->fw)
1041                         goto out;
1042
1043                 ret = psp_execute_np_fw_load(psp, ucode);
1044                 if (ret)
1045                         return ret;
1046         }
1047
1048 out:
1049         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1050                 ucode = &adev->firmware.ucode[i];
1051                 if (!ucode->fw)
1052                         continue;
1053
1054                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1055                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1056                         continue;
1057
1058                 if (amdgpu_sriov_vf(adev) &&
1059                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1060                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1061                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1062                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1063                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1064                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1065                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1066                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1067                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
1068                         /*skip ucode loading in SRIOV VF */
1069                         continue;
1070
1071                 if (psp->autoload_supported &&
1072                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1073                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1074                         /* skip mec JT when autoload is enabled */
1075                         continue;
1076                 /* Renoir only needs to load mec jump table one time */
1077                 if (adev->asic_type == CHIP_RENOIR &&
1078                     ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)
1079                         continue;
1080
1081                 psp_print_fw_hdr(psp, ucode);
1082
1083                 ret = psp_execute_np_fw_load(psp, ucode);
1084                 if (ret)
1085                         return ret;
1086
1087                 /* Start rlc autoload after psp recieved all the gfx firmware */
1088                 if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
1089                         ret = psp_rlc_autoload(psp);
1090                         if (ret) {
1091                                 DRM_ERROR("Failed to start rlc autoload\n");
1092                                 return ret;
1093                         }
1094                 }
1095 #if 0
1096                 /* check if firmware loaded sucessfully */
1097                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1098                         return -EINVAL;
1099 #endif
1100         }
1101
1102         return 0;
1103 }
1104
1105 static int psp_load_fw(struct amdgpu_device *adev)
1106 {
1107         int ret;
1108         struct psp_context *psp = &adev->psp;
1109
1110         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1111                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1112                 goto skip_memalloc;
1113         }
1114
1115         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1116         if (!psp->cmd)
1117                 return -ENOMEM;
1118
1119         /* this fw pri bo is not used under SRIOV */
1120         if (!amdgpu_sriov_vf(psp->adev)) {
1121                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1122                                               AMDGPU_GEM_DOMAIN_GTT,
1123                                               &psp->fw_pri_bo,
1124                                               &psp->fw_pri_mc_addr,
1125                                               &psp->fw_pri_buf);
1126                 if (ret)
1127                         goto failed;
1128         }
1129
1130         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1131                                         AMDGPU_GEM_DOMAIN_VRAM,
1132                                         &psp->fence_buf_bo,
1133                                         &psp->fence_buf_mc_addr,
1134                                         &psp->fence_buf);
1135         if (ret)
1136                 goto failed;
1137
1138         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1139                                       AMDGPU_GEM_DOMAIN_VRAM,
1140                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1141                                       (void **)&psp->cmd_buf_mem);
1142         if (ret)
1143                 goto failed;
1144
1145         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1146
1147         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1148         if (ret) {
1149                 DRM_ERROR("PSP ring init failed!\n");
1150                 goto failed;
1151         }
1152
1153 skip_memalloc:
1154         ret = psp_hw_start(psp);
1155         if (ret)
1156                 goto failed;
1157
1158         ret = psp_np_fw_load(psp);
1159         if (ret)
1160                 goto failed;
1161
1162         return 0;
1163
1164 failed:
1165         /*
1166          * all cleanup jobs (xgmi terminate, ras terminate,
1167          * ring destroy, cmd/fence/fw buffers destory,
1168          * psp->cmd destory) are delayed to psp_hw_fini
1169          */
1170         return ret;
1171 }
1172
1173 static int psp_hw_init(void *handle)
1174 {
1175         int ret;
1176         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1177
1178         mutex_lock(&adev->firmware.mutex);
1179         /*
1180          * This sequence is just used on hw_init only once, no need on
1181          * resume.
1182          */
1183         ret = amdgpu_ucode_init_bo(adev);
1184         if (ret)
1185                 goto failed;
1186
1187         ret = psp_load_fw(adev);
1188         if (ret) {
1189                 DRM_ERROR("PSP firmware loading failed\n");
1190                 goto failed;
1191         }
1192
1193         mutex_unlock(&adev->firmware.mutex);
1194         return 0;
1195
1196 failed:
1197         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1198         mutex_unlock(&adev->firmware.mutex);
1199         return -EINVAL;
1200 }
1201
1202 static int psp_hw_fini(void *handle)
1203 {
1204         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1205         struct psp_context *psp = &adev->psp;
1206         void *tmr_buf;
1207         void **pptr;
1208
1209         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1210             psp->xgmi_context.initialized == 1)
1211                 psp_xgmi_terminate(psp);
1212
1213         if (psp->adev->psp.ta_fw)
1214                 psp_ras_terminate(psp);
1215
1216         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1217
1218         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1219         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1220         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1221                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1222         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1223                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1224         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1225                               &psp->asd_shared_buf);
1226         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1227                               (void **)&psp->cmd_buf_mem);
1228
1229         kfree(psp->cmd);
1230         psp->cmd = NULL;
1231
1232         return 0;
1233 }
1234
1235 static int psp_suspend(void *handle)
1236 {
1237         int ret;
1238         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1239         struct psp_context *psp = &adev->psp;
1240
1241         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1242             psp->xgmi_context.initialized == 1) {
1243                 ret = psp_xgmi_terminate(psp);
1244                 if (ret) {
1245                         DRM_ERROR("Failed to terminate xgmi ta\n");
1246                         return ret;
1247                 }
1248         }
1249
1250         if (psp->adev->psp.ta_fw) {
1251                 ret = psp_ras_terminate(psp);
1252                 if (ret) {
1253                         DRM_ERROR("Failed to terminate ras ta\n");
1254                         return ret;
1255                 }
1256         }
1257
1258         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1259         if (ret) {
1260                 DRM_ERROR("PSP ring stop failed\n");
1261                 return ret;
1262         }
1263
1264         return 0;
1265 }
1266
1267 static int psp_resume(void *handle)
1268 {
1269         int ret;
1270         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1271         struct psp_context *psp = &adev->psp;
1272
1273         DRM_INFO("PSP is resuming...\n");
1274
1275         mutex_lock(&adev->firmware.mutex);
1276
1277         ret = psp_hw_start(psp);
1278         if (ret)
1279                 goto failed;
1280
1281         ret = psp_np_fw_load(psp);
1282         if (ret)
1283                 goto failed;
1284
1285         mutex_unlock(&adev->firmware.mutex);
1286
1287         return 0;
1288
1289 failed:
1290         DRM_ERROR("PSP resume failed\n");
1291         mutex_unlock(&adev->firmware.mutex);
1292         return ret;
1293 }
1294
1295 int psp_gpu_reset(struct amdgpu_device *adev)
1296 {
1297         int ret;
1298
1299         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1300                 return 0;
1301
1302         mutex_lock(&adev->psp.mutex);
1303         ret = psp_mode1_reset(&adev->psp);
1304         mutex_unlock(&adev->psp.mutex);
1305
1306         return ret;
1307 }
1308
1309 int psp_rlc_autoload_start(struct psp_context *psp)
1310 {
1311         int ret;
1312         struct psp_gfx_cmd_resp *cmd;
1313
1314         if (amdgpu_sriov_vf(psp->adev))
1315                 return 0;
1316
1317         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1318         if (!cmd)
1319                 return -ENOMEM;
1320
1321         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1322
1323         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1324                                  psp->fence_buf_mc_addr);
1325         kfree(cmd);
1326         return ret;
1327 }
1328
1329 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1330                         uint64_t cmd_gpu_addr, int cmd_size)
1331 {
1332         struct amdgpu_firmware_info ucode = {0};
1333
1334         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1335                 AMDGPU_UCODE_ID_VCN0_RAM;
1336         ucode.mc_addr = cmd_gpu_addr;
1337         ucode.ucode_size = cmd_size;
1338
1339         return psp_execute_np_fw_load(&adev->psp, &ucode);
1340 }
1341
1342 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1343                                         enum AMDGPU_UCODE_ID ucode_type)
1344 {
1345         struct amdgpu_firmware_info *ucode = NULL;
1346
1347         if (!adev->firmware.fw_size)
1348                 return false;
1349
1350         ucode = &adev->firmware.ucode[ucode_type];
1351         if (!ucode->fw || !ucode->ucode_size)
1352                 return false;
1353
1354         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1355 }
1356
1357 static int psp_set_clockgating_state(void *handle,
1358                                      enum amd_clockgating_state state)
1359 {
1360         return 0;
1361 }
1362
1363 static int psp_set_powergating_state(void *handle,
1364                                      enum amd_powergating_state state)
1365 {
1366         return 0;
1367 }
1368
1369 const struct amd_ip_funcs psp_ip_funcs = {
1370         .name = "psp",
1371         .early_init = psp_early_init,
1372         .late_init = NULL,
1373         .sw_init = psp_sw_init,
1374         .sw_fini = psp_sw_fini,
1375         .hw_init = psp_hw_init,
1376         .hw_fini = psp_hw_fini,
1377         .suspend = psp_suspend,
1378         .resume = psp_resume,
1379         .is_idle = NULL,
1380         .check_soft_reset = NULL,
1381         .wait_for_idle = NULL,
1382         .soft_reset = NULL,
1383         .set_clockgating_state = psp_set_clockgating_state,
1384         .set_powergating_state = psp_set_powergating_state,
1385 };
1386
1387 static const struct amdgpu_psp_funcs psp_funcs = {
1388         .check_fw_loading_status = psp_check_fw_loading_status,
1389 };
1390
1391 static void psp_set_funcs(struct amdgpu_device *adev)
1392 {
1393         if (NULL == adev->firmware.funcs)
1394                 adev->firmware.funcs = &psp_funcs;
1395 }
1396
1397 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1398 {
1399         .type = AMD_IP_BLOCK_TYPE_PSP,
1400         .major = 3,
1401         .minor = 1,
1402         .rev = 0,
1403         .funcs = &psp_ip_funcs,
1404 };
1405
1406 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1407 {
1408         .type = AMD_IP_BLOCK_TYPE_PSP,
1409         .major = 10,
1410         .minor = 0,
1411         .rev = 0,
1412         .funcs = &psp_ip_funcs,
1413 };
1414
1415 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1416 {
1417         .type = AMD_IP_BLOCK_TYPE_PSP,
1418         .major = 11,
1419         .minor = 0,
1420         .rev = 0,
1421         .funcs = &psp_ip_funcs,
1422 };
1423
1424 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1425 {
1426         .type = AMD_IP_BLOCK_TYPE_PSP,
1427         .major = 12,
1428         .minor = 0,
1429         .rev = 0,
1430         .funcs = &psp_ip_funcs,
1431 };