drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drmP.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35
  36 static void psp_set_funcs(struct amdgpu_device *adev);
  37
  38 static int psp_early_init(void *handle)
  39 {
  40         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  41         struct psp_context *psp = &adev->psp;
  42
  43         psp_set_funcs(adev);
  44
  45         switch (adev->asic_type) {
  46         case CHIP_VEGA10:
  47         case CHIP_VEGA12:
  48                 psp_v3_1_set_psp_funcs(psp);
  49                 break;
  50         case CHIP_RAVEN:
  51                 psp_v10_0_set_psp_funcs(psp);
  52                 break;
  53         case CHIP_VEGA20:
  54                 psp_v11_0_set_psp_funcs(psp);
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         psp->adev = adev;
  61
  62         return 0;
  63 }
  64
  65 static int psp_sw_init(void *handle)
  66 {
  67         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  68         struct psp_context *psp = &adev->psp;
  69         int ret;
  70
  71         ret = psp_init_microcode(psp);
  72         if (ret) {
  73                 DRM_ERROR("Failed to load psp firmware!\n");
  74                 return ret;
  75         }
  76
  77         return 0;
  78 }
  79
  80 static int psp_sw_fini(void *handle)
  81 {
  82         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  83
  84         release_firmware(adev->psp.sos_fw);
  85         adev->psp.sos_fw = NULL;
  86         release_firmware(adev->psp.asd_fw);
  87         adev->psp.asd_fw = NULL;
  88         if (adev->psp.ta_fw) {
  89                 release_firmware(adev->psp.ta_fw);
  90                 adev->psp.ta_fw = NULL;
  91         }
  92         return 0;
  93 }
  94
  95 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
  96                  uint32_t reg_val, uint32_t mask, bool check_changed)
  97 {
  98         uint32_t val;
  99         int i;
 100         struct amdgpu_device *adev = psp->adev;
 101
 102         for (i = 0; i < adev->usec_timeout; i++) {
 103                 val = RREG32(reg_index);
 104                 if (check_changed) {
 105                         if (val != reg_val)
 106                                 return 0;
 107                 } else {
 108                         if ((val & mask) == reg_val)
 109                                 return 0;
 110                 }
 111                 udelay(1);
 112         }
 113
 114         return -ETIME;
 115 }
 116
 117 static int
 118 psp_cmd_submit_buf(struct psp_context *psp,
 119                    struct amdgpu_firmware_info *ucode,
 120                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 121 {
 122         int ret;
 123         int index;
 124         int timeout = 2000;
 125
 126         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 127
 128         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 129
 130         index = atomic_inc_return(&psp->fence_value);
 131         ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr,
 132                              fence_mc_addr, index);
 133         if (ret) {
 134                 atomic_dec(&psp->fence_value);
 135                 return ret;
 136         }
 137
 138         while (*((unsigned int *)psp->fence_buf) != index) {
 139                 if (--timeout == 0)
 140                         break;
 141                 msleep(1);
 142         }
 143
 144         /* In some cases, psp response status is not 0 even there is no
 145          * problem while the command is submitted. Some version of PSP FW
 146          * doesn't write 0 to that field.
 147          * So here we would like to only print a warning instead of an error
 148          * during psp initialization to avoid breaking hw_init and it doesn't
 149          * return -EINVAL.
 150          */
 151         if (psp->cmd_buf_mem->resp.status || !timeout) {
 152                 if (ucode)
 153                         DRM_WARN("failed to load ucode id (%d) ",
 154                                   ucode->ucode_id);
 155                 DRM_WARN("psp command failed and response status is (%d)\n",
 156                           psp->cmd_buf_mem->resp.status);
 157                 if (!timeout)
 158                         return -EINVAL;
 159         }
 160
 161         /* get xGMI session id from response buffer */
 162         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 163
 164         if (ucode) {
 165                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 166                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 167         }
 168
 169         return ret;
 170 }
 171
 172 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 173                                  struct psp_gfx_cmd_resp *cmd,
 174                                  uint64_t tmr_mc, uint32_t size)
 175 {
 176         if (psp_support_vmr_ring(psp))
 177                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 178         else
 179                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 180         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 181         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 182         cmd->cmd.cmd_setup_tmr.buf_size = size;
 183 }
 184
 185 /* Set up Trusted Memory Region */
 186 static int psp_tmr_init(struct psp_context *psp)
 187 {
 188         int ret;
 189
 190         /*
 191          * According to HW engineer, they prefer the TMR address be "naturally
 192          * aligned" , e.g. the start address be an integer divide of TMR size.
 193          *
 194          * Note: this memory need be reserved till the driver
 195          * uninitializes.
 196          */
 197         ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
 198                                       AMDGPU_GEM_DOMAIN_VRAM,
 199                                       &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 200
 201         return ret;
 202 }
 203
 204 static int psp_tmr_load(struct psp_context *psp)
 205 {
 206         int ret;
 207         struct psp_gfx_cmd_resp *cmd;
 208
 209         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 210         if (!cmd)
 211                 return -ENOMEM;
 212
 213         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, PSP_TMR_SIZE);
 214         DRM_INFO("reserve 0x%x from 0x%llx for PSP TMR SIZE\n",
 215                         PSP_TMR_SIZE, psp->tmr_mc_addr);
 216
 217         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 218                                  psp->fence_buf_mc_addr);
 219         if (ret)
 220                 goto failed;
 221
 222         kfree(cmd);
 223
 224         return 0;
 225
 226 failed:
 227         kfree(cmd);
 228         return ret;
 229 }
 230
 231 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 232                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 233                                  uint32_t size, uint32_t shared_size)
 234 {
 235         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 236         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 237         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 238         cmd->cmd.cmd_load_ta.app_len = size;
 239
 240         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 241         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 242         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 243 }
 244
 245 static int psp_asd_init(struct psp_context *psp)
 246 {
 247         int ret;
 248
 249         /*
 250          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 251          * physical) for shared ASD <-> Driver
 252          */
 253         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 254                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 255                                       &psp->asd_shared_bo,
 256                                       &psp->asd_shared_mc_addr,
 257                                       &psp->asd_shared_buf);
 258
 259         return ret;
 260 }
 261
 262 static int psp_asd_load(struct psp_context *psp)
 263 {
 264         int ret;
 265         struct psp_gfx_cmd_resp *cmd;
 266
 267         /* If PSP version doesn't match ASD version, asd loading will be failed.
 268          * add workaround to bypass it for sriov now.
 269          * TODO: add version check to make it common
 270          */
 271         if (amdgpu_sriov_vf(psp->adev))
 272                 return 0;
 273
 274         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 275         if (!cmd)
 276                 return -ENOMEM;
 277
 278         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 279         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 280
 281         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 282                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 283
 284         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 285                                  psp->fence_buf_mc_addr);
 286
 287         kfree(cmd);
 288
 289         return ret;
 290 }
 291
 292 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 293                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 294                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 295 {
 296         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 297         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 298         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 299         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 300
 301         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 302         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 303         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 304 }
 305
 306 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 307 {
 308         int ret;
 309
 310         /*
 311          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 312          * physical) for xgmi ta <-> Driver
 313          */
 314         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 315                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 316                                       &psp->xgmi_context.xgmi_shared_bo,
 317                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 318                                       &psp->xgmi_context.xgmi_shared_buf);
 319
 320         return ret;
 321 }
 322
 323 static int psp_xgmi_load(struct psp_context *psp)
 324 {
 325         int ret;
 326         struct psp_gfx_cmd_resp *cmd;
 327
 328         /*
 329          * TODO: bypass the loading in sriov for now
 330          */
 331         if (amdgpu_sriov_vf(psp->adev))
 332                 return 0;
 333
 334         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 335         if (!cmd)
 336                 return -ENOMEM;
 337
 338         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 339         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 340
 341         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 342                                       psp->xgmi_context.xgmi_shared_mc_addr,
 343                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 344
 345         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 346                                  psp->fence_buf_mc_addr);
 347
 348         if (!ret) {
 349                 psp->xgmi_context.initialized = 1;
 350                 psp->xgmi_context.session_id = cmd->resp.session_id;
 351         }
 352
 353         kfree(cmd);
 354
 355         return ret;
 356 }
 357
 358 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 359                                             uint32_t xgmi_session_id)
 360 {
 361         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 362         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 363 }
 364
 365 static int psp_xgmi_unload(struct psp_context *psp)
 366 {
 367         int ret;
 368         struct psp_gfx_cmd_resp *cmd;
 369
 370         /*
 371          * TODO: bypass the unloading in sriov for now
 372          */
 373         if (amdgpu_sriov_vf(psp->adev))
 374                 return 0;
 375
 376         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 377         if (!cmd)
 378                 return -ENOMEM;
 379
 380         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 381
 382         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 383                                  psp->fence_buf_mc_addr);
 384
 385         kfree(cmd);
 386
 387         return ret;
 388 }
 389
 390 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 391                                             uint32_t ta_cmd_id,
 392                                             uint32_t xgmi_session_id)
 393 {
 394         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 395         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 396         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 397         /* Note: cmd_invoke_cmd.buf is not used for now */
 398 }
 399
 400 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 401 {
 402         int ret;
 403         struct psp_gfx_cmd_resp *cmd;
 404
 405         /*
 406          * TODO: bypass the loading in sriov for now
 407         */
 408         if (amdgpu_sriov_vf(psp->adev))
 409                 return 0;
 410
 411         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 412         if (!cmd)
 413                 return -ENOMEM;
 414
 415         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 416                                         psp->xgmi_context.session_id);
 417
 418         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 419                                  psp->fence_buf_mc_addr);
 420
 421         kfree(cmd);
 422
 423         return ret;
 424 }
 425
 426 static int psp_xgmi_terminate(struct psp_context *psp)
 427 {
 428         int ret;
 429
 430         if (!psp->xgmi_context.initialized)
 431                 return 0;
 432
 433         ret = psp_xgmi_unload(psp);
 434         if (ret)
 435                 return ret;
 436
 437         psp->xgmi_context.initialized = 0;
 438
 439         /* free xgmi shared memory */
 440         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 441                         &psp->xgmi_context.xgmi_shared_mc_addr,
 442                         &psp->xgmi_context.xgmi_shared_buf);
 443
 444         return 0;
 445 }
 446
 447 static int psp_xgmi_initialize(struct psp_context *psp)
 448 {
 449         struct ta_xgmi_shared_memory *xgmi_cmd;
 450         int ret;
 451
 452         if (!psp->adev->psp.ta_fw)
 453                 return -ENOENT;
 454
 455         if (!psp->xgmi_context.initialized) {
 456                 ret = psp_xgmi_init_shared_buf(psp);
 457                 if (ret)
 458                         return ret;
 459         }
 460
 461         /* Load XGMI TA */
 462         ret = psp_xgmi_load(psp);
 463         if (ret)
 464                 return ret;
 465
 466         /* Initialize XGMI session */
 467         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 468         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 469         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 470
 471         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 472
 473         return ret;
 474 }
 475
 476 // ras begin
 477 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 478                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 479                 uint32_t ras_ta_size, uint32_t shared_size)
 480 {
 481         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 482         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 483         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 484         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 485
 486         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 487         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 488         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 489 }
 490
 491 static int psp_ras_init_shared_buf(struct psp_context *psp)
 492 {
 493         int ret;
 494
 495         /*
 496          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 497          * physical) for ras ta <-> Driver
 498          */
 499         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 500                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 501                         &psp->ras.ras_shared_bo,
 502                         &psp->ras.ras_shared_mc_addr,
 503                         &psp->ras.ras_shared_buf);
 504
 505         return ret;
 506 }
 507
 508 static int psp_ras_load(struct psp_context *psp)
 509 {
 510         int ret;
 511         struct psp_gfx_cmd_resp *cmd;
 512
 513         /*
 514          * TODO: bypass the loading in sriov for now
 515          */
 516         if (amdgpu_sriov_vf(psp->adev))
 517                 return 0;
 518
 519         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 520         if (!cmd)
 521                 return -ENOMEM;
 522
 523         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 524         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 525
 526         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 527                         psp->ras.ras_shared_mc_addr,
 528                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 529
 530         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 531                         psp->fence_buf_mc_addr);
 532
 533         if (!ret) {
 534                 psp->ras.ras_initialized = 1;
 535                 psp->ras.session_id = cmd->resp.session_id;
 536         }
 537
 538         kfree(cmd);
 539
 540         return ret;
 541 }
 542
 543 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 544                                                 uint32_t ras_session_id)
 545 {
 546         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 547         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 548 }
 549
 550 static int psp_ras_unload(struct psp_context *psp)
 551 {
 552         int ret;
 553         struct psp_gfx_cmd_resp *cmd;
 554
 555         /*
 556          * TODO: bypass the unloading in sriov for now
 557          */
 558         if (amdgpu_sriov_vf(psp->adev))
 559                 return 0;
 560
 561         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 562         if (!cmd)
 563                 return -ENOMEM;
 564
 565         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 566
 567         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 568                         psp->fence_buf_mc_addr);
 569
 570         kfree(cmd);
 571
 572         return ret;
 573 }
 574
 575 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 576                 uint32_t ta_cmd_id,
 577                 uint32_t ras_session_id)
 578 {
 579         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 580         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 581         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 582         /* Note: cmd_invoke_cmd.buf is not used for now */
 583 }
 584
 585 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 586 {
 587         int ret;
 588         struct psp_gfx_cmd_resp *cmd;
 589
 590         /*
 591          * TODO: bypass the loading in sriov for now
 592          */
 593         if (amdgpu_sriov_vf(psp->adev))
 594                 return 0;
 595
 596         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 597         if (!cmd)
 598                 return -ENOMEM;
 599
 600         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 601                         psp->ras.session_id);
 602
 603         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 604                         psp->fence_buf_mc_addr);
 605
 606         kfree(cmd);
 607
 608         return ret;
 609 }
 610
 611 int psp_ras_enable_features(struct psp_context *psp,
 612                 union ta_ras_cmd_input *info, bool enable)
 613 {
 614         struct ta_ras_shared_memory *ras_cmd;
 615         int ret;
 616
 617         if (!psp->ras.ras_initialized)
 618                 return -EINVAL;
 619
 620         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 621         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 622
 623         if (enable)
 624                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 625         else
 626                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 627
 628         ras_cmd->ras_in_message = *info;
 629
 630         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 631         if (ret)
 632                 return -EINVAL;
 633
 634         return ras_cmd->ras_status;
 635 }
 636
 637 static int psp_ras_terminate(struct psp_context *psp)
 638 {
 639         int ret;
 640
 641         if (!psp->ras.ras_initialized)
 642                 return 0;
 643
 644         ret = psp_ras_unload(psp);
 645         if (ret)
 646                 return ret;
 647
 648         psp->ras.ras_initialized = 0;
 649
 650         /* free ras shared memory */
 651         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 652                         &psp->ras.ras_shared_mc_addr,
 653                         &psp->ras.ras_shared_buf);
 654
 655         return 0;
 656 }
 657
 658 static int psp_ras_initialize(struct psp_context *psp)
 659 {
 660         int ret;
 661
 662         if (!psp->ras.ras_initialized) {
 663                 ret = psp_ras_init_shared_buf(psp);
 664                 if (ret)
 665                         return ret;
 666         }
 667
 668         ret = psp_ras_load(psp);
 669         if (ret)
 670                 return ret;
 671
 672         return 0;
 673 }
 674 // ras end
 675
 676 static int psp_hw_start(struct psp_context *psp)
 677 {
 678         struct amdgpu_device *adev = psp->adev;
 679         int ret;
 680
 681         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 682                 ret = psp_bootloader_load_sysdrv(psp);
 683                 if (ret) {
 684                         DRM_ERROR("PSP load sysdrv failed!\n");
 685                         return ret;
 686                 }
 687
 688                 ret = psp_bootloader_load_sos(psp);
 689                 if (ret) {
 690                         DRM_ERROR("PSP load sos failed!\n");
 691                         return ret;
 692                 }
 693         }
 694
 695         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 696         if (ret) {
 697                 DRM_ERROR("PSP create ring failed!\n");
 698                 return ret;
 699         }
 700
 701         ret = psp_tmr_load(psp);
 702         if (ret) {
 703                 DRM_ERROR("PSP load tmr failed!\n");
 704                 return ret;
 705         }
 706
 707         ret = psp_asd_load(psp);
 708         if (ret) {
 709                 DRM_ERROR("PSP load asd failed!\n");
 710                 return ret;
 711         }
 712
 713         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 714                 ret = psp_xgmi_initialize(psp);
 715                 /* Warning the XGMI seesion initialize failure
 716                  * Instead of stop driver initialization
 717                  */
 718                 if (ret)
 719                         dev_err(psp->adev->dev,
 720                                 "XGMI: Failed to initialize XGMI session\n");
 721         }
 722
 723
 724         if (psp->adev->psp.ta_fw) {
 725                 ret = psp_ras_initialize(psp);
 726                 if (ret)
 727                         dev_err(psp->adev->dev,
 728                                         "RAS: Failed to initialize RAS\n");
 729         }
 730
 731         return 0;
 732 }
 733
 734 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 735                            enum psp_gfx_fw_type *type)
 736 {
 737         switch (ucode->ucode_id) {
 738         case AMDGPU_UCODE_ID_SDMA0:
 739                 *type = GFX_FW_TYPE_SDMA0;
 740                 break;
 741         case AMDGPU_UCODE_ID_SDMA1:
 742                 *type = GFX_FW_TYPE_SDMA1;
 743                 break;
 744         case AMDGPU_UCODE_ID_CP_CE:
 745                 *type = GFX_FW_TYPE_CP_CE;
 746                 break;
 747         case AMDGPU_UCODE_ID_CP_PFP:
 748                 *type = GFX_FW_TYPE_CP_PFP;
 749                 break;
 750         case AMDGPU_UCODE_ID_CP_ME:
 751                 *type = GFX_FW_TYPE_CP_ME;
 752                 break;
 753         case AMDGPU_UCODE_ID_CP_MEC1:
 754                 *type = GFX_FW_TYPE_CP_MEC;
 755                 break;
 756         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 757                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 758                 break;
 759         case AMDGPU_UCODE_ID_CP_MEC2:
 760                 *type = GFX_FW_TYPE_CP_MEC;
 761                 break;
 762         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 763                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 764                 break;
 765         case AMDGPU_UCODE_ID_RLC_G:
 766                 *type = GFX_FW_TYPE_RLC_G;
 767                 break;
 768         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 769                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 770                 break;
 771         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 772                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 773                 break;
 774         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 775                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 776                 break;
 777         case AMDGPU_UCODE_ID_SMC:
 778                 *type = GFX_FW_TYPE_SMU;
 779                 break;
 780         case AMDGPU_UCODE_ID_UVD:
 781                 *type = GFX_FW_TYPE_UVD;
 782                 break;
 783         case AMDGPU_UCODE_ID_UVD1:
 784                 *type = GFX_FW_TYPE_UVD1;
 785                 break;
 786         case AMDGPU_UCODE_ID_VCE:
 787                 *type = GFX_FW_TYPE_VCE;
 788                 break;
 789         case AMDGPU_UCODE_ID_VCN:
 790                 *type = GFX_FW_TYPE_VCN;
 791                 break;
 792         case AMDGPU_UCODE_ID_DMCU_ERAM:
 793                 *type = GFX_FW_TYPE_DMCU_ERAM;
 794                 break;
 795         case AMDGPU_UCODE_ID_DMCU_INTV:
 796                 *type = GFX_FW_TYPE_DMCU_ISR;
 797                 break;
 798         case AMDGPU_UCODE_ID_MAXIMUM:
 799         default:
 800                 return -EINVAL;
 801         }
 802
 803         return 0;
 804 }
 805
 806 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
 807                                        struct psp_gfx_cmd_resp *cmd)
 808 {
 809         int ret;
 810         uint64_t fw_mem_mc_addr = ucode->mc_addr;
 811
 812         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 813
 814         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 815         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
 816         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
 817         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 818
 819         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
 820         if (ret)
 821                 DRM_ERROR("Unknown firmware type\n");
 822
 823         return ret;
 824 }
 825
 826 static int psp_np_fw_load(struct psp_context *psp)
 827 {
 828         int i, ret;
 829         struct amdgpu_firmware_info *ucode;
 830         struct amdgpu_device* adev = psp->adev;
 831
 832         for (i = 0; i < adev->firmware.max_ucodes; i++) {
 833                 ucode = &adev->firmware.ucode[i];
 834                 if (!ucode->fw)
 835                         continue;
 836
 837                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
 838                     psp_smu_reload_quirk(psp))
 839                         continue;
 840                 if (amdgpu_sriov_vf(adev) &&
 841                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
 842                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
 843                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
 844                         /*skip ucode loading in SRIOV VF */
 845                         continue;
 846
 847                 ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
 848                 if (ret)
 849                         return ret;
 850
 851                 ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
 852                                          psp->fence_buf_mc_addr);
 853                 if (ret)
 854                         return ret;
 855
 856 #if 0
 857                 /* check if firmware loaded sucessfully */
 858                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
 859                         return -EINVAL;
 860 #endif
 861         }
 862
 863         return 0;
 864 }
 865
 866 static int psp_load_fw(struct amdgpu_device *adev)
 867 {
 868         int ret;
 869         struct psp_context *psp = &adev->psp;
 870
 871         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
 872                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
 873                 goto skip_memalloc;
 874         }
 875
 876         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 877         if (!psp->cmd)
 878                 return -ENOMEM;
 879
 880         /* this fw pri bo is not used under SRIOV */
 881         if (!amdgpu_sriov_vf(psp->adev)) {
 882                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 883                                               AMDGPU_GEM_DOMAIN_GTT,
 884                                               &psp->fw_pri_bo,
 885                                               &psp->fw_pri_mc_addr,
 886                                               &psp->fw_pri_buf);
 887                 if (ret)
 888                         goto failed;
 889         }
 890
 891         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 892                                         AMDGPU_GEM_DOMAIN_VRAM,
 893                                         &psp->fence_buf_bo,
 894                                         &psp->fence_buf_mc_addr,
 895                                         &psp->fence_buf);
 896         if (ret)
 897                 goto failed;
 898
 899         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 900                                       AMDGPU_GEM_DOMAIN_VRAM,
 901                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 902                                       (void **)&psp->cmd_buf_mem);
 903         if (ret)
 904                 goto failed;
 905
 906         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
 907
 908         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
 909         if (ret) {
 910                 DRM_ERROR("PSP ring init failed!\n");
 911                 goto failed;
 912         }
 913
 914         ret = psp_tmr_init(psp);
 915         if (ret) {
 916                 DRM_ERROR("PSP tmr init failed!\n");
 917                 goto failed;
 918         }
 919
 920         ret = psp_asd_init(psp);
 921         if (ret) {
 922                 DRM_ERROR("PSP asd init failed!\n");
 923                 goto failed;
 924         }
 925
 926 skip_memalloc:
 927         ret = psp_hw_start(psp);
 928         if (ret)
 929                 goto failed;
 930
 931         ret = psp_np_fw_load(psp);
 932         if (ret)
 933                 goto failed;
 934
 935         return 0;
 936
 937 failed:
 938         /*
 939          * all cleanup jobs (xgmi terminate, ras terminate,
 940          * ring destroy, cmd/fence/fw buffers destory,
 941          * psp->cmd destory) are delayed to psp_hw_fini
 942          */
 943         return ret;
 944 }
 945
 946 static int psp_hw_init(void *handle)
 947 {
 948         int ret;
 949         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 950
 951         mutex_lock(&adev->firmware.mutex);
 952         /*
 953          * This sequence is just used on hw_init only once, no need on
 954          * resume.
 955          */
 956         ret = amdgpu_ucode_init_bo(adev);
 957         if (ret)
 958                 goto failed;
 959
 960         ret = psp_load_fw(adev);
 961         if (ret) {
 962                 DRM_ERROR("PSP firmware loading failed\n");
 963                 goto failed;
 964         }
 965
 966         mutex_unlock(&adev->firmware.mutex);
 967         return 0;
 968
 969 failed:
 970         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
 971         mutex_unlock(&adev->firmware.mutex);
 972         return -EINVAL;
 973 }
 974
 975 static int psp_hw_fini(void *handle)
 976 {
 977         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 978         struct psp_context *psp = &adev->psp;
 979
 980         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
 981             psp->xgmi_context.initialized == 1)
 982                 psp_xgmi_terminate(psp);
 983
 984         if (psp->adev->psp.ta_fw)
 985                 psp_ras_terminate(psp);
 986
 987         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 988
 989         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 990         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 991                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 992         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 993                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 994         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
 995                               &psp->asd_shared_buf);
 996         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 997                               (void **)&psp->cmd_buf_mem);
 998
 999         kfree(psp->cmd);
1000         psp->cmd = NULL;
1001
1002         return 0;
1003 }
1004
1005 static int psp_suspend(void *handle)
1006 {
1007         int ret;
1008         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1009         struct psp_context *psp = &adev->psp;
1010
1011         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1012             psp->xgmi_context.initialized == 1) {
1013                 ret = psp_xgmi_terminate(psp);
1014                 if (ret) {
1015                         DRM_ERROR("Failed to terminate xgmi ta\n");
1016                         return ret;
1017                 }
1018         }
1019
1020         if (psp->adev->psp.ta_fw) {
1021                 ret = psp_ras_terminate(psp);
1022                 if (ret) {
1023                         DRM_ERROR("Failed to terminate ras ta\n");
1024                         return ret;
1025                 }
1026         }
1027
1028         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1029         if (ret) {
1030                 DRM_ERROR("PSP ring stop failed\n");
1031                 return ret;
1032         }
1033
1034         return 0;
1035 }
1036
1037 static int psp_resume(void *handle)
1038 {
1039         int ret;
1040         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1041         struct psp_context *psp = &adev->psp;
1042
1043         DRM_INFO("PSP is resuming...\n");
1044
1045         mutex_lock(&adev->firmware.mutex);
1046
1047         ret = psp_hw_start(psp);
1048         if (ret)
1049                 goto failed;
1050
1051         ret = psp_np_fw_load(psp);
1052         if (ret)
1053                 goto failed;
1054
1055         mutex_unlock(&adev->firmware.mutex);
1056
1057         return 0;
1058
1059 failed:
1060         DRM_ERROR("PSP resume failed\n");
1061         mutex_unlock(&adev->firmware.mutex);
1062         return ret;
1063 }
1064
1065 int psp_gpu_reset(struct amdgpu_device *adev)
1066 {
1067         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1068                 return 0;
1069
1070         return psp_mode1_reset(&adev->psp);
1071 }
1072
1073 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1074                                         enum AMDGPU_UCODE_ID ucode_type)
1075 {
1076         struct amdgpu_firmware_info *ucode = NULL;
1077
1078         if (!adev->firmware.fw_size)
1079                 return false;
1080
1081         ucode = &adev->firmware.ucode[ucode_type];
1082         if (!ucode->fw || !ucode->ucode_size)
1083                 return false;
1084
1085         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1086 }
1087
1088 static int psp_set_clockgating_state(void *handle,
1089                                      enum amd_clockgating_state state)
1090 {
1091         return 0;
1092 }
1093
1094 static int psp_set_powergating_state(void *handle,
1095                                      enum amd_powergating_state state)
1096 {
1097         return 0;
1098 }
1099
1100 const struct amd_ip_funcs psp_ip_funcs = {
1101         .name = "psp",
1102         .early_init = psp_early_init,
1103         .late_init = NULL,
1104         .sw_init = psp_sw_init,
1105         .sw_fini = psp_sw_fini,
1106         .hw_init = psp_hw_init,
1107         .hw_fini = psp_hw_fini,
1108         .suspend = psp_suspend,
1109         .resume = psp_resume,
1110         .is_idle = NULL,
1111         .check_soft_reset = NULL,
1112         .wait_for_idle = NULL,
1113         .soft_reset = NULL,
1114         .set_clockgating_state = psp_set_clockgating_state,
1115         .set_powergating_state = psp_set_powergating_state,
1116 };
1117
1118 static const struct amdgpu_psp_funcs psp_funcs = {
1119         .check_fw_loading_status = psp_check_fw_loading_status,
1120 };
1121
1122 static void psp_set_funcs(struct amdgpu_device *adev)
1123 {
1124         if (NULL == adev->firmware.funcs)
1125                 adev->firmware.funcs = &psp_funcs;
1126 }
1127
1128 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1129 {
1130         .type = AMD_IP_BLOCK_TYPE_PSP,
1131         .major = 3,
1132         .minor = 1,
1133         .rev = 0,
1134         .funcs = &psp_ip_funcs,
1135 };
1136
1137 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1138 {
1139         .type = AMD_IP_BLOCK_TYPE_PSP,
1140         .major = 10,
1141         .minor = 0,
1142         .rev = 0,
1143         .funcs = &psp_ip_funcs,
1144 };
1145
1146 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1147 {
1148         .type = AMD_IP_BLOCK_TYPE_PSP,
1149         .major = 11,
1150         .minor = 0,
1151         .rev = 0,
1152         .funcs = &psp_ip_funcs,
1153 };