drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "amdgpu_amdkfd.h"
  24 #include "amd_shared.h"
  25 #include <drm/drmP.h>
  26 #include "amdgpu.h"
  27 #include "amdgpu_gfx.h"
  28 #include <linux/module.h>
  29
  30 const struct kgd2kfd_calls *kgd2kfd;
  31 bool (*kgd2kfd_init_p)(unsigned int, const struct kgd2kfd_calls**);
  32
  33 static const unsigned int compute_vmid_bitmap = 0xFF00;
  34
  35 int amdgpu_amdkfd_init(void)
  36 {
  37         int ret;
  38
  39 #ifdef CONFIG_HSA_AMD
  40         ret = kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd);
  41         if (ret)
  42                 kgd2kfd = NULL;
  43         amdgpu_amdkfd_gpuvm_init_mem_limits();
  44 #else
  45         kgd2kfd = NULL;
  46         ret = -ENOENT;
  47 #endif
  48
  49         return ret;
  50 }
  51
  52 void amdgpu_amdkfd_fini(void)
  53 {
  54         if (kgd2kfd) {
  55                 kgd2kfd->exit();
  56                 symbol_put(kgd2kfd_init);
  57         }
  58 }
  59
  60 void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
  61 {
  62         const struct kfd2kgd_calls *kfd2kgd;
  63
  64         if (!kgd2kfd)
  65                 return;
  66
  67         switch (adev->asic_type) {
  68 #ifdef CONFIG_DRM_AMDGPU_CIK
  69         case CHIP_KAVERI:
  70         case CHIP_HAWAII:
  71                 kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions();
  72                 break;
  73 #endif
  74         case CHIP_CARRIZO:
  75         case CHIP_TONGA:
  76         case CHIP_FIJI:
  77         case CHIP_POLARIS10:
  78         case CHIP_POLARIS11:
  79                 kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
  80                 break;
  81         case CHIP_VEGA10:
  82         case CHIP_RAVEN:
  83                 kfd2kgd = amdgpu_amdkfd_gfx_9_0_get_functions();
  84                 break;
  85         default:
  86                 dev_info(adev->dev, "kfd not supported on this ASIC\n");
  87                 return;
  88         }
  89
  90         adev->kfd = kgd2kfd->probe((struct kgd_dev *)adev,
  91                                    adev->pdev, kfd2kgd);
  92 }
  93
  94 /**
  95  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
  96  *                                setup amdkfd
  97  *
  98  * @adev: amdgpu_device pointer
  99  * @aperture_base: output returning doorbell aperture base physical address
 100  * @aperture_size: output returning doorbell aperture size in bytes
 101  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 102  *
 103  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 104  * takes doorbells required for its own rings and reports the setup to amdkfd.
 105  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 106  */
 107 static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 108                                          phys_addr_t *aperture_base,
 109                                          size_t *aperture_size,
 110                                          size_t *start_offset)
 111 {
 112         /*
 113          * The first num_doorbells are used by amdgpu.
 114          * amdkfd takes whatever's left in the aperture.
 115          */
 116         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 117                 *aperture_base = adev->doorbell.base;
 118                 *aperture_size = adev->doorbell.size;
 119                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 120         } else {
 121                 *aperture_base = 0;
 122                 *aperture_size = 0;
 123                 *start_offset = 0;
 124         }
 125 }
 126
 127 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 128 {
 129         int i;
 130         int last_valid_bit;
 131         if (adev->kfd) {
 132                 struct kgd2kfd_shared_resources gpu_resources = {
 133                         .compute_vmid_bitmap = compute_vmid_bitmap,
 134                         .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec,
 135                         .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe,
 136                         .gpuvm_size = min(adev->vm_manager.max_pfn
 137                                           << AMDGPU_GPU_PAGE_SHIFT,
 138                                           AMDGPU_VA_HOLE_START),
 139                         .drm_render_minor = adev->ddev->render->index
 140                 };
 141
 142                 /* this is going to have a few of the MSBs set that we need to
 143                  * clear */
 144                 bitmap_complement(gpu_resources.queue_bitmap,
 145                                   adev->gfx.mec.queue_bitmap,
 146                                   KGD_MAX_QUEUES);
 147
 148                 /* remove the KIQ bit as well */
 149                 if (adev->gfx.kiq.ring.ready)
 150                         clear_bit(amdgpu_gfx_queue_to_bit(adev,
 151                                                           adev->gfx.kiq.ring.me - 1,
 152                                                           adev->gfx.kiq.ring.pipe,
 153                                                           adev->gfx.kiq.ring.queue),
 154                                   gpu_resources.queue_bitmap);
 155
 156                 /* According to linux/bitmap.h we shouldn't use bitmap_clear if
 157                  * nbits is not compile time constant */
 158                 last_valid_bit = 1 /* only first MEC can have compute queues */
 159                                 * adev->gfx.mec.num_pipe_per_mec
 160                                 * adev->gfx.mec.num_queue_per_pipe;
 161                 for (i = last_valid_bit; i < KGD_MAX_QUEUES; ++i)
 162                         clear_bit(i, gpu_resources.queue_bitmap);
 163
 164                 amdgpu_doorbell_get_kfd_info(adev,
 165                                 &gpu_resources.doorbell_physical_address,
 166                                 &gpu_resources.doorbell_aperture_size,
 167                                 &gpu_resources.doorbell_start_offset);
 168                 if (adev->asic_type >= CHIP_VEGA10) {
 169                         /* On SOC15 the BIF is involved in routing
 170                          * doorbells using the low 12 bits of the
 171                          * address. Communicate the assignments to
 172                          * KFD. KFD uses two doorbell pages per
 173                          * process in case of 64-bit doorbells so we
 174                          * can use each doorbell assignment twice.
 175                          */
 176                         gpu_resources.sdma_doorbell[0][0] =
 177                                 AMDGPU_DOORBELL64_sDMA_ENGINE0;
 178                         gpu_resources.sdma_doorbell[0][1] =
 179                                 AMDGPU_DOORBELL64_sDMA_ENGINE0 + 0x200;
 180                         gpu_resources.sdma_doorbell[1][0] =
 181                                 AMDGPU_DOORBELL64_sDMA_ENGINE1;
 182                         gpu_resources.sdma_doorbell[1][1] =
 183                                 AMDGPU_DOORBELL64_sDMA_ENGINE1 + 0x200;
 184                         /* Doorbells 0x0f0-0ff and 0x2f0-2ff are reserved for
 185                          * SDMA, IH and VCN. So don't use them for the CP.
 186                          */
 187                         gpu_resources.reserved_doorbell_mask = 0x1f0;
 188                         gpu_resources.reserved_doorbell_val  = 0x0f0;
 189                 }
 190
 191                 kgd2kfd->device_init(adev->kfd, &gpu_resources);
 192         }
 193 }
 194
 195 void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev)
 196 {
 197         if (adev->kfd) {
 198                 kgd2kfd->device_exit(adev->kfd);
 199                 adev->kfd = NULL;
 200         }
 201 }
 202
 203 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
 204                 const void *ih_ring_entry)
 205 {
 206         if (adev->kfd)
 207                 kgd2kfd->interrupt(adev->kfd, ih_ring_entry);
 208 }
 209
 210 void amdgpu_amdkfd_suspend(struct amdgpu_device *adev)
 211 {
 212         if (adev->kfd)
 213                 kgd2kfd->suspend(adev->kfd);
 214 }
 215
 216 int amdgpu_amdkfd_resume(struct amdgpu_device *adev)
 217 {
 218         int r = 0;
 219
 220         if (adev->kfd)
 221                 r = kgd2kfd->resume(adev->kfd);
 222
 223         return r;
 224 }
 225
 226 int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev)
 227 {
 228         int r = 0;
 229
 230         if (adev->kfd)
 231                 r = kgd2kfd->pre_reset(adev->kfd);
 232
 233         return r;
 234 }
 235
 236 int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev)
 237 {
 238         int r = 0;
 239
 240         if (adev->kfd)
 241                 r = kgd2kfd->post_reset(adev->kfd);
 242
 243         return r;
 244 }
 245
 246 void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd)
 247 {
 248         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 249
 250         if (amdgpu_device_should_recover_gpu(adev))
 251                 amdgpu_device_gpu_recover(adev, NULL);
 252 }
 253
 254 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 255                         void **mem_obj, uint64_t *gpu_addr,
 256                         void **cpu_ptr)
 257 {
 258         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 259         struct amdgpu_bo *bo = NULL;
 260         struct amdgpu_bo_param bp;
 261         int r;
 262         void *cpu_ptr_tmp = NULL;
 263
 264         memset(&bp, 0, sizeof(bp));
 265         bp.size = size;
 266         bp.byte_align = PAGE_SIZE;
 267         bp.domain = AMDGPU_GEM_DOMAIN_GTT;
 268         bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 269         bp.type = ttm_bo_type_kernel;
 270         bp.resv = NULL;
 271         r = amdgpu_bo_create(adev, &bp, &bo);
 272         if (r) {
 273                 dev_err(adev->dev,
 274                         "failed to allocate BO for amdkfd (%d)\n", r);
 275                 return r;
 276         }
 277
 278         /* map the buffer */
 279         r = amdgpu_bo_reserve(bo, true);
 280         if (r) {
 281                 dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r);
 282                 goto allocate_mem_reserve_bo_failed;
 283         }
 284
 285         r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
 286         if (r) {
 287                 dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r);
 288                 goto allocate_mem_pin_bo_failed;
 289         }
 290
 291         r = amdgpu_ttm_alloc_gart(&bo->tbo);
 292         if (r) {
 293                 dev_err(adev->dev, "%p bind failed\n", bo);
 294                 goto allocate_mem_kmap_bo_failed;
 295         }
 296
 297         r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp);
 298         if (r) {
 299                 dev_err(adev->dev,
 300                         "(%d) failed to map bo to kernel for amdkfd\n", r);
 301                 goto allocate_mem_kmap_bo_failed;
 302         }
 303
 304         *mem_obj = bo;
 305         *gpu_addr = amdgpu_bo_gpu_offset(bo);
 306         *cpu_ptr = cpu_ptr_tmp;
 307
 308         amdgpu_bo_unreserve(bo);
 309
 310         return 0;
 311
 312 allocate_mem_kmap_bo_failed:
 313         amdgpu_bo_unpin(bo);
 314 allocate_mem_pin_bo_failed:
 315         amdgpu_bo_unreserve(bo);
 316 allocate_mem_reserve_bo_failed:
 317         amdgpu_bo_unref(&bo);
 318
 319         return r;
 320 }
 321
 322 void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
 323 {
 324         struct amdgpu_bo *bo = (struct amdgpu_bo *) mem_obj;
 325
 326         amdgpu_bo_reserve(bo, true);
 327         amdgpu_bo_kunmap(bo);
 328         amdgpu_bo_unpin(bo);
 329         amdgpu_bo_unreserve(bo);
 330         amdgpu_bo_unref(&(bo));
 331 }
 332
 333 void get_local_mem_info(struct kgd_dev *kgd,
 334                         struct kfd_local_mem_info *mem_info)
 335 {
 336         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 337         uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask :
 338                                              ~((1ULL << 32) - 1);
 339         resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size;
 340
 341         memset(mem_info, 0, sizeof(*mem_info));
 342         if (!(adev->gmc.aper_base & address_mask || aper_limit & address_mask)) {
 343                 mem_info->local_mem_size_public = adev->gmc.visible_vram_size;
 344                 mem_info->local_mem_size_private = adev->gmc.real_vram_size -
 345                                 adev->gmc.visible_vram_size;
 346         } else {
 347                 mem_info->local_mem_size_public = 0;
 348                 mem_info->local_mem_size_private = adev->gmc.real_vram_size;
 349         }
 350         mem_info->vram_width = adev->gmc.vram_width;
 351
 352         pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n",
 353                         &adev->gmc.aper_base, &aper_limit,
 354                         mem_info->local_mem_size_public,
 355                         mem_info->local_mem_size_private);
 356
 357         if (amdgpu_sriov_vf(adev))
 358                 mem_info->mem_clk_max = adev->clock.default_mclk / 100;
 359         else if (adev->powerplay.pp_funcs)
 360                 mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100;
 361         else
 362                 mem_info->mem_clk_max = 100;
 363 }
 364
 365 uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
 366 {
 367         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 368
 369         if (adev->gfx.funcs->get_gpu_clock_counter)
 370                 return adev->gfx.funcs->get_gpu_clock_counter(adev);
 371         return 0;
 372 }
 373
 374 uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
 375 {
 376         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 377
 378         /* the sclk is in quantas of 10kHz */
 379         if (amdgpu_sriov_vf(adev))
 380                 return adev->clock.default_sclk / 100;
 381         else if (adev->powerplay.pp_funcs)
 382                 return amdgpu_dpm_get_sclk(adev, false) / 100;
 383         else
 384                 return 100;
 385 }
 386
 387 void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
 388 {
 389         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 390         struct amdgpu_cu_info acu_info = adev->gfx.cu_info;
 391
 392         memset(cu_info, 0, sizeof(*cu_info));
 393         if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap))
 394                 return;
 395
 396         cu_info->cu_active_number = acu_info.number;
 397         cu_info->cu_ao_mask = acu_info.ao_cu_mask;
 398         memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0],
 399                sizeof(acu_info.bitmap));
 400         cu_info->num_shader_engines = adev->gfx.config.max_shader_engines;
 401         cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se;
 402         cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh;
 403         cu_info->simd_per_cu = acu_info.simd_per_cu;
 404         cu_info->max_waves_per_simd = acu_info.max_waves_per_simd;
 405         cu_info->wave_front_size = acu_info.wave_front_size;
 406         cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu;
 407         cu_info->lds_size = acu_info.lds_size;
 408 }
 409
 410 uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd)
 411 {
 412         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 413
 414         return amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 415 }
 416
 417 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 418                                 uint32_t vmid, uint64_t gpu_addr,
 419                                 uint32_t *ib_cmd, uint32_t ib_len)
 420 {
 421         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 422         struct amdgpu_job *job;
 423         struct amdgpu_ib *ib;
 424         struct amdgpu_ring *ring;
 425         struct dma_fence *f = NULL;
 426         int ret;
 427
 428         switch (engine) {
 429         case KGD_ENGINE_MEC1:
 430                 ring = &adev->gfx.compute_ring[0];
 431                 break;
 432         case KGD_ENGINE_SDMA1:
 433                 ring = &adev->sdma.instance[0].ring;
 434                 break;
 435         case KGD_ENGINE_SDMA2:
 436                 ring = &adev->sdma.instance[1].ring;
 437                 break;
 438         default:
 439                 pr_err("Invalid engine in IB submission: %d\n", engine);
 440                 ret = -EINVAL;
 441                 goto err;
 442         }
 443
 444         ret = amdgpu_job_alloc(adev, 1, &job, NULL);
 445         if (ret)
 446                 goto err;
 447
 448         ib = &job->ibs[0];
 449         memset(ib, 0, sizeof(struct amdgpu_ib));
 450
 451         ib->gpu_addr = gpu_addr;
 452         ib->ptr = ib_cmd;
 453         ib->length_dw = ib_len;
 454         /* This works for NO_HWS. TODO: need to handle without knowing VMID */
 455         job->vmid = vmid;
 456
 457         ret = amdgpu_ib_schedule(ring, 1, ib, job, &f);
 458         if (ret) {
 459                 DRM_ERROR("amdgpu: failed to schedule IB.\n");
 460                 goto err_ib_sched;
 461         }
 462
 463         ret = dma_fence_wait(f, false);
 464
 465 err_ib_sched:
 466         dma_fence_put(f);
 467         amdgpu_job_free(job);
 468 err:
 469         return ret;
 470 }
 471
 472 void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle)
 473 {
 474         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 475
 476         amdgpu_dpm_switch_power_profile(adev,
 477                                         PP_SMC_POWER_PROFILE_COMPUTE, !idle);
 478 }
 479
 480 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
 481 {
 482         if (adev->kfd) {
 483                 if ((1 << vmid) & compute_vmid_bitmap)
 484                         return true;
 485         }
 486
 487         return false;
 488 }
 489
 490 #ifndef CONFIG_HSA_AMD
 491 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
 492 {
 493         return false;
 494 }
 495
 496 void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
 497 {
 498 }
 499
 500 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
 501                                         struct amdgpu_vm *vm)
 502 {
 503 }
 504
 505 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
 506 {
 507         return NULL;
 508 }
 509
 510 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
 511 {
 512         return 0;
 513 }
 514
 515 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
 516 {
 517         return NULL;
 518 }
 519
 520 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
 521 {
 522         return NULL;
 523 }
 524
 525 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
 526 {
 527         return NULL;
 528 }
 529 #endif