2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/kthread.h>
29 #include <linux/console.h>
30 #include <linux/slab.h>
31 #include <linux/debugfs.h>
33 #include <drm/drm_crtc_helper.h>
34 #include <drm/amdgpu_drm.h>
35 #include <linux/vgaarb.h>
36 #include <linux/vga_switcheroo.h>
37 #include <linux/efi.h>
39 #include "amdgpu_trace.h"
40 #include "amdgpu_i2c.h"
42 #include "amdgpu_atombios.h"
43 #include "amdgpu_atomfirmware.h"
45 #ifdef CONFIG_DRM_AMDGPU_SI
48 #ifdef CONFIG_DRM_AMDGPU_CIK
53 #include "bif/bif_4_1_d.h"
54 #include <linux/pci.h>
55 #include <linux/firmware.h>
56 #include "amdgpu_vf_error.h"
58 #include "amdgpu_amdkfd.h"
60 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
61 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
63 #define AMDGPU_RESUME_MS 2000
65 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
66 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
67 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
68 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
70 static const char *amdgpu_asic_name[] = {
94 bool amdgpu_device_is_px(struct drm_device *dev)
96 struct amdgpu_device *adev = dev->dev_private;
98 if (adev->flags & AMD_IS_PX)
104 * MMIO register access helper functions.
106 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
111 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
112 BUG_ON(in_interrupt());
113 return amdgpu_virt_kiq_rreg(adev, reg);
116 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
117 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
121 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
122 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
123 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
124 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
126 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
130 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
133 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
135 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
136 adev->last_mm_index = v;
139 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
140 BUG_ON(in_interrupt());
141 return amdgpu_virt_kiq_wreg(adev, reg, v);
144 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
145 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
149 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
150 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
151 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
152 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
155 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
160 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
162 if ((reg * 4) < adev->rio_mem_size)
163 return ioread32(adev->rio_mem + (reg * 4));
165 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
166 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
170 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
172 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
173 adev->last_mm_index = v;
176 if ((reg * 4) < adev->rio_mem_size)
177 iowrite32(v, adev->rio_mem + (reg * 4));
179 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
180 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
183 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
189 * amdgpu_mm_rdoorbell - read a doorbell dword
191 * @adev: amdgpu_device pointer
192 * @index: doorbell index
194 * Returns the value in the doorbell aperture at the
195 * requested doorbell index (CIK).
197 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
199 if (index < adev->doorbell.num_doorbells) {
200 return readl(adev->doorbell.ptr + index);
202 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
208 * amdgpu_mm_wdoorbell - write a doorbell dword
210 * @adev: amdgpu_device pointer
211 * @index: doorbell index
214 * Writes @v to the doorbell aperture at the
215 * requested doorbell index (CIK).
217 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
219 if (index < adev->doorbell.num_doorbells) {
220 writel(v, adev->doorbell.ptr + index);
222 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
227 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
229 * @adev: amdgpu_device pointer
230 * @index: doorbell index
232 * Returns the value in the doorbell aperture at the
233 * requested doorbell index (VEGA10+).
235 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
237 if (index < adev->doorbell.num_doorbells) {
238 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
240 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
246 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
248 * @adev: amdgpu_device pointer
249 * @index: doorbell index
252 * Writes @v to the doorbell aperture at the
253 * requested doorbell index (VEGA10+).
255 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
257 if (index < adev->doorbell.num_doorbells) {
258 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
260 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
265 * amdgpu_invalid_rreg - dummy reg read function
267 * @adev: amdgpu device pointer
268 * @reg: offset of register
270 * Dummy register read function. Used for register blocks
271 * that certain asics don't have (all asics).
272 * Returns the value in the register.
274 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
276 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
282 * amdgpu_invalid_wreg - dummy reg write function
284 * @adev: amdgpu device pointer
285 * @reg: offset of register
286 * @v: value to write to the register
288 * Dummy register read function. Used for register blocks
289 * that certain asics don't have (all asics).
291 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
293 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
299 * amdgpu_block_invalid_rreg - dummy reg read function
301 * @adev: amdgpu device pointer
302 * @block: offset of instance
303 * @reg: offset of register
305 * Dummy register read function. Used for register blocks
306 * that certain asics don't have (all asics).
307 * Returns the value in the register.
309 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
310 uint32_t block, uint32_t reg)
312 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
319 * amdgpu_block_invalid_wreg - dummy reg write function
321 * @adev: amdgpu device pointer
322 * @block: offset of instance
323 * @reg: offset of register
324 * @v: value to write to the register
326 * Dummy register read function. Used for register blocks
327 * that certain asics don't have (all asics).
329 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
331 uint32_t reg, uint32_t v)
333 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
338 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
340 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
341 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
342 &adev->vram_scratch.robj,
343 &adev->vram_scratch.gpu_addr,
344 (void **)&adev->vram_scratch.ptr);
347 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
349 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
353 * amdgpu_program_register_sequence - program an array of registers.
355 * @adev: amdgpu_device pointer
356 * @registers: pointer to the register array
357 * @array_size: size of the register array
359 * Programs an array or registers with and and or masks.
360 * This is a helper for setting golden registers.
362 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
363 const u32 *registers,
364 const u32 array_size)
366 u32 tmp, reg, and_mask, or_mask;
372 for (i = 0; i < array_size; i +=3) {
373 reg = registers[i + 0];
374 and_mask = registers[i + 1];
375 or_mask = registers[i + 2];
377 if (and_mask == 0xffffffff) {
388 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
390 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
394 * GPU doorbell aperture helpers function.
397 * amdgpu_doorbell_init - Init doorbell driver information.
399 * @adev: amdgpu_device pointer
401 * Init doorbell driver information (CIK)
402 * Returns 0 on success, error on failure.
404 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
406 /* No doorbell on SI hardware generation */
407 if (adev->asic_type < CHIP_BONAIRE) {
408 adev->doorbell.base = 0;
409 adev->doorbell.size = 0;
410 adev->doorbell.num_doorbells = 0;
411 adev->doorbell.ptr = NULL;
415 /* doorbell bar mapping */
416 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
417 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
419 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
420 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
421 if (adev->doorbell.num_doorbells == 0)
424 adev->doorbell.ptr = ioremap(adev->doorbell.base,
425 adev->doorbell.num_doorbells *
427 if (adev->doorbell.ptr == NULL)
434 * amdgpu_doorbell_fini - Tear down doorbell driver information.
436 * @adev: amdgpu_device pointer
438 * Tear down doorbell driver information (CIK)
440 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
442 iounmap(adev->doorbell.ptr);
443 adev->doorbell.ptr = NULL;
447 * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
450 * @adev: amdgpu_device pointer
451 * @aperture_base: output returning doorbell aperture base physical address
452 * @aperture_size: output returning doorbell aperture size in bytes
453 * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
455 * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
456 * takes doorbells required for its own rings and reports the setup to amdkfd.
457 * amdgpu reserved doorbells are at the start of the doorbell aperture.
459 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
460 phys_addr_t *aperture_base,
461 size_t *aperture_size,
462 size_t *start_offset)
465 * The first num_doorbells are used by amdgpu.
466 * amdkfd takes whatever's left in the aperture.
468 if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
469 *aperture_base = adev->doorbell.base;
470 *aperture_size = adev->doorbell.size;
471 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
481 * Writeback is the method by which the GPU updates special pages in memory
482 * with the status of certain GPU events (fences, ring pointers,etc.).
486 * amdgpu_wb_fini - Disable Writeback and free memory
488 * @adev: amdgpu_device pointer
490 * Disables Writeback and frees the Writeback memory (all asics).
491 * Used at driver shutdown.
493 static void amdgpu_wb_fini(struct amdgpu_device *adev)
495 if (adev->wb.wb_obj) {
496 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
498 (void **)&adev->wb.wb);
499 adev->wb.wb_obj = NULL;
504 * amdgpu_wb_init- Init Writeback driver info and allocate memory
506 * @adev: amdgpu_device pointer
508 * Initializes writeback and allocates writeback memory (all asics).
509 * Used at driver startup.
510 * Returns 0 on success or an -error on failure.
512 static int amdgpu_wb_init(struct amdgpu_device *adev)
516 if (adev->wb.wb_obj == NULL) {
517 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
518 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
519 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
520 &adev->wb.wb_obj, &adev->wb.gpu_addr,
521 (void **)&adev->wb.wb);
523 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
527 adev->wb.num_wb = AMDGPU_MAX_WB;
528 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
530 /* clear wb memory */
531 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
538 * amdgpu_wb_get - Allocate a wb entry
540 * @adev: amdgpu_device pointer
543 * Allocate a wb slot for use by the driver (all asics).
544 * Returns 0 on success or -EINVAL on failure.
546 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
548 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
550 if (offset < adev->wb.num_wb) {
551 __set_bit(offset, adev->wb.used);
552 *wb = offset * 8; /* convert to dw offset */
560 * amdgpu_wb_free - Free a wb entry
562 * @adev: amdgpu_device pointer
565 * Free a wb slot allocated for use by the driver (all asics)
567 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
569 if (wb < adev->wb.num_wb)
570 __clear_bit(wb, adev->wb.used);
574 * amdgpu_vram_location - try to find VRAM location
575 * @adev: amdgpu device structure holding all necessary informations
576 * @mc: memory controller structure holding memory informations
577 * @base: base address at which to put VRAM
579 * Function will try to place VRAM at base address provided
580 * as parameter (which is so far either PCI aperture address or
581 * for IGP TOM base address).
583 * If there is not enough space to fit the unvisible VRAM in the 32bits
584 * address space then we limit the VRAM size to the aperture.
586 * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
587 * this shouldn't be a problem as we are using the PCI aperture as a reference.
588 * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
591 * Note: we use mc_vram_size as on some board we need to program the mc to
592 * cover the whole aperture even if VRAM size is inferior to aperture size
593 * Novell bug 204882 + along with lots of ubuntu ones
595 * Note: when limiting vram it's safe to overwritte real_vram_size because
596 * we are not in case where real_vram_size is inferior to mc_vram_size (ie
597 * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
600 * Note: IGP TOM addr should be the same as the aperture addr, we don't
601 * explicitly check for that though.
603 * FIXME: when reducing VRAM size align new size on power of 2.
605 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
607 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
609 mc->vram_start = base;
610 if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
611 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
612 mc->real_vram_size = mc->aper_size;
613 mc->mc_vram_size = mc->aper_size;
615 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
616 if (limit && limit < mc->real_vram_size)
617 mc->real_vram_size = limit;
618 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
619 mc->mc_vram_size >> 20, mc->vram_start,
620 mc->vram_end, mc->real_vram_size >> 20);
624 * amdgpu_gart_location - try to find GTT location
625 * @adev: amdgpu device structure holding all necessary informations
626 * @mc: memory controller structure holding memory informations
628 * Function will place try to place GTT before or after VRAM.
630 * If GTT size is bigger than space left then we ajust GTT size.
631 * Thus function will never fails.
633 * FIXME: when reducing GTT size align new size on power of 2.
635 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
637 u64 size_af, size_bf;
639 size_af = adev->mc.mc_mask - mc->vram_end;
640 size_bf = mc->vram_start;
641 if (size_bf > size_af) {
642 if (mc->gart_size > size_bf) {
643 dev_warn(adev->dev, "limiting GTT\n");
644 mc->gart_size = size_bf;
648 if (mc->gart_size > size_af) {
649 dev_warn(adev->dev, "limiting GTT\n");
650 mc->gart_size = size_af;
652 mc->gart_start = mc->vram_end + 1;
654 mc->gart_end = mc->gart_start + mc->gart_size - 1;
655 dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
656 mc->gart_size >> 20, mc->gart_start, mc->gart_end);
660 * GPU helpers function.
663 * amdgpu_need_post - check if the hw need post or not
665 * @adev: amdgpu_device pointer
667 * Check if the asic has been initialized (all asics) at driver startup
668 * or post is needed if hw reset is performed.
669 * Returns true if need or false if not.
671 bool amdgpu_need_post(struct amdgpu_device *adev)
675 if (adev->has_hw_reset) {
676 adev->has_hw_reset = false;
680 /* bios scratch used on CIK+ */
681 if (adev->asic_type >= CHIP_BONAIRE)
682 return amdgpu_atombios_scratch_need_asic_init(adev);
684 /* check MEM_SIZE for older asics */
685 reg = amdgpu_asic_get_config_memsize(adev);
687 if ((reg != 0) && (reg != 0xffffffff))
694 static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
696 if (amdgpu_sriov_vf(adev))
699 if (amdgpu_passthrough(adev)) {
700 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
701 * some old smc fw still need driver do vPost otherwise gpu hang, while
702 * those smc fw version above 22.15 doesn't have this flaw, so we force
703 * vpost executed for smc version below 22.15
705 if (adev->asic_type == CHIP_FIJI) {
708 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
709 /* force vPost if error occured */
713 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
714 if (fw_ver < 0x00160e00)
718 return amdgpu_need_post(adev);
722 * amdgpu_dummy_page_init - init dummy page used by the driver
724 * @adev: amdgpu_device pointer
726 * Allocate the dummy page used by the driver (all asics).
727 * This dummy page is used by the driver as a filler for gart entries
728 * when pages are taken out of the GART
729 * Returns 0 on sucess, -ENOMEM on failure.
731 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
733 if (adev->dummy_page.page)
735 adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
736 if (adev->dummy_page.page == NULL)
738 adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
739 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
740 if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
741 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
742 __free_page(adev->dummy_page.page);
743 adev->dummy_page.page = NULL;
750 * amdgpu_dummy_page_fini - free dummy page used by the driver
752 * @adev: amdgpu_device pointer
754 * Frees the dummy page used by the driver (all asics).
756 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
758 if (adev->dummy_page.page == NULL)
760 pci_unmap_page(adev->pdev, adev->dummy_page.addr,
761 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
762 __free_page(adev->dummy_page.page);
763 adev->dummy_page.page = NULL;
767 /* ATOM accessor methods */
769 * ATOM is an interpreted byte code stored in tables in the vbios. The
770 * driver registers callbacks to access registers and the interpreter
771 * in the driver parses the tables and executes then to program specific
772 * actions (set display modes, asic init, etc.). See amdgpu_atombios.c,
773 * atombios.h, and atom.c
777 * cail_pll_read - read PLL register
779 * @info: atom card_info pointer
780 * @reg: PLL register offset
782 * Provides a PLL register accessor for the atom interpreter (r4xx+).
783 * Returns the value of the PLL register.
785 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
791 * cail_pll_write - write PLL register
793 * @info: atom card_info pointer
794 * @reg: PLL register offset
795 * @val: value to write to the pll register
797 * Provides a PLL register accessor for the atom interpreter (r4xx+).
799 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
805 * cail_mc_read - read MC (Memory Controller) register
807 * @info: atom card_info pointer
808 * @reg: MC register offset
810 * Provides an MC register accessor for the atom interpreter (r4xx+).
811 * Returns the value of the MC register.
813 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
819 * cail_mc_write - write MC (Memory Controller) register
821 * @info: atom card_info pointer
822 * @reg: MC register offset
823 * @val: value to write to the pll register
825 * Provides a MC register accessor for the atom interpreter (r4xx+).
827 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
833 * cail_reg_write - write MMIO register
835 * @info: atom card_info pointer
836 * @reg: MMIO register offset
837 * @val: value to write to the pll register
839 * Provides a MMIO register accessor for the atom interpreter (r4xx+).
841 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
843 struct amdgpu_device *adev = info->dev->dev_private;
849 * cail_reg_read - read MMIO register
851 * @info: atom card_info pointer
852 * @reg: MMIO register offset
854 * Provides an MMIO register accessor for the atom interpreter (r4xx+).
855 * Returns the value of the MMIO register.
857 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
859 struct amdgpu_device *adev = info->dev->dev_private;
867 * cail_ioreg_write - write IO register
869 * @info: atom card_info pointer
870 * @reg: IO register offset
871 * @val: value to write to the pll register
873 * Provides a IO register accessor for the atom interpreter (r4xx+).
875 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
877 struct amdgpu_device *adev = info->dev->dev_private;
883 * cail_ioreg_read - read IO register
885 * @info: atom card_info pointer
886 * @reg: IO register offset
888 * Provides an IO register accessor for the atom interpreter (r4xx+).
889 * Returns the value of the IO register.
891 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
893 struct amdgpu_device *adev = info->dev->dev_private;
900 static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
901 struct device_attribute *attr,
904 struct drm_device *ddev = dev_get_drvdata(dev);
905 struct amdgpu_device *adev = ddev->dev_private;
906 struct atom_context *ctx = adev->mode_info.atom_context;
908 return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
911 static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
915 * amdgpu_atombios_fini - free the driver info and callbacks for atombios
917 * @adev: amdgpu_device pointer
919 * Frees the driver info and register access callbacks for the ATOM
920 * interpreter (r4xx+).
921 * Called at driver shutdown.
923 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
925 if (adev->mode_info.atom_context) {
926 kfree(adev->mode_info.atom_context->scratch);
927 kfree(adev->mode_info.atom_context->iio);
929 kfree(adev->mode_info.atom_context);
930 adev->mode_info.atom_context = NULL;
931 kfree(adev->mode_info.atom_card_info);
932 adev->mode_info.atom_card_info = NULL;
933 device_remove_file(adev->dev, &dev_attr_vbios_version);
937 * amdgpu_atombios_init - init the driver info and callbacks for atombios
939 * @adev: amdgpu_device pointer
941 * Initializes the driver info and register access callbacks for the
942 * ATOM interpreter (r4xx+).
943 * Returns 0 on sucess, -ENOMEM on failure.
944 * Called at driver startup.
946 static int amdgpu_atombios_init(struct amdgpu_device *adev)
948 struct card_info *atom_card_info =
949 kzalloc(sizeof(struct card_info), GFP_KERNEL);
955 adev->mode_info.atom_card_info = atom_card_info;
956 atom_card_info->dev = adev->ddev;
957 atom_card_info->reg_read = cail_reg_read;
958 atom_card_info->reg_write = cail_reg_write;
959 /* needed for iio ops */
961 atom_card_info->ioreg_read = cail_ioreg_read;
962 atom_card_info->ioreg_write = cail_ioreg_write;
964 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
965 atom_card_info->ioreg_read = cail_reg_read;
966 atom_card_info->ioreg_write = cail_reg_write;
968 atom_card_info->mc_read = cail_mc_read;
969 atom_card_info->mc_write = cail_mc_write;
970 atom_card_info->pll_read = cail_pll_read;
971 atom_card_info->pll_write = cail_pll_write;
973 adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
974 if (!adev->mode_info.atom_context) {
975 amdgpu_atombios_fini(adev);
979 mutex_init(&adev->mode_info.atom_context->mutex);
980 if (adev->is_atom_fw) {
981 amdgpu_atomfirmware_scratch_regs_init(adev);
982 amdgpu_atomfirmware_allocate_fb_scratch(adev);
984 amdgpu_atombios_scratch_regs_init(adev);
985 amdgpu_atombios_allocate_fb_scratch(adev);
988 ret = device_create_file(adev->dev, &dev_attr_vbios_version);
990 DRM_ERROR("Failed to create device file for VBIOS version\n");
997 /* if we get transitioned to only one device, take VGA back */
999 * amdgpu_vga_set_decode - enable/disable vga decode
1001 * @cookie: amdgpu_device pointer
1002 * @state: enable/disable vga decode
1004 * Enable/disable vga decode (all asics).
1005 * Returns VGA resource flags.
1007 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
1009 struct amdgpu_device *adev = cookie;
1010 amdgpu_asic_set_vga_state(adev, state);
1012 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1013 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1015 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1018 static void amdgpu_check_block_size(struct amdgpu_device *adev)
1020 /* defines number of bits in page table versus page directory,
1021 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1022 * page table and the remaining bits are in the page directory */
1023 if (amdgpu_vm_block_size == -1)
1026 if (amdgpu_vm_block_size < 9) {
1027 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1028 amdgpu_vm_block_size);
1032 if (amdgpu_vm_block_size > 24 ||
1033 (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1034 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1035 amdgpu_vm_block_size);
1042 amdgpu_vm_block_size = -1;
1045 static void amdgpu_check_vm_size(struct amdgpu_device *adev)
1047 /* no need to check the default value */
1048 if (amdgpu_vm_size == -1)
1051 if (!is_power_of_2(amdgpu_vm_size)) {
1052 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
1057 if (amdgpu_vm_size < 1) {
1058 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1064 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
1066 if (amdgpu_vm_size > 1024) {
1067 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1075 amdgpu_vm_size = -1;
1079 * amdgpu_check_arguments - validate module params
1081 * @adev: amdgpu_device pointer
1083 * Validates certain module parameters and updates
1084 * the associated values used by the driver (all asics).
1086 static void amdgpu_check_arguments(struct amdgpu_device *adev)
1088 if (amdgpu_sched_jobs < 4) {
1089 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1091 amdgpu_sched_jobs = 4;
1092 } else if (!is_power_of_2(amdgpu_sched_jobs)){
1093 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1095 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1098 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1099 /* gart size must be greater or equal to 32M */
1100 dev_warn(adev->dev, "gart size (%d) too small\n",
1102 amdgpu_gart_size = -1;
1105 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1106 /* gtt size must be greater or equal to 32M */
1107 dev_warn(adev->dev, "gtt size (%d) too small\n",
1109 amdgpu_gtt_size = -1;
1112 /* valid range is between 4 and 9 inclusive */
1113 if (amdgpu_vm_fragment_size != -1 &&
1114 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1115 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1116 amdgpu_vm_fragment_size = -1;
1119 amdgpu_check_vm_size(adev);
1121 amdgpu_check_block_size(adev);
1123 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1124 !is_power_of_2(amdgpu_vram_page_split))) {
1125 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1126 amdgpu_vram_page_split);
1127 amdgpu_vram_page_split = 1024;
1132 * amdgpu_switcheroo_set_state - set switcheroo state
1134 * @pdev: pci dev pointer
1135 * @state: vga_switcheroo state
1137 * Callback for the switcheroo driver. Suspends or resumes the
1138 * the asics before or after it is powered up using ACPI methods.
1140 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1142 struct drm_device *dev = pci_get_drvdata(pdev);
1144 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1147 if (state == VGA_SWITCHEROO_ON) {
1148 pr_info("amdgpu: switched on\n");
1149 /* don't suspend or resume card normally */
1150 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1152 amdgpu_device_resume(dev, true, true);
1154 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1155 drm_kms_helper_poll_enable(dev);
1157 pr_info("amdgpu: switched off\n");
1158 drm_kms_helper_poll_disable(dev);
1159 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1160 amdgpu_device_suspend(dev, true, true);
1161 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1166 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1168 * @pdev: pci dev pointer
1170 * Callback for the switcheroo driver. Check of the switcheroo
1171 * state can be changed.
1172 * Returns true if the state can be changed, false if not.
1174 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1176 struct drm_device *dev = pci_get_drvdata(pdev);
1179 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1180 * locking inversion with the driver load path. And the access here is
1181 * completely racy anyway. So don't bother with locking for now.
1183 return dev->open_count == 0;
1186 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1187 .set_gpu_state = amdgpu_switcheroo_set_state,
1189 .can_switch = amdgpu_switcheroo_can_switch,
1192 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1193 enum amd_ip_block_type block_type,
1194 enum amd_clockgating_state state)
1198 for (i = 0; i < adev->num_ip_blocks; i++) {
1199 if (!adev->ip_blocks[i].status.valid)
1201 if (adev->ip_blocks[i].version->type != block_type)
1203 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1205 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1206 (void *)adev, state);
1208 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1209 adev->ip_blocks[i].version->funcs->name, r);
1214 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1215 enum amd_ip_block_type block_type,
1216 enum amd_powergating_state state)
1220 for (i = 0; i < adev->num_ip_blocks; i++) {
1221 if (!adev->ip_blocks[i].status.valid)
1223 if (adev->ip_blocks[i].version->type != block_type)
1225 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1227 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1228 (void *)adev, state);
1230 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1231 adev->ip_blocks[i].version->funcs->name, r);
1236 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1240 for (i = 0; i < adev->num_ip_blocks; i++) {
1241 if (!adev->ip_blocks[i].status.valid)
1243 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1244 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1248 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1249 enum amd_ip_block_type block_type)
1253 for (i = 0; i < adev->num_ip_blocks; i++) {
1254 if (!adev->ip_blocks[i].status.valid)
1256 if (adev->ip_blocks[i].version->type == block_type) {
1257 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1267 bool amdgpu_is_idle(struct amdgpu_device *adev,
1268 enum amd_ip_block_type block_type)
1272 for (i = 0; i < adev->num_ip_blocks; i++) {
1273 if (!adev->ip_blocks[i].status.valid)
1275 if (adev->ip_blocks[i].version->type == block_type)
1276 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1282 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1283 enum amd_ip_block_type type)
1287 for (i = 0; i < adev->num_ip_blocks; i++)
1288 if (adev->ip_blocks[i].version->type == type)
1289 return &adev->ip_blocks[i];
1295 * amdgpu_ip_block_version_cmp
1297 * @adev: amdgpu_device pointer
1298 * @type: enum amd_ip_block_type
1299 * @major: major version
1300 * @minor: minor version
1302 * return 0 if equal or greater
1303 * return 1 if smaller or the ip_block doesn't exist
1305 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1306 enum amd_ip_block_type type,
1307 u32 major, u32 minor)
1309 struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1311 if (ip_block && ((ip_block->version->major > major) ||
1312 ((ip_block->version->major == major) &&
1313 (ip_block->version->minor >= minor))))
1320 * amdgpu_ip_block_add
1322 * @adev: amdgpu_device pointer
1323 * @ip_block_version: pointer to the IP to add
1325 * Adds the IP block driver information to the collection of IPs
1328 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1329 const struct amdgpu_ip_block_version *ip_block_version)
1331 if (!ip_block_version)
1334 DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1335 ip_block_version->funcs->name);
1337 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1342 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1344 adev->enable_virtual_display = false;
1346 if (amdgpu_virtual_display) {
1347 struct drm_device *ddev = adev->ddev;
1348 const char *pci_address_name = pci_name(ddev->pdev);
1349 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1351 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1352 pciaddstr_tmp = pciaddstr;
1353 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1354 pciaddname = strsep(&pciaddname_tmp, ",");
1355 if (!strcmp("all", pciaddname)
1356 || !strcmp(pci_address_name, pciaddname)) {
1360 adev->enable_virtual_display = true;
1363 res = kstrtol(pciaddname_tmp, 10,
1371 adev->mode_info.num_crtc = num_crtc;
1373 adev->mode_info.num_crtc = 1;
1379 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1380 amdgpu_virtual_display, pci_address_name,
1381 adev->enable_virtual_display, adev->mode_info.num_crtc);
1387 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1389 const char *chip_name;
1392 const struct gpu_info_firmware_header_v1_0 *hdr;
1394 adev->firmware.gpu_info_fw = NULL;
1396 switch (adev->asic_type) {
1400 case CHIP_POLARIS11:
1401 case CHIP_POLARIS10:
1402 case CHIP_POLARIS12:
1405 #ifdef CONFIG_DRM_AMDGPU_SI
1412 #ifdef CONFIG_DRM_AMDGPU_CIK
1422 chip_name = "vega10";
1425 chip_name = "raven";
1429 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1430 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1433 "Failed to load gpu_info firmware \"%s\"\n",
1437 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1440 "Failed to validate gpu_info firmware \"%s\"\n",
1445 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1446 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1448 switch (hdr->version_major) {
1451 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1452 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1453 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1455 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1456 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1457 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1458 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1459 adev->gfx.config.max_texture_channel_caches =
1460 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1461 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1462 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1463 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1464 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1465 adev->gfx.config.double_offchip_lds_buf =
1466 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1467 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1468 adev->gfx.cu_info.max_waves_per_simd =
1469 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1470 adev->gfx.cu_info.max_scratch_slots_per_cu =
1471 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1472 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1477 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1485 static int amdgpu_early_init(struct amdgpu_device *adev)
1489 amdgpu_device_enable_virtual_display(adev);
1491 switch (adev->asic_type) {
1495 case CHIP_POLARIS11:
1496 case CHIP_POLARIS10:
1497 case CHIP_POLARIS12:
1500 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1501 adev->family = AMDGPU_FAMILY_CZ;
1503 adev->family = AMDGPU_FAMILY_VI;
1505 r = vi_set_ip_blocks(adev);
1509 #ifdef CONFIG_DRM_AMDGPU_SI
1515 adev->family = AMDGPU_FAMILY_SI;
1516 r = si_set_ip_blocks(adev);
1521 #ifdef CONFIG_DRM_AMDGPU_CIK
1527 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1528 adev->family = AMDGPU_FAMILY_CI;
1530 adev->family = AMDGPU_FAMILY_KV;
1532 r = cik_set_ip_blocks(adev);
1539 if (adev->asic_type == CHIP_RAVEN)
1540 adev->family = AMDGPU_FAMILY_RV;
1542 adev->family = AMDGPU_FAMILY_AI;
1544 r = soc15_set_ip_blocks(adev);
1549 /* FIXME: not supported yet */
1553 r = amdgpu_device_parse_gpu_info_fw(adev);
1557 if (amdgpu_sriov_vf(adev)) {
1558 r = amdgpu_virt_request_full_gpu(adev, true);
1563 for (i = 0; i < adev->num_ip_blocks; i++) {
1564 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1565 DRM_ERROR("disabled ip block: %d <%s>\n",
1566 i, adev->ip_blocks[i].version->funcs->name);
1567 adev->ip_blocks[i].status.valid = false;
1569 if (adev->ip_blocks[i].version->funcs->early_init) {
1570 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1572 adev->ip_blocks[i].status.valid = false;
1574 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1575 adev->ip_blocks[i].version->funcs->name, r);
1578 adev->ip_blocks[i].status.valid = true;
1581 adev->ip_blocks[i].status.valid = true;
1586 adev->cg_flags &= amdgpu_cg_mask;
1587 adev->pg_flags &= amdgpu_pg_mask;
1592 static int amdgpu_init(struct amdgpu_device *adev)
1596 for (i = 0; i < adev->num_ip_blocks; i++) {
1597 if (!adev->ip_blocks[i].status.valid)
1599 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1601 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1602 adev->ip_blocks[i].version->funcs->name, r);
1605 adev->ip_blocks[i].status.sw = true;
1606 /* need to do gmc hw init early so we can allocate gpu mem */
1607 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1608 r = amdgpu_vram_scratch_init(adev);
1610 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1613 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1615 DRM_ERROR("hw_init %d failed %d\n", i, r);
1618 r = amdgpu_wb_init(adev);
1620 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1623 adev->ip_blocks[i].status.hw = true;
1625 /* right after GMC hw init, we create CSA */
1626 if (amdgpu_sriov_vf(adev)) {
1627 r = amdgpu_allocate_static_csa(adev);
1629 DRM_ERROR("allocate CSA failed %d\n", r);
1636 for (i = 0; i < adev->num_ip_blocks; i++) {
1637 if (!adev->ip_blocks[i].status.sw)
1639 /* gmc hw init is done early */
1640 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1642 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1644 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1645 adev->ip_blocks[i].version->funcs->name, r);
1648 adev->ip_blocks[i].status.hw = true;
1654 static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
1656 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1659 static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
1661 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1662 AMDGPU_RESET_MAGIC_NUM);
1665 static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
1669 for (i = 0; i < adev->num_ip_blocks; i++) {
1670 if (!adev->ip_blocks[i].status.valid)
1672 /* skip CG for VCE/UVD, it's handled specially */
1673 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1674 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1675 /* enable clockgating to save power */
1676 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1679 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1680 adev->ip_blocks[i].version->funcs->name, r);
1688 static int amdgpu_late_init(struct amdgpu_device *adev)
1692 for (i = 0; i < adev->num_ip_blocks; i++) {
1693 if (!adev->ip_blocks[i].status.valid)
1695 if (adev->ip_blocks[i].version->funcs->late_init) {
1696 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1698 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1699 adev->ip_blocks[i].version->funcs->name, r);
1702 adev->ip_blocks[i].status.late_initialized = true;
1706 mod_delayed_work(system_wq, &adev->late_init_work,
1707 msecs_to_jiffies(AMDGPU_RESUME_MS));
1709 amdgpu_fill_reset_magic(adev);
1714 static int amdgpu_fini(struct amdgpu_device *adev)
1718 /* need to disable SMC first */
1719 for (i = 0; i < adev->num_ip_blocks; i++) {
1720 if (!adev->ip_blocks[i].status.hw)
1722 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1723 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1724 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1725 AMD_CG_STATE_UNGATE);
1727 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1728 adev->ip_blocks[i].version->funcs->name, r);
1731 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1732 /* XXX handle errors */
1734 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1735 adev->ip_blocks[i].version->funcs->name, r);
1737 adev->ip_blocks[i].status.hw = false;
1742 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1743 if (!adev->ip_blocks[i].status.hw)
1745 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1746 amdgpu_wb_fini(adev);
1747 amdgpu_vram_scratch_fini(adev);
1750 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1751 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1752 /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1753 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1754 AMD_CG_STATE_UNGATE);
1756 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1757 adev->ip_blocks[i].version->funcs->name, r);
1762 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1763 /* XXX handle errors */
1765 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1766 adev->ip_blocks[i].version->funcs->name, r);
1769 adev->ip_blocks[i].status.hw = false;
1772 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1773 if (!adev->ip_blocks[i].status.sw)
1775 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1776 /* XXX handle errors */
1778 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1779 adev->ip_blocks[i].version->funcs->name, r);
1781 adev->ip_blocks[i].status.sw = false;
1782 adev->ip_blocks[i].status.valid = false;
1785 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1786 if (!adev->ip_blocks[i].status.late_initialized)
1788 if (adev->ip_blocks[i].version->funcs->late_fini)
1789 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1790 adev->ip_blocks[i].status.late_initialized = false;
1793 if (amdgpu_sriov_vf(adev)) {
1794 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1795 amdgpu_virt_release_full_gpu(adev, false);
1801 static void amdgpu_late_init_func_handler(struct work_struct *work)
1803 struct amdgpu_device *adev =
1804 container_of(work, struct amdgpu_device, late_init_work.work);
1805 amdgpu_late_set_cg_state(adev);
1808 int amdgpu_suspend(struct amdgpu_device *adev)
1812 if (amdgpu_sriov_vf(adev))
1813 amdgpu_virt_request_full_gpu(adev, false);
1815 /* ungate SMC block first */
1816 r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1817 AMD_CG_STATE_UNGATE);
1819 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1822 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1823 if (!adev->ip_blocks[i].status.valid)
1825 /* ungate blocks so that suspend can properly shut them down */
1826 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1827 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1828 AMD_CG_STATE_UNGATE);
1830 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1831 adev->ip_blocks[i].version->funcs->name, r);
1834 /* XXX handle errors */
1835 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1836 /* XXX handle errors */
1838 DRM_ERROR("suspend of IP block <%s> failed %d\n",
1839 adev->ip_blocks[i].version->funcs->name, r);
1843 if (amdgpu_sriov_vf(adev))
1844 amdgpu_virt_release_full_gpu(adev, false);
1849 static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
1853 static enum amd_ip_block_type ip_order[] = {
1854 AMD_IP_BLOCK_TYPE_GMC,
1855 AMD_IP_BLOCK_TYPE_COMMON,
1856 AMD_IP_BLOCK_TYPE_IH,
1859 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1861 struct amdgpu_ip_block *block;
1863 for (j = 0; j < adev->num_ip_blocks; j++) {
1864 block = &adev->ip_blocks[j];
1866 if (block->version->type != ip_order[i] ||
1867 !block->status.valid)
1870 r = block->version->funcs->hw_init(adev);
1871 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1878 static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
1882 static enum amd_ip_block_type ip_order[] = {
1883 AMD_IP_BLOCK_TYPE_SMC,
1884 AMD_IP_BLOCK_TYPE_DCE,
1885 AMD_IP_BLOCK_TYPE_GFX,
1886 AMD_IP_BLOCK_TYPE_SDMA,
1887 AMD_IP_BLOCK_TYPE_UVD,
1888 AMD_IP_BLOCK_TYPE_VCE
1891 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1893 struct amdgpu_ip_block *block;
1895 for (j = 0; j < adev->num_ip_blocks; j++) {
1896 block = &adev->ip_blocks[j];
1898 if (block->version->type != ip_order[i] ||
1899 !block->status.valid)
1902 r = block->version->funcs->hw_init(adev);
1903 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1910 static int amdgpu_resume_phase1(struct amdgpu_device *adev)
1914 for (i = 0; i < adev->num_ip_blocks; i++) {
1915 if (!adev->ip_blocks[i].status.valid)
1917 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1918 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1919 adev->ip_blocks[i].version->type ==
1920 AMD_IP_BLOCK_TYPE_IH) {
1921 r = adev->ip_blocks[i].version->funcs->resume(adev);
1923 DRM_ERROR("resume of IP block <%s> failed %d\n",
1924 adev->ip_blocks[i].version->funcs->name, r);
1933 static int amdgpu_resume_phase2(struct amdgpu_device *adev)
1937 for (i = 0; i < adev->num_ip_blocks; i++) {
1938 if (!adev->ip_blocks[i].status.valid)
1940 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1941 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1942 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1944 r = adev->ip_blocks[i].version->funcs->resume(adev);
1946 DRM_ERROR("resume of IP block <%s> failed %d\n",
1947 adev->ip_blocks[i].version->funcs->name, r);
1955 static int amdgpu_resume(struct amdgpu_device *adev)
1959 r = amdgpu_resume_phase1(adev);
1962 r = amdgpu_resume_phase2(adev);
1967 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1969 if (adev->is_atom_fw) {
1970 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
1971 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1973 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1974 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1979 * amdgpu_device_init - initialize the driver
1981 * @adev: amdgpu_device pointer
1982 * @pdev: drm dev pointer
1983 * @pdev: pci dev pointer
1984 * @flags: driver flags
1986 * Initializes the driver info and hw (all asics).
1987 * Returns 0 for success or an error on failure.
1988 * Called at driver startup.
1990 int amdgpu_device_init(struct amdgpu_device *adev,
1991 struct drm_device *ddev,
1992 struct pci_dev *pdev,
1996 bool runtime = false;
1999 adev->shutdown = false;
2000 adev->dev = &pdev->dev;
2003 adev->flags = flags;
2004 adev->asic_type = flags & AMD_ASIC_MASK;
2005 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2006 adev->mc.gart_size = 512 * 1024 * 1024;
2007 adev->accel_working = false;
2008 adev->num_rings = 0;
2009 adev->mman.buffer_funcs = NULL;
2010 adev->mman.buffer_funcs_ring = NULL;
2011 adev->vm_manager.vm_pte_funcs = NULL;
2012 adev->vm_manager.vm_pte_num_rings = 0;
2013 adev->gart.gart_funcs = NULL;
2014 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2016 adev->smc_rreg = &amdgpu_invalid_rreg;
2017 adev->smc_wreg = &amdgpu_invalid_wreg;
2018 adev->pcie_rreg = &amdgpu_invalid_rreg;
2019 adev->pcie_wreg = &amdgpu_invalid_wreg;
2020 adev->pciep_rreg = &amdgpu_invalid_rreg;
2021 adev->pciep_wreg = &amdgpu_invalid_wreg;
2022 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2023 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2024 adev->didt_rreg = &amdgpu_invalid_rreg;
2025 adev->didt_wreg = &amdgpu_invalid_wreg;
2026 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2027 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2028 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2029 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2032 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2033 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2034 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2036 /* mutex initialization are all done here so we
2037 * can recall function without having locking issues */
2038 atomic_set(&adev->irq.ih.lock, 0);
2039 mutex_init(&adev->firmware.mutex);
2040 mutex_init(&adev->pm.mutex);
2041 mutex_init(&adev->gfx.gpu_clock_mutex);
2042 mutex_init(&adev->srbm_mutex);
2043 mutex_init(&adev->grbm_idx_mutex);
2044 mutex_init(&adev->mn_lock);
2045 hash_init(adev->mn_hash);
2047 amdgpu_check_arguments(adev);
2049 spin_lock_init(&adev->mmio_idx_lock);
2050 spin_lock_init(&adev->smc_idx_lock);
2051 spin_lock_init(&adev->pcie_idx_lock);
2052 spin_lock_init(&adev->uvd_ctx_idx_lock);
2053 spin_lock_init(&adev->didt_idx_lock);
2054 spin_lock_init(&adev->gc_cac_idx_lock);
2055 spin_lock_init(&adev->se_cac_idx_lock);
2056 spin_lock_init(&adev->audio_endpt_idx_lock);
2057 spin_lock_init(&adev->mm_stats.lock);
2059 INIT_LIST_HEAD(&adev->shadow_list);
2060 mutex_init(&adev->shadow_list_lock);
2062 INIT_LIST_HEAD(&adev->gtt_list);
2063 spin_lock_init(&adev->gtt_list_lock);
2065 INIT_LIST_HEAD(&adev->ring_lru_list);
2066 spin_lock_init(&adev->ring_lru_list_lock);
2068 INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
2070 /* Registers mapping */
2071 /* TODO: block userspace mapping of io register */
2072 if (adev->asic_type >= CHIP_BONAIRE) {
2073 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2074 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2076 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2077 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2080 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2081 if (adev->rmmio == NULL) {
2084 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2085 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2087 /* doorbell bar mapping */
2088 amdgpu_doorbell_init(adev);
2090 /* io port mapping */
2091 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2092 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2093 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2094 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2098 if (adev->rio_mem == NULL)
2099 DRM_INFO("PCI I/O BAR is not found.\n");
2101 /* early init functions */
2102 r = amdgpu_early_init(adev);
2106 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2107 /* this will fail for cards that aren't VGA class devices, just
2109 vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
2111 if (amdgpu_runtime_pm == 1)
2113 if (amdgpu_device_is_px(ddev))
2115 if (!pci_is_thunderbolt_attached(adev->pdev))
2116 vga_switcheroo_register_client(adev->pdev,
2117 &amdgpu_switcheroo_ops, runtime);
2119 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2122 if (!amdgpu_get_bios(adev)) {
2127 r = amdgpu_atombios_init(adev);
2129 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2130 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2134 /* detect if we are with an SRIOV vbios */
2135 amdgpu_device_detect_sriov_bios(adev);
2137 /* Post card if necessary */
2138 if (amdgpu_vpost_needed(adev)) {
2140 dev_err(adev->dev, "no vBIOS found\n");
2141 amdgpu_vf_error_put(AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2145 DRM_INFO("GPU posting now...\n");
2146 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2148 dev_err(adev->dev, "gpu post error!\n");
2149 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_POST_ERROR, 0, 0);
2153 DRM_INFO("GPU post is not needed\n");
2156 if (adev->is_atom_fw) {
2157 /* Initialize clocks */
2158 r = amdgpu_atomfirmware_get_clock_info(adev);
2160 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2161 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2165 /* Initialize clocks */
2166 r = amdgpu_atombios_get_clock_info(adev);
2168 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2169 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2172 /* init i2c buses */
2173 amdgpu_atombios_i2c_init(adev);
2177 r = amdgpu_fence_driver_init(adev);
2179 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2180 amdgpu_vf_error_put(AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2184 /* init the mode config */
2185 drm_mode_config_init(adev->ddev);
2187 r = amdgpu_init(adev);
2189 dev_err(adev->dev, "amdgpu_init failed\n");
2190 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2195 adev->accel_working = true;
2197 amdgpu_vm_check_compute_bug(adev);
2199 /* Initialize the buffer migration limit. */
2200 if (amdgpu_moverate >= 0)
2201 max_MBps = amdgpu_moverate;
2203 max_MBps = 8; /* Allow 8 MB/s. */
2204 /* Get a log2 for easy divisions. */
2205 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2207 r = amdgpu_ib_pool_init(adev);
2209 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2210 amdgpu_vf_error_put(AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2214 r = amdgpu_ib_ring_tests(adev);
2216 DRM_ERROR("ib ring test failed (%d).\n", r);
2218 amdgpu_fbdev_init(adev);
2220 r = amdgpu_gem_debugfs_init(adev);
2222 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2224 r = amdgpu_debugfs_regs_init(adev);
2226 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2228 r = amdgpu_debugfs_test_ib_ring_init(adev);
2230 DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
2232 r = amdgpu_debugfs_firmware_init(adev);
2234 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2236 r = amdgpu_debugfs_vbios_dump_init(adev);
2238 DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
2240 if ((amdgpu_testing & 1)) {
2241 if (adev->accel_working)
2242 amdgpu_test_moves(adev);
2244 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2246 if (amdgpu_benchmarking) {
2247 if (adev->accel_working)
2248 amdgpu_benchmark(adev, amdgpu_benchmarking);
2250 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2253 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2254 * explicit gating rather than handling it automatically.
2256 r = amdgpu_late_init(adev);
2258 dev_err(adev->dev, "amdgpu_late_init failed\n");
2259 amdgpu_vf_error_put(AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2266 amdgpu_vf_error_trans_all(adev);
2268 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2273 * amdgpu_device_fini - tear down the driver
2275 * @adev: amdgpu_device pointer
2277 * Tear down the driver info (all asics).
2278 * Called at driver shutdown.
2280 void amdgpu_device_fini(struct amdgpu_device *adev)
2284 DRM_INFO("amdgpu: finishing device.\n");
2285 adev->shutdown = true;
2286 if (adev->mode_info.mode_config_initialized)
2287 drm_crtc_force_disable_all(adev->ddev);
2288 /* evict vram memory */
2289 amdgpu_bo_evict_vram(adev);
2290 amdgpu_ib_pool_fini(adev);
2291 amdgpu_fence_driver_fini(adev);
2292 amdgpu_fbdev_fini(adev);
2293 r = amdgpu_fini(adev);
2294 if (adev->firmware.gpu_info_fw) {
2295 release_firmware(adev->firmware.gpu_info_fw);
2296 adev->firmware.gpu_info_fw = NULL;
2298 adev->accel_working = false;
2299 cancel_delayed_work_sync(&adev->late_init_work);
2300 /* free i2c buses */
2301 amdgpu_i2c_fini(adev);
2302 amdgpu_atombios_fini(adev);
2305 if (!pci_is_thunderbolt_attached(adev->pdev))
2306 vga_switcheroo_unregister_client(adev->pdev);
2307 if (adev->flags & AMD_IS_PX)
2308 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2309 vga_client_register(adev->pdev, NULL, NULL, NULL);
2311 pci_iounmap(adev->pdev, adev->rio_mem);
2312 adev->rio_mem = NULL;
2313 iounmap(adev->rmmio);
2315 amdgpu_doorbell_fini(adev);
2316 amdgpu_debugfs_regs_cleanup(adev);
2324 * amdgpu_device_suspend - initiate device suspend
2326 * @pdev: drm dev pointer
2327 * @state: suspend state
2329 * Puts the hw in the suspend state (all asics).
2330 * Returns 0 for success or an error on failure.
2331 * Called at driver suspend.
2333 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2335 struct amdgpu_device *adev;
2336 struct drm_crtc *crtc;
2337 struct drm_connector *connector;
2340 if (dev == NULL || dev->dev_private == NULL) {
2344 adev = dev->dev_private;
2346 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2349 drm_kms_helper_poll_disable(dev);
2351 /* turn off display hw */
2352 drm_modeset_lock_all(dev);
2353 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2354 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2356 drm_modeset_unlock_all(dev);
2358 amdgpu_amdkfd_suspend(adev);
2360 /* unpin the front buffers and cursors */
2361 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2362 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2363 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2364 struct amdgpu_bo *robj;
2366 if (amdgpu_crtc->cursor_bo) {
2367 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2368 r = amdgpu_bo_reserve(aobj, true);
2370 amdgpu_bo_unpin(aobj);
2371 amdgpu_bo_unreserve(aobj);
2375 if (rfb == NULL || rfb->obj == NULL) {
2378 robj = gem_to_amdgpu_bo(rfb->obj);
2379 /* don't unpin kernel fb objects */
2380 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2381 r = amdgpu_bo_reserve(robj, true);
2383 amdgpu_bo_unpin(robj);
2384 amdgpu_bo_unreserve(robj);
2388 /* evict vram memory */
2389 amdgpu_bo_evict_vram(adev);
2391 amdgpu_fence_driver_suspend(adev);
2393 r = amdgpu_suspend(adev);
2395 /* evict remaining vram memory
2396 * This second call to evict vram is to evict the gart page table
2399 amdgpu_bo_evict_vram(adev);
2401 amdgpu_atombios_scratch_regs_save(adev);
2402 pci_save_state(dev->pdev);
2404 /* Shut down the device */
2405 pci_disable_device(dev->pdev);
2406 pci_set_power_state(dev->pdev, PCI_D3hot);
2408 r = amdgpu_asic_reset(adev);
2410 DRM_ERROR("amdgpu asic reset failed\n");
2415 amdgpu_fbdev_set_suspend(adev, 1);
2422 * amdgpu_device_resume - initiate device resume
2424 * @pdev: drm dev pointer
2426 * Bring the hw back to operating state (all asics).
2427 * Returns 0 for success or an error on failure.
2428 * Called at driver resume.
2430 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2432 struct drm_connector *connector;
2433 struct amdgpu_device *adev = dev->dev_private;
2434 struct drm_crtc *crtc;
2437 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2444 pci_set_power_state(dev->pdev, PCI_D0);
2445 pci_restore_state(dev->pdev);
2446 r = pci_enable_device(dev->pdev);
2450 amdgpu_atombios_scratch_regs_restore(adev);
2453 if (amdgpu_need_post(adev)) {
2454 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2456 DRM_ERROR("amdgpu asic init failed\n");
2459 r = amdgpu_resume(adev);
2461 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2464 amdgpu_fence_driver_resume(adev);
2467 r = amdgpu_ib_ring_tests(adev);
2469 DRM_ERROR("ib ring test failed (%d).\n", r);
2472 r = amdgpu_late_init(adev);
2477 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2478 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2480 if (amdgpu_crtc->cursor_bo) {
2481 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2482 r = amdgpu_bo_reserve(aobj, true);
2484 r = amdgpu_bo_pin(aobj,
2485 AMDGPU_GEM_DOMAIN_VRAM,
2486 &amdgpu_crtc->cursor_addr);
2488 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2489 amdgpu_bo_unreserve(aobj);
2493 r = amdgpu_amdkfd_resume(adev);
2497 /* blat the mode back in */
2499 drm_helper_resume_force_mode(dev);
2500 /* turn on display hw */
2501 drm_modeset_lock_all(dev);
2502 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2503 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2505 drm_modeset_unlock_all(dev);
2508 drm_kms_helper_poll_enable(dev);
2511 * Most of the connector probing functions try to acquire runtime pm
2512 * refs to ensure that the GPU is powered on when connector polling is
2513 * performed. Since we're calling this from a runtime PM callback,
2514 * trying to acquire rpm refs will cause us to deadlock.
2516 * Since we're guaranteed to be holding the rpm lock, it's safe to
2517 * temporarily disable the rpm helpers so this doesn't deadlock us.
2520 dev->dev->power.disable_depth++;
2522 drm_helper_hpd_irq_event(dev);
2524 dev->dev->power.disable_depth--;
2528 amdgpu_fbdev_set_suspend(adev, 0);
2537 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2540 bool asic_hang = false;
2542 for (i = 0; i < adev->num_ip_blocks; i++) {
2543 if (!adev->ip_blocks[i].status.valid)
2545 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2546 adev->ip_blocks[i].status.hang =
2547 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2548 if (adev->ip_blocks[i].status.hang) {
2549 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2556 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2560 for (i = 0; i < adev->num_ip_blocks; i++) {
2561 if (!adev->ip_blocks[i].status.valid)
2563 if (adev->ip_blocks[i].status.hang &&
2564 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2565 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2574 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2578 for (i = 0; i < adev->num_ip_blocks; i++) {
2579 if (!adev->ip_blocks[i].status.valid)
2581 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2582 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2583 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2584 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2585 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2586 if (adev->ip_blocks[i].status.hang) {
2587 DRM_INFO("Some block need full reset!\n");
2595 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2599 for (i = 0; i < adev->num_ip_blocks; i++) {
2600 if (!adev->ip_blocks[i].status.valid)
2602 if (adev->ip_blocks[i].status.hang &&
2603 adev->ip_blocks[i].version->funcs->soft_reset) {
2604 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2613 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2617 for (i = 0; i < adev->num_ip_blocks; i++) {
2618 if (!adev->ip_blocks[i].status.valid)
2620 if (adev->ip_blocks[i].status.hang &&
2621 adev->ip_blocks[i].version->funcs->post_soft_reset)
2622 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2630 bool amdgpu_need_backup(struct amdgpu_device *adev)
2632 if (adev->flags & AMD_IS_APU)
2635 return amdgpu_lockup_timeout > 0 ? true : false;
2638 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2639 struct amdgpu_ring *ring,
2640 struct amdgpu_bo *bo,
2641 struct dma_fence **fence)
2649 r = amdgpu_bo_reserve(bo, true);
2652 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2653 /* if bo has been evicted, then no need to recover */
2654 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2655 r = amdgpu_bo_validate(bo->shadow);
2657 DRM_ERROR("bo validate failed!\n");
2661 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2664 DRM_ERROR("recover page table failed!\n");
2669 amdgpu_bo_unreserve(bo);
2674 * amdgpu_sriov_gpu_reset - reset the asic
2676 * @adev: amdgpu device pointer
2677 * @job: which job trigger hang
2679 * Attempt the reset the GPU if it has hung (all asics).
2681 * Returns 0 for success or an error on failure.
2683 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
2687 struct amdgpu_bo *bo, *tmp;
2688 struct amdgpu_ring *ring;
2689 struct dma_fence *fence = NULL, *next = NULL;
2691 mutex_lock(&adev->virt.lock_reset);
2692 atomic_inc(&adev->gpu_reset_counter);
2693 adev->gfx.in_reset = true;
2696 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2698 /* we start from the ring trigger GPU hang */
2699 j = job ? job->ring->idx : 0;
2701 /* block scheduler */
2702 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2703 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2704 if (!ring || !ring->sched.thread)
2707 kthread_park(ring->sched.thread);
2712 /* here give the last chance to check if job removed from mirror-list
2713 * since we already pay some time on kthread_park */
2714 if (job && list_empty(&job->base.node)) {
2715 kthread_unpark(ring->sched.thread);
2719 if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
2720 amd_sched_job_kickout(&job->base);
2722 /* only do job_reset on the hang ring if @job not NULL */
2723 amd_sched_hw_job_reset(&ring->sched);
2725 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2726 amdgpu_fence_driver_force_completion_ring(ring);
2729 /* request to take full control of GPU before re-initialization */
2731 amdgpu_virt_reset_gpu(adev);
2733 amdgpu_virt_request_full_gpu(adev, true);
2736 /* Resume IP prior to SMC */
2737 amdgpu_sriov_reinit_early(adev);
2739 /* we need recover gart prior to run SMC/CP/SDMA resume */
2740 amdgpu_ttm_recover_gart(adev);
2742 /* now we are okay to resume SMC/CP/SDMA */
2743 amdgpu_sriov_reinit_late(adev);
2745 amdgpu_irq_gpu_reset_resume_helper(adev);
2747 if (amdgpu_ib_ring_tests(adev))
2748 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2750 /* release full control of GPU after ib test */
2751 amdgpu_virt_release_full_gpu(adev, true);
2753 DRM_INFO("recover vram bo from shadow\n");
2755 ring = adev->mman.buffer_funcs_ring;
2756 mutex_lock(&adev->shadow_list_lock);
2757 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2759 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2761 r = dma_fence_wait(fence, false);
2763 WARN(r, "recovery from shadow isn't completed\n");
2768 dma_fence_put(fence);
2771 mutex_unlock(&adev->shadow_list_lock);
2774 r = dma_fence_wait(fence, false);
2776 WARN(r, "recovery from shadow isn't completed\n");
2778 dma_fence_put(fence);
2780 for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
2781 ring = adev->rings[i % AMDGPU_MAX_RINGS];
2782 if (!ring || !ring->sched.thread)
2785 if (job && j != i) {
2786 kthread_unpark(ring->sched.thread);
2790 amd_sched_job_recovery(&ring->sched);
2791 kthread_unpark(ring->sched.thread);
2794 drm_helper_resume_force_mode(adev->ddev);
2796 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2798 /* bad news, how to tell it to userspace ? */
2799 dev_info(adev->dev, "GPU reset failed\n");
2801 dev_info(adev->dev, "GPU reset successed!\n");
2804 adev->gfx.in_reset = false;
2805 mutex_unlock(&adev->virt.lock_reset);
2810 * amdgpu_gpu_reset - reset the asic
2812 * @adev: amdgpu device pointer
2814 * Attempt the reset the GPU if it has hung (all asics).
2815 * Returns 0 for success or an error on failure.
2817 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2821 bool need_full_reset, vram_lost = false;
2823 if (!amdgpu_check_soft_reset(adev)) {
2824 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2828 atomic_inc(&adev->gpu_reset_counter);
2831 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2833 /* block scheduler */
2834 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2835 struct amdgpu_ring *ring = adev->rings[i];
2837 if (!ring || !ring->sched.thread)
2839 kthread_park(ring->sched.thread);
2840 amd_sched_hw_job_reset(&ring->sched);
2842 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2843 amdgpu_fence_driver_force_completion(adev);
2845 need_full_reset = amdgpu_need_full_reset(adev);
2847 if (!need_full_reset) {
2848 amdgpu_pre_soft_reset(adev);
2849 r = amdgpu_soft_reset(adev);
2850 amdgpu_post_soft_reset(adev);
2851 if (r || amdgpu_check_soft_reset(adev)) {
2852 DRM_INFO("soft reset failed, will fallback to full reset!\n");
2853 need_full_reset = true;
2857 if (need_full_reset) {
2858 r = amdgpu_suspend(adev);
2861 amdgpu_atombios_scratch_regs_save(adev);
2862 r = amdgpu_asic_reset(adev);
2863 amdgpu_atombios_scratch_regs_restore(adev);
2865 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2868 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2869 r = amdgpu_resume_phase1(adev);
2872 vram_lost = amdgpu_check_vram_lost(adev);
2874 DRM_ERROR("VRAM is lost!\n");
2875 atomic_inc(&adev->vram_lost_counter);
2877 r = amdgpu_ttm_recover_gart(adev);
2880 r = amdgpu_resume_phase2(adev);
2884 amdgpu_fill_reset_magic(adev);
2889 amdgpu_irq_gpu_reset_resume_helper(adev);
2890 r = amdgpu_ib_ring_tests(adev);
2892 dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2893 r = amdgpu_suspend(adev);
2894 need_full_reset = true;
2898 * recovery vm page tables, since we cannot depend on VRAM is
2899 * consistent after gpu full reset.
2901 if (need_full_reset && amdgpu_need_backup(adev)) {
2902 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2903 struct amdgpu_bo *bo, *tmp;
2904 struct dma_fence *fence = NULL, *next = NULL;
2906 DRM_INFO("recover vram bo from shadow\n");
2907 mutex_lock(&adev->shadow_list_lock);
2908 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2910 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2912 r = dma_fence_wait(fence, false);
2914 WARN(r, "recovery from shadow isn't completed\n");
2919 dma_fence_put(fence);
2922 mutex_unlock(&adev->shadow_list_lock);
2924 r = dma_fence_wait(fence, false);
2926 WARN(r, "recovery from shadow isn't completed\n");
2928 dma_fence_put(fence);
2930 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2931 struct amdgpu_ring *ring = adev->rings[i];
2933 if (!ring || !ring->sched.thread)
2936 amd_sched_job_recovery(&ring->sched);
2937 kthread_unpark(ring->sched.thread);
2940 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2941 amdgpu_vf_error_put(AMDGIM_ERROR_VF_ASIC_RESUME_FAIL, 0, r);
2942 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2943 if (adev->rings[i] && adev->rings[i]->sched.thread) {
2944 kthread_unpark(adev->rings[i]->sched.thread);
2949 drm_helper_resume_force_mode(adev->ddev);
2951 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2953 /* bad news, how to tell it to userspace ? */
2954 dev_info(adev->dev, "GPU reset failed\n");
2955 amdgpu_vf_error_put(AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
2958 dev_info(adev->dev, "GPU reset successed!\n");
2961 amdgpu_vf_error_trans_all(adev);
2965 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2970 if (amdgpu_pcie_gen_cap)
2971 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2973 if (amdgpu_pcie_lane_cap)
2974 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2976 /* covers APUs as well */
2977 if (pci_is_root_bus(adev->pdev->bus)) {
2978 if (adev->pm.pcie_gen_mask == 0)
2979 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2980 if (adev->pm.pcie_mlw_mask == 0)
2981 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2985 if (adev->pm.pcie_gen_mask == 0) {
2986 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2988 adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2989 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2990 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2992 if (mask & DRM_PCIE_SPEED_25)
2993 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2994 if (mask & DRM_PCIE_SPEED_50)
2995 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2996 if (mask & DRM_PCIE_SPEED_80)
2997 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2999 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3002 if (adev->pm.pcie_mlw_mask == 0) {
3003 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
3007 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
3008 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3009 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3010 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3011 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3012 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3013 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3016 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
3017 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3018 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3019 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3020 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3021 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3024 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
3025 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3026 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3027 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3028 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3031 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
3032 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3033 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3034 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3037 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
3038 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3039 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3042 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3043 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3046 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3052 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3060 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
3061 const struct drm_info_list *files,
3066 for (i = 0; i < adev->debugfs_count; i++) {
3067 if (adev->debugfs[i].files == files) {
3068 /* Already registered */
3073 i = adev->debugfs_count + 1;
3074 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
3075 DRM_ERROR("Reached maximum number of debugfs components.\n");
3076 DRM_ERROR("Report so we increase "
3077 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
3080 adev->debugfs[adev->debugfs_count].files = files;
3081 adev->debugfs[adev->debugfs_count].num_files = nfiles;
3082 adev->debugfs_count = i;
3083 #if defined(CONFIG_DEBUG_FS)
3084 drm_debugfs_create_files(files, nfiles,
3085 adev->ddev->primary->debugfs_root,
3086 adev->ddev->primary);
3091 #if defined(CONFIG_DEBUG_FS)
3093 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
3094 size_t size, loff_t *pos)
3096 struct amdgpu_device *adev = file_inode(f)->i_private;
3099 bool pm_pg_lock, use_bank;
3100 unsigned instance_bank, sh_bank, se_bank;
3102 if (size & 0x3 || *pos & 0x3)
3105 /* are we reading registers for which a PG lock is necessary? */
3106 pm_pg_lock = (*pos >> 23) & 1;
3108 if (*pos & (1ULL << 62)) {
3109 se_bank = (*pos >> 24) & 0x3FF;
3110 sh_bank = (*pos >> 34) & 0x3FF;
3111 instance_bank = (*pos >> 44) & 0x3FF;
3113 if (se_bank == 0x3FF)
3114 se_bank = 0xFFFFFFFF;
3115 if (sh_bank == 0x3FF)
3116 sh_bank = 0xFFFFFFFF;
3117 if (instance_bank == 0x3FF)
3118 instance_bank = 0xFFFFFFFF;
3124 *pos &= (1UL << 22) - 1;
3127 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3128 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3130 mutex_lock(&adev->grbm_idx_mutex);
3131 amdgpu_gfx_select_se_sh(adev, se_bank,
3132 sh_bank, instance_bank);
3136 mutex_lock(&adev->pm.mutex);
3141 if (*pos > adev->rmmio_size)
3144 value = RREG32(*pos >> 2);
3145 r = put_user(value, (uint32_t *)buf);
3159 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3160 mutex_unlock(&adev->grbm_idx_mutex);
3164 mutex_unlock(&adev->pm.mutex);
3169 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
3170 size_t size, loff_t *pos)
3172 struct amdgpu_device *adev = file_inode(f)->i_private;
3175 bool pm_pg_lock, use_bank;
3176 unsigned instance_bank, sh_bank, se_bank;
3178 if (size & 0x3 || *pos & 0x3)
3181 /* are we reading registers for which a PG lock is necessary? */
3182 pm_pg_lock = (*pos >> 23) & 1;
3184 if (*pos & (1ULL << 62)) {
3185 se_bank = (*pos >> 24) & 0x3FF;
3186 sh_bank = (*pos >> 34) & 0x3FF;
3187 instance_bank = (*pos >> 44) & 0x3FF;
3189 if (se_bank == 0x3FF)
3190 se_bank = 0xFFFFFFFF;
3191 if (sh_bank == 0x3FF)
3192 sh_bank = 0xFFFFFFFF;
3193 if (instance_bank == 0x3FF)
3194 instance_bank = 0xFFFFFFFF;
3200 *pos &= (1UL << 22) - 1;
3203 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
3204 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
3206 mutex_lock(&adev->grbm_idx_mutex);
3207 amdgpu_gfx_select_se_sh(adev, se_bank,
3208 sh_bank, instance_bank);
3212 mutex_lock(&adev->pm.mutex);
3217 if (*pos > adev->rmmio_size)
3220 r = get_user(value, (uint32_t *)buf);
3224 WREG32(*pos >> 2, value);
3233 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3234 mutex_unlock(&adev->grbm_idx_mutex);
3238 mutex_unlock(&adev->pm.mutex);
3243 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
3244 size_t size, loff_t *pos)
3246 struct amdgpu_device *adev = file_inode(f)->i_private;
3250 if (size & 0x3 || *pos & 0x3)
3256 value = RREG32_PCIE(*pos >> 2);
3257 r = put_user(value, (uint32_t *)buf);
3270 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
3271 size_t size, loff_t *pos)
3273 struct amdgpu_device *adev = file_inode(f)->i_private;
3277 if (size & 0x3 || *pos & 0x3)
3283 r = get_user(value, (uint32_t *)buf);
3287 WREG32_PCIE(*pos >> 2, value);
3298 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
3299 size_t size, loff_t *pos)
3301 struct amdgpu_device *adev = file_inode(f)->i_private;
3305 if (size & 0x3 || *pos & 0x3)
3311 value = RREG32_DIDT(*pos >> 2);
3312 r = put_user(value, (uint32_t *)buf);
3325 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
3326 size_t size, loff_t *pos)
3328 struct amdgpu_device *adev = file_inode(f)->i_private;
3332 if (size & 0x3 || *pos & 0x3)
3338 r = get_user(value, (uint32_t *)buf);
3342 WREG32_DIDT(*pos >> 2, value);
3353 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
3354 size_t size, loff_t *pos)
3356 struct amdgpu_device *adev = file_inode(f)->i_private;
3360 if (size & 0x3 || *pos & 0x3)
3366 value = RREG32_SMC(*pos);
3367 r = put_user(value, (uint32_t *)buf);
3380 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3381 size_t size, loff_t *pos)
3383 struct amdgpu_device *adev = file_inode(f)->i_private;
3387 if (size & 0x3 || *pos & 0x3)
3393 r = get_user(value, (uint32_t *)buf);
3397 WREG32_SMC(*pos, value);
3408 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3409 size_t size, loff_t *pos)
3411 struct amdgpu_device *adev = file_inode(f)->i_private;
3414 uint32_t *config, no_regs = 0;
3416 if (size & 0x3 || *pos & 0x3)
3419 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
3423 /* version, increment each time something is added */
3424 config[no_regs++] = 3;
3425 config[no_regs++] = adev->gfx.config.max_shader_engines;
3426 config[no_regs++] = adev->gfx.config.max_tile_pipes;
3427 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3428 config[no_regs++] = adev->gfx.config.max_sh_per_se;
3429 config[no_regs++] = adev->gfx.config.max_backends_per_se;
3430 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3431 config[no_regs++] = adev->gfx.config.max_gprs;
3432 config[no_regs++] = adev->gfx.config.max_gs_threads;
3433 config[no_regs++] = adev->gfx.config.max_hw_contexts;
3434 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3435 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3436 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3437 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3438 config[no_regs++] = adev->gfx.config.num_tile_pipes;
3439 config[no_regs++] = adev->gfx.config.backend_enable_mask;
3440 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3441 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3442 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3443 config[no_regs++] = adev->gfx.config.num_gpus;
3444 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3445 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3446 config[no_regs++] = adev->gfx.config.gb_addr_config;
3447 config[no_regs++] = adev->gfx.config.num_rbs;
3450 config[no_regs++] = adev->rev_id;
3451 config[no_regs++] = adev->pg_flags;
3452 config[no_regs++] = adev->cg_flags;
3455 config[no_regs++] = adev->family;
3456 config[no_regs++] = adev->external_rev_id;
3459 config[no_regs++] = adev->pdev->device;
3460 config[no_regs++] = adev->pdev->revision;
3461 config[no_regs++] = adev->pdev->subsystem_device;
3462 config[no_regs++] = adev->pdev->subsystem_vendor;
3464 while (size && (*pos < no_regs * 4)) {
3467 value = config[*pos >> 2];
3468 r = put_user(value, (uint32_t *)buf);
3484 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3485 size_t size, loff_t *pos)
3487 struct amdgpu_device *adev = file_inode(f)->i_private;
3488 int idx, x, outsize, r, valuesize;
3489 uint32_t values[16];
3491 if (size & 3 || *pos & 0x3)
3494 if (amdgpu_dpm == 0)
3497 /* convert offset to sensor number */
3500 valuesize = sizeof(values);
3501 if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
3502 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
3506 if (size > valuesize)
3513 r = put_user(values[x++], (int32_t *)buf);
3520 return !r ? outsize : r;
3523 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3524 size_t size, loff_t *pos)
3526 struct amdgpu_device *adev = f->f_inode->i_private;
3529 uint32_t offset, se, sh, cu, wave, simd, data[32];
3531 if (size & 3 || *pos & 3)
3535 offset = (*pos & 0x7F);
3536 se = ((*pos >> 7) & 0xFF);
3537 sh = ((*pos >> 15) & 0xFF);
3538 cu = ((*pos >> 23) & 0xFF);
3539 wave = ((*pos >> 31) & 0xFF);
3540 simd = ((*pos >> 37) & 0xFF);
3542 /* switch to the specific se/sh/cu */
3543 mutex_lock(&adev->grbm_idx_mutex);
3544 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3547 if (adev->gfx.funcs->read_wave_data)
3548 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3550 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3551 mutex_unlock(&adev->grbm_idx_mutex);
3556 while (size && (offset < x * 4)) {
3559 value = data[offset >> 2];
3560 r = put_user(value, (uint32_t *)buf);
3573 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3574 size_t size, loff_t *pos)
3576 struct amdgpu_device *adev = f->f_inode->i_private;
3579 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3581 if (size & 3 || *pos & 3)
3585 offset = (*pos & 0xFFF); /* in dwords */
3586 se = ((*pos >> 12) & 0xFF);
3587 sh = ((*pos >> 20) & 0xFF);
3588 cu = ((*pos >> 28) & 0xFF);
3589 wave = ((*pos >> 36) & 0xFF);
3590 simd = ((*pos >> 44) & 0xFF);
3591 thread = ((*pos >> 52) & 0xFF);
3592 bank = ((*pos >> 60) & 1);
3594 data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3598 /* switch to the specific se/sh/cu */
3599 mutex_lock(&adev->grbm_idx_mutex);
3600 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3603 if (adev->gfx.funcs->read_wave_vgprs)
3604 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3606 if (adev->gfx.funcs->read_wave_sgprs)
3607 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3610 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3611 mutex_unlock(&adev->grbm_idx_mutex);
3616 value = data[offset++];
3617 r = put_user(value, (uint32_t *)buf);
3633 static const struct file_operations amdgpu_debugfs_regs_fops = {
3634 .owner = THIS_MODULE,
3635 .read = amdgpu_debugfs_regs_read,
3636 .write = amdgpu_debugfs_regs_write,
3637 .llseek = default_llseek
3639 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3640 .owner = THIS_MODULE,
3641 .read = amdgpu_debugfs_regs_didt_read,
3642 .write = amdgpu_debugfs_regs_didt_write,
3643 .llseek = default_llseek
3645 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3646 .owner = THIS_MODULE,
3647 .read = amdgpu_debugfs_regs_pcie_read,
3648 .write = amdgpu_debugfs_regs_pcie_write,
3649 .llseek = default_llseek
3651 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3652 .owner = THIS_MODULE,
3653 .read = amdgpu_debugfs_regs_smc_read,
3654 .write = amdgpu_debugfs_regs_smc_write,
3655 .llseek = default_llseek
3658 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3659 .owner = THIS_MODULE,
3660 .read = amdgpu_debugfs_gca_config_read,
3661 .llseek = default_llseek
3664 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3665 .owner = THIS_MODULE,
3666 .read = amdgpu_debugfs_sensor_read,
3667 .llseek = default_llseek
3670 static const struct file_operations amdgpu_debugfs_wave_fops = {
3671 .owner = THIS_MODULE,
3672 .read = amdgpu_debugfs_wave_read,
3673 .llseek = default_llseek
3675 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3676 .owner = THIS_MODULE,
3677 .read = amdgpu_debugfs_gpr_read,
3678 .llseek = default_llseek
3681 static const struct file_operations *debugfs_regs[] = {
3682 &amdgpu_debugfs_regs_fops,
3683 &amdgpu_debugfs_regs_didt_fops,
3684 &amdgpu_debugfs_regs_pcie_fops,
3685 &amdgpu_debugfs_regs_smc_fops,
3686 &amdgpu_debugfs_gca_config_fops,
3687 &amdgpu_debugfs_sensors_fops,
3688 &amdgpu_debugfs_wave_fops,
3689 &amdgpu_debugfs_gpr_fops,
3692 static const char *debugfs_regs_names[] = {
3697 "amdgpu_gca_config",
3703 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3705 struct drm_minor *minor = adev->ddev->primary;
3706 struct dentry *ent, *root = minor->debugfs_root;
3709 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3710 ent = debugfs_create_file(debugfs_regs_names[i],
3711 S_IFREG | S_IRUGO, root,
3712 adev, debugfs_regs[i]);
3714 for (j = 0; j < i; j++) {
3715 debugfs_remove(adev->debugfs_regs[i]);
3716 adev->debugfs_regs[i] = NULL;
3718 return PTR_ERR(ent);
3722 i_size_write(ent->d_inode, adev->rmmio_size);
3723 adev->debugfs_regs[i] = ent;
3729 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3733 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3734 if (adev->debugfs_regs[i]) {
3735 debugfs_remove(adev->debugfs_regs[i]);
3736 adev->debugfs_regs[i] = NULL;
3741 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
3743 struct drm_info_node *node = (struct drm_info_node *) m->private;
3744 struct drm_device *dev = node->minor->dev;
3745 struct amdgpu_device *adev = dev->dev_private;
3748 /* hold on the scheduler */
3749 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3750 struct amdgpu_ring *ring = adev->rings[i];
3752 if (!ring || !ring->sched.thread)
3754 kthread_park(ring->sched.thread);
3757 seq_printf(m, "run ib test:\n");
3758 r = amdgpu_ib_ring_tests(adev);
3760 seq_printf(m, "ib ring tests failed (%d).\n", r);
3762 seq_printf(m, "ib ring tests passed.\n");
3764 /* go on the scheduler */
3765 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
3766 struct amdgpu_ring *ring = adev->rings[i];
3768 if (!ring || !ring->sched.thread)
3770 kthread_unpark(ring->sched.thread);
3776 static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
3777 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
3780 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3782 return amdgpu_debugfs_add_files(adev,
3783 amdgpu_debugfs_test_ib_ring_list, 1);
3786 int amdgpu_debugfs_init(struct drm_minor *minor)
3791 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
3793 struct drm_info_node *node = (struct drm_info_node *) m->private;
3794 struct drm_device *dev = node->minor->dev;
3795 struct amdgpu_device *adev = dev->dev_private;
3797 seq_write(m, adev->bios, adev->bios_size);
3801 static const struct drm_info_list amdgpu_vbios_dump_list[] = {
3803 amdgpu_debugfs_get_vbios_dump,
3807 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
3809 return amdgpu_debugfs_add_files(adev,
3810 amdgpu_vbios_dump_list, 1);
3813 static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
3817 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3821 static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
3825 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }