2 * Copyright 2022 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/delay.h>
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
30 #include "amdgpu_xcp.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_trace.h"
34 #include "sdma/sdma_4_4_2_offset.h"
35 #include "sdma/sdma_4_4_2_sh_mask.h"
37 #include "soc15_common.h"
39 #include "vega10_sdma_pkt_open.h"
41 #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
42 #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
44 #include "amdgpu_ras.h"
46 MODULE_FIRMWARE("amdgpu/sdma_4_4_2.bin");
48 #define WREG32_SDMA(instance, offset, value) \
49 WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value)
50 #define RREG32_SDMA(instance, offset) \
51 RREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)))
53 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev);
54 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev);
55 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev);
56 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev);
57 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev);
59 static u32 sdma_v4_4_2_get_reg_offset(struct amdgpu_device *adev,
60 u32 instance, u32 offset)
62 u32 dev_inst = GET_INST(SDMA0, instance);
64 return (adev->reg_offset[SDMA0_HWIP][dev_inst][0] + offset);
67 static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
71 return SOC15_IH_CLIENTID_SDMA0;
73 return SOC15_IH_CLIENTID_SDMA1;
75 return SOC15_IH_CLIENTID_SDMA2;
77 return SOC15_IH_CLIENTID_SDMA3;
83 static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
86 case SOC15_IH_CLIENTID_SDMA0:
88 case SOC15_IH_CLIENTID_SDMA1:
90 case SOC15_IH_CLIENTID_SDMA2:
92 case SOC15_IH_CLIENTID_SDMA3:
99 static void sdma_v4_4_2_inst_init_golden_registers(struct amdgpu_device *adev,
105 for (i = 0; i < adev->sdma.num_instances; i++) {
106 val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG);
107 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG, NUM_BANKS, 4);
108 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG,
109 PIPE_INTERLEAVE_SIZE, 0);
110 WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG, val);
112 val = RREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ);
113 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ, NUM_BANKS,
115 val = REG_SET_FIELD(val, SDMA_GB_ADDR_CONFIG_READ,
116 PIPE_INTERLEAVE_SIZE, 0);
117 WREG32_SDMA(i, regSDMA_GB_ADDR_CONFIG_READ, val);
122 * sdma_v4_4_2_init_microcode - load ucode images from disk
124 * @adev: amdgpu_device pointer
126 * Use the firmware interface to load the ucode images into
127 * the driver (not loaded into hw).
128 * Returns 0 on success, error on failure.
130 static int sdma_v4_4_2_init_microcode(struct amdgpu_device *adev)
134 for (i = 0; i < adev->sdma.num_instances; i++) {
135 if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2)) {
136 ret = amdgpu_sdma_init_microcode(adev, 0, true);
139 ret = amdgpu_sdma_init_microcode(adev, i, false);
149 * sdma_v4_4_2_ring_get_rptr - get the current read pointer
151 * @ring: amdgpu ring pointer
153 * Get the current rptr from the hardware.
155 static uint64_t sdma_v4_4_2_ring_get_rptr(struct amdgpu_ring *ring)
159 /* XXX check if swapping is necessary on BE */
160 rptr = ((u64 *)&ring->adev->wb.wb[ring->rptr_offs]);
162 DRM_DEBUG("rptr before shift == 0x%016llx\n", *rptr);
163 return ((*rptr) >> 2);
167 * sdma_v4_4_2_ring_get_wptr - get the current write pointer
169 * @ring: amdgpu ring pointer
171 * Get the current wptr from the hardware.
173 static uint64_t sdma_v4_4_2_ring_get_wptr(struct amdgpu_ring *ring)
175 struct amdgpu_device *adev = ring->adev;
178 if (ring->use_doorbell) {
179 /* XXX check if swapping is necessary on BE */
180 wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
181 DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
183 wptr = RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI);
185 wptr |= RREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR);
186 DRM_DEBUG("wptr before shift [%i] wptr == 0x%016llx\n",
194 * sdma_v4_4_2_ring_set_wptr - commit the write pointer
196 * @ring: amdgpu ring pointer
198 * Write the wptr back to the hardware.
200 static void sdma_v4_4_2_ring_set_wptr(struct amdgpu_ring *ring)
202 struct amdgpu_device *adev = ring->adev;
204 DRM_DEBUG("Setting write pointer\n");
205 if (ring->use_doorbell) {
206 u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
208 DRM_DEBUG("Using doorbell -- "
209 "wptr_offs == 0x%08x "
210 "lower_32_bits(ring->wptr) << 2 == 0x%08x "
211 "upper_32_bits(ring->wptr) << 2 == 0x%08x\n",
213 lower_32_bits(ring->wptr << 2),
214 upper_32_bits(ring->wptr << 2));
215 /* XXX check if swapping is necessary on BE */
216 WRITE_ONCE(*wb, (ring->wptr << 2));
217 DRM_DEBUG("calling WDOORBELL64(0x%08x, 0x%016llx)\n",
218 ring->doorbell_index, ring->wptr << 2);
219 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
221 DRM_DEBUG("Not using doorbell -- "
222 "regSDMA%i_GFX_RB_WPTR == 0x%08x "
223 "regSDMA%i_GFX_RB_WPTR_HI == 0x%08x\n",
225 lower_32_bits(ring->wptr << 2),
227 upper_32_bits(ring->wptr << 2));
228 WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR,
229 lower_32_bits(ring->wptr << 2));
230 WREG32_SDMA(ring->me, regSDMA_GFX_RB_WPTR_HI,
231 upper_32_bits(ring->wptr << 2));
236 * sdma_v4_4_2_page_ring_get_wptr - get the current write pointer
238 * @ring: amdgpu ring pointer
240 * Get the current wptr from the hardware.
242 static uint64_t sdma_v4_4_2_page_ring_get_wptr(struct amdgpu_ring *ring)
244 struct amdgpu_device *adev = ring->adev;
247 if (ring->use_doorbell) {
248 /* XXX check if swapping is necessary on BE */
249 wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
251 wptr = RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI);
253 wptr |= RREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR);
260 * sdma_v4_4_2_page_ring_set_wptr - commit the write pointer
262 * @ring: amdgpu ring pointer
264 * Write the wptr back to the hardware.
266 static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
268 struct amdgpu_device *adev = ring->adev;
270 if (ring->use_doorbell) {
271 u64 *wb = (u64 *)&adev->wb.wb[ring->wptr_offs];
273 /* XXX check if swapping is necessary on BE */
274 WRITE_ONCE(*wb, (ring->wptr << 2));
275 WDOORBELL64(ring->doorbell_index, ring->wptr << 2);
277 uint64_t wptr = ring->wptr << 2;
279 WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR,
280 lower_32_bits(wptr));
281 WREG32_SDMA(ring->me, regSDMA_PAGE_RB_WPTR_HI,
282 upper_32_bits(wptr));
286 static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
288 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
291 for (i = 0; i < count; i++)
292 if (sdma && sdma->burst_nop && (i == 0))
293 amdgpu_ring_write(ring, ring->funcs->nop |
294 SDMA_PKT_NOP_HEADER_COUNT(count - 1));
296 amdgpu_ring_write(ring, ring->funcs->nop);
300 * sdma_v4_4_2_ring_emit_ib - Schedule an IB on the DMA engine
302 * @ring: amdgpu ring pointer
303 * @job: job to retrieve vmid from
304 * @ib: IB object to schedule
307 * Schedule an IB in the DMA ring.
309 static void sdma_v4_4_2_ring_emit_ib(struct amdgpu_ring *ring,
310 struct amdgpu_job *job,
311 struct amdgpu_ib *ib,
314 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
316 /* IB packet must end on a 8 DW boundary */
317 sdma_v4_4_2_ring_insert_nop(ring, (2 - lower_32_bits(ring->wptr)) & 7);
319 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
320 SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
321 /* base must be 32 byte aligned */
322 amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
323 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
324 amdgpu_ring_write(ring, ib->length_dw);
325 amdgpu_ring_write(ring, 0);
326 amdgpu_ring_write(ring, 0);
330 static void sdma_v4_4_2_wait_reg_mem(struct amdgpu_ring *ring,
331 int mem_space, int hdp,
332 uint32_t addr0, uint32_t addr1,
333 uint32_t ref, uint32_t mask,
336 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
337 SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(hdp) |
338 SDMA_PKT_POLL_REGMEM_HEADER_MEM_POLL(mem_space) |
339 SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
342 amdgpu_ring_write(ring, addr0);
343 amdgpu_ring_write(ring, addr1);
346 amdgpu_ring_write(ring, addr0 << 2);
347 amdgpu_ring_write(ring, addr1 << 2);
349 amdgpu_ring_write(ring, ref); /* reference */
350 amdgpu_ring_write(ring, mask); /* mask */
351 amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
352 SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(inv)); /* retry count, poll interval */
356 * sdma_v4_4_2_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
358 * @ring: amdgpu ring pointer
360 * Emit an hdp flush packet on the requested DMA ring.
362 static void sdma_v4_4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
364 struct amdgpu_device *adev = ring->adev;
365 u32 ref_and_mask = 0;
366 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
368 ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0 << ring->me;
370 sdma_v4_4_2_wait_reg_mem(ring, 0, 1,
371 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
372 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
373 ref_and_mask, ref_and_mask, 10);
377 * sdma_v4_4_2_ring_emit_fence - emit a fence on the DMA ring
379 * @ring: amdgpu ring pointer
381 * @seq: sequence number
382 * @flags: fence related flags
384 * Add a DMA fence packet to the ring to write
385 * the fence seq number and DMA trap packet to generate
386 * an interrupt if needed.
388 static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
391 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
392 /* write the fence */
393 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
394 /* zero in first two bits */
396 amdgpu_ring_write(ring, lower_32_bits(addr));
397 amdgpu_ring_write(ring, upper_32_bits(addr));
398 amdgpu_ring_write(ring, lower_32_bits(seq));
400 /* optionally write high bits as well */
403 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE));
404 /* zero in first two bits */
406 amdgpu_ring_write(ring, lower_32_bits(addr));
407 amdgpu_ring_write(ring, upper_32_bits(addr));
408 amdgpu_ring_write(ring, upper_32_bits(seq));
411 /* generate an interrupt */
412 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_TRAP));
413 amdgpu_ring_write(ring, SDMA_PKT_TRAP_INT_CONTEXT_INT_CONTEXT(0));
418 * sdma_v4_4_2_inst_gfx_stop - stop the gfx async dma engines
420 * @adev: amdgpu_device pointer
421 * @inst_mask: mask of dma engine instances to be disabled
423 * Stop the gfx async dma ring buffers.
425 static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
428 struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
429 u32 rb_cntl, ib_cntl;
432 for_each_inst(i, inst_mask) {
433 sdma[i] = &adev->sdma.instance[i].ring;
435 if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) {
436 amdgpu_ttm_set_buffer_funcs_status(adev, false);
440 rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
441 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 0);
442 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
443 ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
444 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 0);
445 WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
450 * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
452 * @adev: amdgpu_device pointer
453 * @inst_mask: mask of dma engine instances to be disabled
455 * Stop the compute async dma queues.
457 static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
464 * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
466 * @adev: amdgpu_device pointer
467 * @inst_mask: mask of dma engine instances to be disabled
469 * Stop the page async dma ring buffers.
471 static void sdma_v4_4_2_inst_page_stop(struct amdgpu_device *adev,
474 struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES];
475 u32 rb_cntl, ib_cntl;
479 for_each_inst(i, inst_mask) {
480 sdma[i] = &adev->sdma.instance[i].page;
482 if ((adev->mman.buffer_funcs_ring == sdma[i]) &&
484 amdgpu_ttm_set_buffer_funcs_status(adev, false);
488 rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
489 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
491 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
492 ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
493 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL,
495 WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
500 * sdma_v4_4_2_inst_ctx_switch_enable - stop the async dma engines context switch
502 * @adev: amdgpu_device pointer
503 * @enable: enable/disable the DMA MEs context switch.
504 * @inst_mask: mask of dma engine instances to be enabled
506 * Halt or unhalt the async dma engines context switch.
508 static void sdma_v4_4_2_inst_ctx_switch_enable(struct amdgpu_device *adev,
509 bool enable, uint32_t inst_mask)
511 u32 f32_cntl, phase_quantum = 0;
514 if (amdgpu_sdma_phase_quantum) {
515 unsigned value = amdgpu_sdma_phase_quantum;
518 while (value > (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
519 SDMA_PHASE0_QUANTUM__VALUE__SHIFT)) {
520 value = (value + 1) >> 1;
523 if (unit > (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
524 SDMA_PHASE0_QUANTUM__UNIT__SHIFT)) {
525 value = (SDMA_PHASE0_QUANTUM__VALUE_MASK >>
526 SDMA_PHASE0_QUANTUM__VALUE__SHIFT);
527 unit = (SDMA_PHASE0_QUANTUM__UNIT_MASK >>
528 SDMA_PHASE0_QUANTUM__UNIT__SHIFT);
530 "clamping sdma_phase_quantum to %uK clock cycles\n",
534 value << SDMA_PHASE0_QUANTUM__VALUE__SHIFT |
535 unit << SDMA_PHASE0_QUANTUM__UNIT__SHIFT;
538 for_each_inst(i, inst_mask) {
539 f32_cntl = RREG32_SDMA(i, regSDMA_CNTL);
540 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_CNTL,
541 AUTO_CTXSW_ENABLE, enable ? 1 : 0);
542 if (enable && amdgpu_sdma_phase_quantum) {
543 WREG32_SDMA(i, regSDMA_PHASE0_QUANTUM, phase_quantum);
544 WREG32_SDMA(i, regSDMA_PHASE1_QUANTUM, phase_quantum);
545 WREG32_SDMA(i, regSDMA_PHASE2_QUANTUM, phase_quantum);
547 WREG32_SDMA(i, regSDMA_CNTL, f32_cntl);
549 /* Extend page fault timeout to avoid interrupt storm */
550 WREG32_SDMA(i, regSDMA_UTCL1_TIMEOUT, 0x00800080);
555 * sdma_v4_4_2_inst_enable - stop the async dma engines
557 * @adev: amdgpu_device pointer
558 * @enable: enable/disable the DMA MEs.
559 * @inst_mask: mask of dma engine instances to be enabled
561 * Halt or unhalt the async dma engines.
563 static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
570 sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
571 sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
572 if (adev->sdma.has_page_queue)
573 sdma_v4_4_2_inst_page_stop(adev, inst_mask);
575 /* SDMA FW needs to respond to FREEZE requests during reset.
576 * Keep it running during reset */
577 if (!amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
581 for_each_inst(i, inst_mask) {
582 f32_cntl = RREG32_SDMA(i, regSDMA_F32_CNTL);
583 f32_cntl = REG_SET_FIELD(f32_cntl, SDMA_F32_CNTL, HALT, enable ? 0 : 1);
584 WREG32_SDMA(i, regSDMA_F32_CNTL, f32_cntl);
589 * sdma_v4_4_2_rb_cntl - get parameters for rb_cntl
591 static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
593 /* Set ring buffer size in dwords */
594 uint32_t rb_bufsz = order_base_2(ring->ring_size / 4);
596 barrier(); /* work around https://bugs.llvm.org/show_bug.cgi?id=42576 */
597 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
599 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
600 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
601 RPTR_WRITEBACK_SWAP_ENABLE, 1);
607 * sdma_v4_4_2_gfx_resume - setup and start the async dma engines
609 * @adev: amdgpu_device pointer
610 * @i: instance to resume
612 * Set up the gfx DMA ring buffers and enable them.
613 * Returns 0 for success, error for failure.
615 static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i)
617 struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
618 u32 rb_cntl, ib_cntl, wptr_poll_cntl;
624 wb_offset = (ring->rptr_offs * 4);
626 rb_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_CNTL);
627 rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
628 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
630 /* Initialize the ring buffer's read and write pointers */
631 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
632 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
633 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
634 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
636 /* set the wb address whether it's enabled or not */
637 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_HI,
638 upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
639 WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_ADDR_LO,
640 lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
642 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL,
643 RPTR_WRITEBACK_ENABLE, 1);
645 WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
646 WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
650 /* before programing wptr to a less value, need set minor_ptr_update first */
651 WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
653 doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
654 doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
656 doorbell = REG_SET_FIELD(doorbell, SDMA_GFX_DOORBELL, ENABLE,
658 doorbell_offset = REG_SET_FIELD(doorbell_offset,
659 SDMA_GFX_DOORBELL_OFFSET,
660 OFFSET, ring->doorbell_index);
661 WREG32_SDMA(i, regSDMA_GFX_DOORBELL, doorbell);
662 WREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET, doorbell_offset);
664 sdma_v4_4_2_ring_set_wptr(ring);
666 /* set minor_ptr_update to 0 after wptr programed */
667 WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 0);
669 /* setup the wptr shadow polling */
670 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
671 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_LO,
672 lower_32_bits(wptr_gpu_addr));
673 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_ADDR_HI,
674 upper_32_bits(wptr_gpu_addr));
675 wptr_poll_cntl = RREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL);
676 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
677 SDMA_GFX_RB_WPTR_POLL_CNTL,
678 F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
679 WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
682 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_GFX_RB_CNTL, RB_ENABLE, 1);
683 WREG32_SDMA(i, regSDMA_GFX_RB_CNTL, rb_cntl);
685 ib_cntl = RREG32_SDMA(i, regSDMA_GFX_IB_CNTL);
686 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_ENABLE, 1);
688 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
691 WREG32_SDMA(i, regSDMA_GFX_IB_CNTL, ib_cntl);
695 * sdma_v4_4_2_page_resume - setup and start the async dma engines
697 * @adev: amdgpu_device pointer
698 * @i: instance to resume
700 * Set up the page DMA ring buffers and enable them.
701 * Returns 0 for success, error for failure.
703 static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i)
705 struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
706 u32 rb_cntl, ib_cntl, wptr_poll_cntl;
712 wb_offset = (ring->rptr_offs * 4);
714 rb_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_CNTL);
715 rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
716 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
718 /* Initialize the ring buffer's read and write pointers */
719 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
720 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
721 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
722 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
724 /* set the wb address whether it's enabled or not */
725 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
726 upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
727 WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_LO,
728 lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
730 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL,
731 RPTR_WRITEBACK_ENABLE, 1);
733 WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
734 WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
738 /* before programing wptr to a less value, need set minor_ptr_update first */
739 WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
741 doorbell = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL);
742 doorbell_offset = RREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET);
744 doorbell = REG_SET_FIELD(doorbell, SDMA_PAGE_DOORBELL, ENABLE,
746 doorbell_offset = REG_SET_FIELD(doorbell_offset,
747 SDMA_PAGE_DOORBELL_OFFSET,
748 OFFSET, ring->doorbell_index);
749 WREG32_SDMA(i, regSDMA_PAGE_DOORBELL, doorbell);
750 WREG32_SDMA(i, regSDMA_PAGE_DOORBELL_OFFSET, doorbell_offset);
752 /* paging queue doorbell range is setup at sdma_v4_4_2_gfx_resume */
753 sdma_v4_4_2_page_ring_set_wptr(ring);
755 /* set minor_ptr_update to 0 after wptr programed */
756 WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 0);
758 /* setup the wptr shadow polling */
759 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
760 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_LO,
761 lower_32_bits(wptr_gpu_addr));
762 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_ADDR_HI,
763 upper_32_bits(wptr_gpu_addr));
764 wptr_poll_cntl = RREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL);
765 wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
766 SDMA_PAGE_RB_WPTR_POLL_CNTL,
767 F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
768 WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
771 rb_cntl = REG_SET_FIELD(rb_cntl, SDMA_PAGE_RB_CNTL, RB_ENABLE, 1);
772 WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
774 ib_cntl = RREG32_SDMA(i, regSDMA_PAGE_IB_CNTL);
775 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_ENABLE, 1);
777 ib_cntl = REG_SET_FIELD(ib_cntl, SDMA_PAGE_IB_CNTL, IB_SWAP_ENABLE, 1);
780 WREG32_SDMA(i, regSDMA_PAGE_IB_CNTL, ib_cntl);
783 static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
789 * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
791 * @adev: amdgpu_device pointer
792 * @inst_mask: mask of dma engine instances to be enabled
794 * Set up the compute DMA queues and enable them.
795 * Returns 0 for success, error for failure.
797 static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
800 sdma_v4_4_2_init_pg(adev);
806 * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
808 * @adev: amdgpu_device pointer
809 * @inst_mask: mask of dma engine instances to be enabled
811 * Loads the sDMA0/1 ucode.
812 * Returns 0 for success, -EINVAL if the ucode is not available.
814 static int sdma_v4_4_2_inst_load_microcode(struct amdgpu_device *adev,
817 const struct sdma_firmware_header_v1_0 *hdr;
818 const __le32 *fw_data;
823 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
825 for_each_inst(i, inst_mask) {
826 if (!adev->sdma.instance[i].fw)
829 hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
830 amdgpu_ucode_print_sdma_hdr(&hdr->header);
831 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
833 fw_data = (const __le32 *)
834 (adev->sdma.instance[i].fw->data +
835 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
837 WREG32_SDMA(i, regSDMA_UCODE_ADDR, 0);
839 for (j = 0; j < fw_size; j++)
840 WREG32_SDMA(i, regSDMA_UCODE_DATA,
841 le32_to_cpup(fw_data++));
843 WREG32_SDMA(i, regSDMA_UCODE_ADDR,
844 adev->sdma.instance[i].fw_version);
851 * sdma_v4_4_2_inst_start - setup and start the async dma engines
853 * @adev: amdgpu_device pointer
854 * @inst_mask: mask of dma engine instances to be enabled
856 * Set up the DMA engines and enable them.
857 * Returns 0 for success, error for failure.
859 static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
862 struct amdgpu_ring *ring;
866 if (amdgpu_sriov_vf(adev)) {
867 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
868 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
870 /* bypass sdma microcode loading on Gopher */
871 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP &&
872 adev->sdma.instance[0].fw) {
873 r = sdma_v4_4_2_inst_load_microcode(adev, inst_mask);
879 sdma_v4_4_2_inst_enable(adev, true, inst_mask);
880 /* enable sdma ring preemption */
881 sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
884 /* start the gfx rings and rlc compute queues */
885 tmp_mask = inst_mask;
886 for_each_inst(i, tmp_mask) {
889 WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
890 sdma_v4_4_2_gfx_resume(adev, i);
891 if (adev->sdma.has_page_queue)
892 sdma_v4_4_2_page_resume(adev, i);
894 /* set utc l1 enable flag always to 1 */
895 temp = RREG32_SDMA(i, regSDMA_CNTL);
896 temp = REG_SET_FIELD(temp, SDMA_CNTL, UTC_L1_ENABLE, 1);
897 /* enable context empty interrupt during initialization */
898 temp = REG_SET_FIELD(temp, SDMA_CNTL, CTXEMPTY_INT_ENABLE, 1);
899 WREG32_SDMA(i, regSDMA_CNTL, temp);
901 if (!amdgpu_sriov_vf(adev)) {
902 ring = &adev->sdma.instance[i].ring;
903 adev->nbio.funcs->sdma_doorbell_range(adev, i,
904 ring->use_doorbell, ring->doorbell_index,
905 adev->doorbell_index.sdma_doorbell_range);
908 temp = RREG32_SDMA(i, regSDMA_F32_CNTL);
909 temp = REG_SET_FIELD(temp, SDMA_F32_CNTL, HALT, 0);
910 WREG32_SDMA(i, regSDMA_F32_CNTL, temp);
914 if (amdgpu_sriov_vf(adev)) {
915 sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
916 sdma_v4_4_2_inst_enable(adev, true, inst_mask);
918 r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
923 tmp_mask = inst_mask;
924 for_each_inst(i, tmp_mask) {
925 ring = &adev->sdma.instance[i].ring;
927 r = amdgpu_ring_test_helper(ring);
931 if (adev->sdma.has_page_queue) {
932 struct amdgpu_ring *page = &adev->sdma.instance[i].page;
934 r = amdgpu_ring_test_helper(page);
938 if (adev->mman.buffer_funcs_ring == page)
939 amdgpu_ttm_set_buffer_funcs_status(adev, true);
942 if (adev->mman.buffer_funcs_ring == ring)
943 amdgpu_ttm_set_buffer_funcs_status(adev, true);
950 * sdma_v4_4_2_ring_test_ring - simple async dma engine test
952 * @ring: amdgpu_ring structure holding ring information
954 * Test the DMA engine by writing using it to write an
956 * Returns 0 for success, error for failure.
958 static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
960 struct amdgpu_device *adev = ring->adev;
967 r = amdgpu_device_wb_get(adev, &index);
971 gpu_addr = adev->wb.gpu_addr + (index * 4);
973 adev->wb.wb[index] = cpu_to_le32(tmp);
975 r = amdgpu_ring_alloc(ring, 5);
979 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
980 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
981 amdgpu_ring_write(ring, lower_32_bits(gpu_addr));
982 amdgpu_ring_write(ring, upper_32_bits(gpu_addr));
983 amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0));
984 amdgpu_ring_write(ring, 0xDEADBEEF);
985 amdgpu_ring_commit(ring);
987 for (i = 0; i < adev->usec_timeout; i++) {
988 tmp = le32_to_cpu(adev->wb.wb[index]);
989 if (tmp == 0xDEADBEEF)
994 if (i >= adev->usec_timeout)
998 amdgpu_device_wb_free(adev, index);
1003 * sdma_v4_4_2_ring_test_ib - test an IB on the DMA engine
1005 * @ring: amdgpu_ring structure holding ring information
1006 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
1008 * Test a simple IB in the DMA ring.
1009 * Returns 0 on success, error on failure.
1011 static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1013 struct amdgpu_device *adev = ring->adev;
1014 struct amdgpu_ib ib;
1015 struct dma_fence *f = NULL;
1021 r = amdgpu_device_wb_get(adev, &index);
1025 gpu_addr = adev->wb.gpu_addr + (index * 4);
1027 adev->wb.wb[index] = cpu_to_le32(tmp);
1028 memset(&ib, 0, sizeof(ib));
1029 r = amdgpu_ib_get(adev, NULL, 256,
1030 AMDGPU_IB_POOL_DIRECT, &ib);
1034 ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1035 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1036 ib.ptr[1] = lower_32_bits(gpu_addr);
1037 ib.ptr[2] = upper_32_bits(gpu_addr);
1038 ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
1039 ib.ptr[4] = 0xDEADBEEF;
1040 ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1041 ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1042 ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
1045 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1049 r = dma_fence_wait_timeout(f, false, timeout);
1056 tmp = le32_to_cpu(adev->wb.wb[index]);
1057 if (tmp == 0xDEADBEEF)
1063 amdgpu_ib_free(adev, &ib, NULL);
1066 amdgpu_device_wb_free(adev, index);
1072 * sdma_v4_4_2_vm_copy_pte - update PTEs by copying them from the GART
1074 * @ib: indirect buffer to fill with commands
1075 * @pe: addr of the page entry
1076 * @src: src addr to copy from
1077 * @count: number of page entries to update
1079 * Update PTEs by copying them from the GART using sDMA.
1081 static void sdma_v4_4_2_vm_copy_pte(struct amdgpu_ib *ib,
1082 uint64_t pe, uint64_t src,
1085 unsigned bytes = count * 8;
1087 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1088 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR);
1089 ib->ptr[ib->length_dw++] = bytes - 1;
1090 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1091 ib->ptr[ib->length_dw++] = lower_32_bits(src);
1092 ib->ptr[ib->length_dw++] = upper_32_bits(src);
1093 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1094 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1099 * sdma_v4_4_2_vm_write_pte - update PTEs by writing them manually
1101 * @ib: indirect buffer to fill with commands
1102 * @pe: addr of the page entry
1103 * @value: dst addr to write into pe
1104 * @count: number of page entries to update
1105 * @incr: increase next addr by incr bytes
1107 * Update PTEs by writing them manually using sDMA.
1109 static void sdma_v4_4_2_vm_write_pte(struct amdgpu_ib *ib, uint64_t pe,
1110 uint64_t value, unsigned count,
1113 unsigned ndw = count * 2;
1115 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
1116 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
1117 ib->ptr[ib->length_dw++] = lower_32_bits(pe);
1118 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1119 ib->ptr[ib->length_dw++] = ndw - 1;
1120 for (; ndw > 0; ndw -= 2) {
1121 ib->ptr[ib->length_dw++] = lower_32_bits(value);
1122 ib->ptr[ib->length_dw++] = upper_32_bits(value);
1128 * sdma_v4_4_2_vm_set_pte_pde - update the page tables using sDMA
1130 * @ib: indirect buffer to fill with commands
1131 * @pe: addr of the page entry
1132 * @addr: dst addr to write into pe
1133 * @count: number of page entries to update
1134 * @incr: increase next addr by incr bytes
1135 * @flags: access flags
1137 * Update the page tables using sDMA.
1139 static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
1141 uint64_t addr, unsigned count,
1142 uint32_t incr, uint64_t flags)
1144 /* for physically contiguous pages (vram) */
1145 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_PTEPDE);
1146 ib->ptr[ib->length_dw++] = lower_32_bits(pe); /* dst addr */
1147 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
1148 ib->ptr[ib->length_dw++] = lower_32_bits(flags); /* mask */
1149 ib->ptr[ib->length_dw++] = upper_32_bits(flags);
1150 ib->ptr[ib->length_dw++] = lower_32_bits(addr); /* value */
1151 ib->ptr[ib->length_dw++] = upper_32_bits(addr);
1152 ib->ptr[ib->length_dw++] = incr; /* increment size */
1153 ib->ptr[ib->length_dw++] = 0;
1154 ib->ptr[ib->length_dw++] = count - 1; /* number of entries */
1158 * sdma_v4_4_2_ring_pad_ib - pad the IB to the required number of dw
1160 * @ring: amdgpu_ring structure holding ring information
1161 * @ib: indirect buffer to fill with padding
1163 static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
1165 struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
1169 pad_count = (-ib->length_dw) & 7;
1170 for (i = 0; i < pad_count; i++)
1171 if (sdma && sdma->burst_nop && (i == 0))
1172 ib->ptr[ib->length_dw++] =
1173 SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
1174 SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
1176 ib->ptr[ib->length_dw++] =
1177 SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
1182 * sdma_v4_4_2_ring_emit_pipeline_sync - sync the pipeline
1184 * @ring: amdgpu_ring pointer
1186 * Make sure all previous operations are completed (CIK).
1188 static void sdma_v4_4_2_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
1190 uint32_t seq = ring->fence_drv.sync_seq;
1191 uint64_t addr = ring->fence_drv.gpu_addr;
1194 sdma_v4_4_2_wait_reg_mem(ring, 1, 0,
1196 upper_32_bits(addr) & 0xffffffff,
1197 seq, 0xffffffff, 4);
1202 * sdma_v4_4_2_ring_emit_vm_flush - vm flush using sDMA
1204 * @ring: amdgpu_ring pointer
1205 * @vmid: vmid number to use
1208 * Update the page table base and flush the VM TLB
1211 static void sdma_v4_4_2_ring_emit_vm_flush(struct amdgpu_ring *ring,
1212 unsigned vmid, uint64_t pd_addr)
1214 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1217 static void sdma_v4_4_2_ring_emit_wreg(struct amdgpu_ring *ring,
1218 uint32_t reg, uint32_t val)
1220 amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
1221 SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
1222 amdgpu_ring_write(ring, reg);
1223 amdgpu_ring_write(ring, val);
1226 static void sdma_v4_4_2_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
1227 uint32_t val, uint32_t mask)
1229 sdma_v4_4_2_wait_reg_mem(ring, 0, 0, reg, 0, val, mask, 10);
1232 static bool sdma_v4_4_2_fw_support_paging_queue(struct amdgpu_device *adev)
1234 switch (adev->ip_versions[SDMA0_HWIP][0]) {
1235 case IP_VERSION(4, 4, 2):
1242 static int sdma_v4_4_2_early_init(void *handle)
1244 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1247 r = sdma_v4_4_2_init_microcode(adev);
1249 DRM_ERROR("Failed to load sdma firmware!\n");
1253 /* TODO: Page queue breaks driver reload under SRIOV */
1254 if (sdma_v4_4_2_fw_support_paging_queue(adev))
1255 adev->sdma.has_page_queue = true;
1257 sdma_v4_4_2_set_ring_funcs(adev);
1258 sdma_v4_4_2_set_buffer_funcs(adev);
1259 sdma_v4_4_2_set_vm_pte_funcs(adev);
1260 sdma_v4_4_2_set_irq_funcs(adev);
1261 sdma_v4_4_2_set_ras_funcs(adev);
1267 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1269 struct amdgpu_iv_entry *entry);
1272 static int sdma_v4_4_2_late_init(void *handle)
1274 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1276 struct ras_ih_if ih_info = {
1277 .cb = sdma_v4_4_2_process_ras_data_cb,
1280 if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1281 if (adev->sdma.ras && adev->sdma.ras->ras_block.hw_ops &&
1282 adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count)
1283 adev->sdma.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1289 static int sdma_v4_4_2_sw_init(void *handle)
1291 struct amdgpu_ring *ring;
1293 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1296 /* SDMA trap event */
1297 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1298 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1299 SDMA0_4_0__SRCID__SDMA_TRAP,
1300 &adev->sdma.trap_irq);
1305 /* SDMA SRAM ECC event */
1306 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1307 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1308 SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
1309 &adev->sdma.ecc_irq);
1314 /* SDMA VM_HOLE/DOORBELL_INV/POLL_TIMEOUT/SRBM_WRITE_PROTECTION event*/
1315 for (i = 0; i < adev->sdma.num_inst_per_aid; i++) {
1316 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1317 SDMA0_4_0__SRCID__SDMA_VM_HOLE,
1318 &adev->sdma.vm_hole_irq);
1322 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1323 SDMA0_4_0__SRCID__SDMA_DOORBELL_INVALID,
1324 &adev->sdma.doorbell_invalid_irq);
1328 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1329 SDMA0_4_0__SRCID__SDMA_POLL_TIMEOUT,
1330 &adev->sdma.pool_timeout_irq);
1334 r = amdgpu_irq_add_id(adev, sdma_v4_4_2_seq_to_irq_id(i),
1335 SDMA0_4_0__SRCID__SDMA_SRBMWRITE,
1336 &adev->sdma.srbm_write_irq);
1341 for (i = 0; i < adev->sdma.num_instances; i++) {
1342 ring = &adev->sdma.instance[i].ring;
1343 ring->ring_obj = NULL;
1344 ring->use_doorbell = true;
1345 aid_id = adev->sdma.instance[i].aid_id;
1347 DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
1348 ring->use_doorbell?"true":"false");
1350 /* doorbell size is 2 dwords, get DWORD offset */
1351 ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
1352 ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1354 sprintf(ring->name, "sdma%d.%d", aid_id,
1355 i % adev->sdma.num_inst_per_aid);
1356 r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
1357 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1358 AMDGPU_RING_PRIO_DEFAULT, NULL);
1362 if (adev->sdma.has_page_queue) {
1363 ring = &adev->sdma.instance[i].page;
1364 ring->ring_obj = NULL;
1365 ring->use_doorbell = true;
1367 /* doorbell index of page queue is assigned right after
1368 * gfx queue on the same instance
1370 ring->doorbell_index =
1371 (adev->doorbell_index.sdma_engine[i] + 1) << 1;
1372 ring->vm_hub = AMDGPU_MMHUB0(aid_id);
1374 sprintf(ring->name, "page%d.%d", aid_id,
1375 i % adev->sdma.num_inst_per_aid);
1376 r = amdgpu_ring_init(adev, ring, 1024,
1377 &adev->sdma.trap_irq,
1378 AMDGPU_SDMA_IRQ_INSTANCE0 + i,
1379 AMDGPU_RING_PRIO_DEFAULT, NULL);
1385 if (amdgpu_sdma_ras_sw_init(adev)) {
1386 dev_err(adev->dev, "fail to initialize sdma ras block\n");
1393 static int sdma_v4_4_2_sw_fini(void *handle)
1395 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1398 for (i = 0; i < adev->sdma.num_instances; i++) {
1399 amdgpu_ring_fini(&adev->sdma.instance[i].ring);
1400 if (adev->sdma.has_page_queue)
1401 amdgpu_ring_fini(&adev->sdma.instance[i].page);
1404 if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 2))
1405 amdgpu_sdma_destroy_inst_ctx(adev, true);
1407 amdgpu_sdma_destroy_inst_ctx(adev, false);
1412 static int sdma_v4_4_2_hw_init(void *handle)
1415 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1418 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1419 if (!amdgpu_sriov_vf(adev))
1420 sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
1422 r = sdma_v4_4_2_inst_start(adev, inst_mask);
1427 static int sdma_v4_4_2_hw_fini(void *handle)
1429 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1433 if (amdgpu_sriov_vf(adev))
1436 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1437 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
1438 for (i = 0; i < adev->sdma.num_instances; i++) {
1439 amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
1440 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
1444 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
1445 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
1450 static int sdma_v4_4_2_set_clockgating_state(void *handle,
1451 enum amd_clockgating_state state);
1453 static int sdma_v4_4_2_suspend(void *handle)
1455 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1457 if (amdgpu_in_reset(adev))
1458 sdma_v4_4_2_set_clockgating_state(adev, AMD_CG_STATE_UNGATE);
1460 return sdma_v4_4_2_hw_fini(adev);
1463 static int sdma_v4_4_2_resume(void *handle)
1465 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1467 return sdma_v4_4_2_hw_init(adev);
1470 static bool sdma_v4_4_2_is_idle(void *handle)
1472 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1475 for (i = 0; i < adev->sdma.num_instances; i++) {
1476 u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
1478 if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
1485 static int sdma_v4_4_2_wait_for_idle(void *handle)
1488 u32 sdma[AMDGPU_MAX_SDMA_INSTANCES];
1489 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1491 for (i = 0; i < adev->usec_timeout; i++) {
1492 for (j = 0; j < adev->sdma.num_instances; j++) {
1493 sdma[j] = RREG32_SDMA(j, regSDMA_STATUS_REG);
1494 if (!(sdma[j] & SDMA_STATUS_REG__IDLE_MASK))
1497 if (j == adev->sdma.num_instances)
1504 static int sdma_v4_4_2_soft_reset(void *handle)
1511 static int sdma_v4_4_2_set_trap_irq_state(struct amdgpu_device *adev,
1512 struct amdgpu_irq_src *source,
1514 enum amdgpu_interrupt_state state)
1518 sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1519 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL, TRAP_ENABLE,
1520 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
1521 WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1526 static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
1527 struct amdgpu_irq_src *source,
1528 struct amdgpu_iv_entry *entry)
1530 uint32_t instance, i;
1532 DRM_DEBUG("IH: SDMA trap\n");
1533 instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1535 /* Client id gives the SDMA instance in AID. To know the exact SDMA
1536 * instance, interrupt entry gives the node id which corresponds to the AID instance.
1537 * Match node id with the AID id associated with the SDMA instance. */
1538 for (i = instance; i < adev->sdma.num_instances;
1539 i += adev->sdma.num_inst_per_aid) {
1540 if (adev->sdma.instance[i].aid_id ==
1541 node_id_to_phys_map[entry->node_id])
1545 if (i >= adev->sdma.num_instances) {
1548 "Couldn't find the right sdma instance in trap handler");
1552 switch (entry->ring_id) {
1554 amdgpu_fence_process(&adev->sdma.instance[i].ring);
1563 static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
1565 struct amdgpu_iv_entry *entry)
1569 /* When “Full RAS” is enabled, the per-IP interrupt sources should
1570 * be disabled and the driver should only look for the aggregated
1571 * interrupt via sync flood
1573 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
1576 instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1580 amdgpu_sdma_process_ras_data_cb(adev, err_data, entry);
1583 return AMDGPU_RAS_SUCCESS;
1587 static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
1588 struct amdgpu_irq_src *source,
1589 struct amdgpu_iv_entry *entry)
1593 DRM_ERROR("Illegal instruction in SDMA command stream\n");
1595 instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1599 switch (entry->ring_id) {
1601 drm_sched_fault(&adev->sdma.instance[instance].ring.sched);
1607 static int sdma_v4_4_2_set_ecc_irq_state(struct amdgpu_device *adev,
1608 struct amdgpu_irq_src *source,
1610 enum amdgpu_interrupt_state state)
1614 sdma_cntl = RREG32_SDMA(type, regSDMA_CNTL);
1616 case AMDGPU_IRQ_STATE_DISABLE:
1617 sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA_CNTL,
1618 DRAM_ECC_INT_ENABLE, 0);
1619 WREG32_SDMA(type, regSDMA_CNTL, sdma_cntl);
1621 /* sdma ecc interrupt is enabled by default
1622 * driver doesn't need to do anything to
1623 * enable the interrupt */
1624 case AMDGPU_IRQ_STATE_ENABLE:
1632 static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
1633 struct amdgpu_iv_entry *entry)
1636 struct amdgpu_task_info task_info;
1639 instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
1640 if (instance < 0 || instance >= adev->sdma.num_instances) {
1641 dev_err(adev->dev, "sdma instance invalid %d\n", instance);
1645 addr = (u64)entry->src_data[0] << 12;
1646 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
1648 memset(&task_info, 0, sizeof(struct amdgpu_task_info));
1649 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
1651 dev_dbg_ratelimited(adev->dev,
1652 "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u "
1653 "pasid:%u, for process %s pid %d thread %s pid %d\n",
1654 instance, addr, entry->src_id, entry->ring_id, entry->vmid,
1655 entry->pasid, task_info.process_name, task_info.tgid,
1656 task_info.task_name, task_info.pid);
1660 static int sdma_v4_4_2_process_vm_hole_irq(struct amdgpu_device *adev,
1661 struct amdgpu_irq_src *source,
1662 struct amdgpu_iv_entry *entry)
1664 dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
1665 sdma_v4_4_2_print_iv_entry(adev, entry);
1669 static int sdma_v4_4_2_process_doorbell_invalid_irq(struct amdgpu_device *adev,
1670 struct amdgpu_irq_src *source,
1671 struct amdgpu_iv_entry *entry)
1674 dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
1675 sdma_v4_4_2_print_iv_entry(adev, entry);
1679 static int sdma_v4_4_2_process_pool_timeout_irq(struct amdgpu_device *adev,
1680 struct amdgpu_irq_src *source,
1681 struct amdgpu_iv_entry *entry)
1683 dev_dbg_ratelimited(adev->dev,
1684 "Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
1685 sdma_v4_4_2_print_iv_entry(adev, entry);
1689 static int sdma_v4_4_2_process_srbm_write_irq(struct amdgpu_device *adev,
1690 struct amdgpu_irq_src *source,
1691 struct amdgpu_iv_entry *entry)
1693 dev_dbg_ratelimited(adev->dev,
1694 "SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
1695 sdma_v4_4_2_print_iv_entry(adev, entry);
1699 static void sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1700 struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1705 /* leave as default if it is not driver controlled */
1706 if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS))
1710 for_each_inst(i, inst_mask) {
1711 /* 1-not override: enable sdma mem light sleep */
1712 def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1713 data |= SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1715 WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1718 for_each_inst(i, inst_mask) {
1719 /* 0-override:disable sdma mem light sleep */
1720 def = data = RREG32_SDMA(i, regSDMA_POWER_CNTL);
1721 data &= ~SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK;
1723 WREG32_SDMA(i, regSDMA_POWER_CNTL, data);
1728 static void sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1729 struct amdgpu_device *adev, bool enable, uint32_t inst_mask)
1734 /* leave as default if it is not driver controlled */
1735 if (!(adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG))
1739 for_each_inst(i, inst_mask) {
1740 def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1741 data &= ~(SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1742 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1743 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1744 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1745 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1746 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1748 WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1751 for_each_inst(i, inst_mask) {
1752 def = data = RREG32_SDMA(i, regSDMA_CLK_CTRL);
1753 data |= (SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK |
1754 SDMA_CLK_CTRL__SOFT_OVERRIDE4_MASK |
1755 SDMA_CLK_CTRL__SOFT_OVERRIDE3_MASK |
1756 SDMA_CLK_CTRL__SOFT_OVERRIDE2_MASK |
1757 SDMA_CLK_CTRL__SOFT_OVERRIDE1_MASK |
1758 SDMA_CLK_CTRL__SOFT_OVERRIDE0_MASK);
1760 WREG32_SDMA(i, regSDMA_CLK_CTRL, data);
1765 static int sdma_v4_4_2_set_clockgating_state(void *handle,
1766 enum amd_clockgating_state state)
1768 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1771 if (amdgpu_sriov_vf(adev))
1774 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
1776 sdma_v4_4_2_inst_update_medium_grain_clock_gating(
1777 adev, state == AMD_CG_STATE_GATE, inst_mask);
1778 sdma_v4_4_2_inst_update_medium_grain_light_sleep(
1779 adev, state == AMD_CG_STATE_GATE, inst_mask);
1783 static int sdma_v4_4_2_set_powergating_state(void *handle,
1784 enum amd_powergating_state state)
1789 static void sdma_v4_4_2_get_clockgating_state(void *handle, u64 *flags)
1791 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1794 if (amdgpu_sriov_vf(adev))
1797 /* AMD_CG_SUPPORT_SDMA_MGCG */
1798 data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_CLK_CTRL));
1799 if (!(data & SDMA_CLK_CTRL__SOFT_OVERRIDE5_MASK))
1800 *flags |= AMD_CG_SUPPORT_SDMA_MGCG;
1802 /* AMD_CG_SUPPORT_SDMA_LS */
1803 data = RREG32(SOC15_REG_OFFSET(SDMA0, GET_INST(SDMA0, 0), regSDMA_POWER_CNTL));
1804 if (data & SDMA_POWER_CNTL__MEM_POWER_OVERRIDE_MASK)
1805 *flags |= AMD_CG_SUPPORT_SDMA_LS;
1808 const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
1809 .name = "sdma_v4_4_2",
1810 .early_init = sdma_v4_4_2_early_init,
1811 .late_init = sdma_v4_4_2_late_init,
1812 .sw_init = sdma_v4_4_2_sw_init,
1813 .sw_fini = sdma_v4_4_2_sw_fini,
1814 .hw_init = sdma_v4_4_2_hw_init,
1815 .hw_fini = sdma_v4_4_2_hw_fini,
1816 .suspend = sdma_v4_4_2_suspend,
1817 .resume = sdma_v4_4_2_resume,
1818 .is_idle = sdma_v4_4_2_is_idle,
1819 .wait_for_idle = sdma_v4_4_2_wait_for_idle,
1820 .soft_reset = sdma_v4_4_2_soft_reset,
1821 .set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
1822 .set_powergating_state = sdma_v4_4_2_set_powergating_state,
1823 .get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
1826 static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
1827 .type = AMDGPU_RING_TYPE_SDMA,
1829 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1830 .support_64bit_ptrs = true,
1831 .get_rptr = sdma_v4_4_2_ring_get_rptr,
1832 .get_wptr = sdma_v4_4_2_ring_get_wptr,
1833 .set_wptr = sdma_v4_4_2_ring_set_wptr,
1835 6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
1836 3 + /* hdp invalidate */
1837 6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
1838 /* sdma_v4_4_2_ring_emit_vm_flush */
1839 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1840 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1841 10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
1842 .emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
1843 .emit_ib = sdma_v4_4_2_ring_emit_ib,
1844 .emit_fence = sdma_v4_4_2_ring_emit_fence,
1845 .emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
1846 .emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
1847 .emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
1848 .test_ring = sdma_v4_4_2_ring_test_ring,
1849 .test_ib = sdma_v4_4_2_ring_test_ib,
1850 .insert_nop = sdma_v4_4_2_ring_insert_nop,
1851 .pad_ib = sdma_v4_4_2_ring_pad_ib,
1852 .emit_wreg = sdma_v4_4_2_ring_emit_wreg,
1853 .emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
1854 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1857 static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
1858 .type = AMDGPU_RING_TYPE_SDMA,
1860 .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP),
1861 .support_64bit_ptrs = true,
1862 .get_rptr = sdma_v4_4_2_ring_get_rptr,
1863 .get_wptr = sdma_v4_4_2_page_ring_get_wptr,
1864 .set_wptr = sdma_v4_4_2_page_ring_set_wptr,
1866 6 + /* sdma_v4_4_2_ring_emit_hdp_flush */
1867 3 + /* hdp invalidate */
1868 6 + /* sdma_v4_4_2_ring_emit_pipeline_sync */
1869 /* sdma_v4_4_2_ring_emit_vm_flush */
1870 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1871 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6 +
1872 10 + 10 + 10, /* sdma_v4_4_2_ring_emit_fence x3 for user fence, vm fence */
1873 .emit_ib_size = 7 + 6, /* sdma_v4_4_2_ring_emit_ib */
1874 .emit_ib = sdma_v4_4_2_ring_emit_ib,
1875 .emit_fence = sdma_v4_4_2_ring_emit_fence,
1876 .emit_pipeline_sync = sdma_v4_4_2_ring_emit_pipeline_sync,
1877 .emit_vm_flush = sdma_v4_4_2_ring_emit_vm_flush,
1878 .emit_hdp_flush = sdma_v4_4_2_ring_emit_hdp_flush,
1879 .test_ring = sdma_v4_4_2_ring_test_ring,
1880 .test_ib = sdma_v4_4_2_ring_test_ib,
1881 .insert_nop = sdma_v4_4_2_ring_insert_nop,
1882 .pad_ib = sdma_v4_4_2_ring_pad_ib,
1883 .emit_wreg = sdma_v4_4_2_ring_emit_wreg,
1884 .emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
1885 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1888 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
1892 for (i = 0; i < adev->sdma.num_instances; i++) {
1893 adev->sdma.instance[i].ring.funcs = &sdma_v4_4_2_ring_funcs;
1894 adev->sdma.instance[i].ring.me = i;
1895 if (adev->sdma.has_page_queue) {
1896 adev->sdma.instance[i].page.funcs =
1897 &sdma_v4_4_2_page_ring_funcs;
1898 adev->sdma.instance[i].page.me = i;
1901 dev_inst = GET_INST(SDMA0, i);
1902 /* AID to which SDMA belongs depends on physical instance */
1903 adev->sdma.instance[i].aid_id =
1904 dev_inst / adev->sdma.num_inst_per_aid;
1908 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_trap_irq_funcs = {
1909 .set = sdma_v4_4_2_set_trap_irq_state,
1910 .process = sdma_v4_4_2_process_trap_irq,
1913 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_illegal_inst_irq_funcs = {
1914 .process = sdma_v4_4_2_process_illegal_inst_irq,
1917 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_ecc_irq_funcs = {
1918 .set = sdma_v4_4_2_set_ecc_irq_state,
1919 .process = amdgpu_sdma_process_ecc_irq,
1922 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_vm_hole_irq_funcs = {
1923 .process = sdma_v4_4_2_process_vm_hole_irq,
1926 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_doorbell_invalid_irq_funcs = {
1927 .process = sdma_v4_4_2_process_doorbell_invalid_irq,
1930 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_pool_timeout_irq_funcs = {
1931 .process = sdma_v4_4_2_process_pool_timeout_irq,
1934 static const struct amdgpu_irq_src_funcs sdma_v4_4_2_srbm_write_irq_funcs = {
1935 .process = sdma_v4_4_2_process_srbm_write_irq,
1938 static void sdma_v4_4_2_set_irq_funcs(struct amdgpu_device *adev)
1940 adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
1941 adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
1942 adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
1943 adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
1944 adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
1945 adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
1947 adev->sdma.trap_irq.funcs = &sdma_v4_4_2_trap_irq_funcs;
1948 adev->sdma.illegal_inst_irq.funcs = &sdma_v4_4_2_illegal_inst_irq_funcs;
1949 adev->sdma.ecc_irq.funcs = &sdma_v4_4_2_ecc_irq_funcs;
1950 adev->sdma.vm_hole_irq.funcs = &sdma_v4_4_2_vm_hole_irq_funcs;
1951 adev->sdma.doorbell_invalid_irq.funcs = &sdma_v4_4_2_doorbell_invalid_irq_funcs;
1952 adev->sdma.pool_timeout_irq.funcs = &sdma_v4_4_2_pool_timeout_irq_funcs;
1953 adev->sdma.srbm_write_irq.funcs = &sdma_v4_4_2_srbm_write_irq_funcs;
1957 * sdma_v4_4_2_emit_copy_buffer - copy buffer using the sDMA engine
1959 * @ib: indirect buffer to copy to
1960 * @src_offset: src GPU address
1961 * @dst_offset: dst GPU address
1962 * @byte_count: number of bytes to xfer
1963 * @tmz: if a secure copy should be used
1965 * Copy GPU buffers using the DMA engine.
1966 * Used by the amdgpu ttm implementation to move pages if
1967 * registered as the asic copy callback.
1969 static void sdma_v4_4_2_emit_copy_buffer(struct amdgpu_ib *ib,
1970 uint64_t src_offset,
1971 uint64_t dst_offset,
1972 uint32_t byte_count,
1975 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) |
1976 SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR) |
1977 SDMA_PKT_COPY_LINEAR_HEADER_TMZ(tmz ? 1 : 0);
1978 ib->ptr[ib->length_dw++] = byte_count - 1;
1979 ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */
1980 ib->ptr[ib->length_dw++] = lower_32_bits(src_offset);
1981 ib->ptr[ib->length_dw++] = upper_32_bits(src_offset);
1982 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
1983 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
1987 * sdma_v4_4_2_emit_fill_buffer - fill buffer using the sDMA engine
1989 * @ib: indirect buffer to copy to
1990 * @src_data: value to write to buffer
1991 * @dst_offset: dst GPU address
1992 * @byte_count: number of bytes to xfer
1994 * Fill GPU buffers using the DMA engine.
1996 static void sdma_v4_4_2_emit_fill_buffer(struct amdgpu_ib *ib,
1998 uint64_t dst_offset,
1999 uint32_t byte_count)
2001 ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL);
2002 ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset);
2003 ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset);
2004 ib->ptr[ib->length_dw++] = src_data;
2005 ib->ptr[ib->length_dw++] = byte_count - 1;
2008 static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
2009 .copy_max_bytes = 0x400000,
2011 .emit_copy_buffer = sdma_v4_4_2_emit_copy_buffer,
2013 .fill_max_bytes = 0x400000,
2015 .emit_fill_buffer = sdma_v4_4_2_emit_fill_buffer,
2018 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
2020 adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
2021 if (adev->sdma.has_page_queue)
2022 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
2024 adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
2027 static const struct amdgpu_vm_pte_funcs sdma_v4_4_2_vm_pte_funcs = {
2028 .copy_pte_num_dw = 7,
2029 .copy_pte = sdma_v4_4_2_vm_copy_pte,
2031 .write_pte = sdma_v4_4_2_vm_write_pte,
2032 .set_pte_pde = sdma_v4_4_2_vm_set_pte_pde,
2035 static void sdma_v4_4_2_set_vm_pte_funcs(struct amdgpu_device *adev)
2037 struct drm_gpu_scheduler *sched;
2040 adev->vm_manager.vm_pte_funcs = &sdma_v4_4_2_vm_pte_funcs;
2041 for (i = 0; i < adev->sdma.num_instances; i++) {
2042 if (adev->sdma.has_page_queue)
2043 sched = &adev->sdma.instance[i].page.sched;
2045 sched = &adev->sdma.instance[i].ring.sched;
2046 adev->vm_manager.vm_pte_scheds[i] = sched;
2048 adev->vm_manager.vm_pte_num_scheds = adev->sdma.num_instances;
2051 const struct amdgpu_ip_block_version sdma_v4_4_2_ip_block = {
2052 .type = AMD_IP_BLOCK_TYPE_SDMA,
2056 .funcs = &sdma_v4_4_2_ip_funcs,
2059 static int sdma_v4_4_2_xcp_resume(void *handle, uint32_t inst_mask)
2061 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2064 if (!amdgpu_sriov_vf(adev))
2065 sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
2067 r = sdma_v4_4_2_inst_start(adev, inst_mask);
2072 static int sdma_v4_4_2_xcp_suspend(void *handle, uint32_t inst_mask)
2074 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2075 uint32_t tmp_mask = inst_mask;
2078 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2079 for_each_inst(i, tmp_mask) {
2080 amdgpu_irq_put(adev, &adev->sdma.ecc_irq,
2081 AMDGPU_SDMA_IRQ_INSTANCE0 + i);
2085 sdma_v4_4_2_inst_ctx_switch_enable(adev, false, inst_mask);
2086 sdma_v4_4_2_inst_enable(adev, false, inst_mask);
2091 struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = {
2092 .suspend = &sdma_v4_4_2_xcp_suspend,
2093 .resume = &sdma_v4_4_2_xcp_resume
2096 static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = {
2097 {AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI),
2098 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"},
2101 static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = {
2102 {AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"},
2103 {AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"},
2104 {AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"},
2105 {AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"},
2106 {AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"},
2107 {AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"},
2108 {AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"},
2109 {AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"},
2110 {AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"},
2111 {AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"},
2112 {AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"},
2113 {AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"},
2114 {AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"},
2115 {AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"},
2116 {AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"},
2117 {AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"},
2118 {AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"},
2119 {AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"},
2120 {AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"},
2121 {AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"},
2122 {AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"},
2123 {AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"},
2124 {AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"},
2125 {AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"},
2128 static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev,
2130 void *ras_err_status)
2132 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status;
2133 uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2135 /* sdma v4_4_2 doesn't support query ce counts */
2136 amdgpu_ras_inst_query_ras_error_count(adev,
2137 sdma_v4_2_2_ue_reg_list,
2138 ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2139 sdma_v4_4_2_ras_memory_list,
2140 ARRAY_SIZE(sdma_v4_4_2_ras_memory_list),
2142 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
2143 &err_data->ue_count);
2146 static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev,
2147 void *ras_err_status)
2152 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2153 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2154 for_each_inst(i, inst_mask)
2155 sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status);
2157 dev_warn(adev->dev, "SDMA RAS is not supported\n");
2161 static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev,
2164 uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst);
2166 amdgpu_ras_inst_reset_ras_error_count(adev,
2167 sdma_v4_2_2_ue_reg_list,
2168 ARRAY_SIZE(sdma_v4_2_2_ue_reg_list),
2172 static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev)
2177 inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
2178 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
2179 for_each_inst(i, inst_mask)
2180 sdma_v4_4_2_inst_reset_ras_error_count(adev, i);
2182 dev_warn(adev->dev, "SDMA RAS is not supported\n");
2186 static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = {
2187 .query_ras_error_count = sdma_v4_4_2_query_ras_error_count,
2188 .reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count,
2191 static struct amdgpu_sdma_ras sdma_v4_4_2_ras = {
2193 .hw_ops = &sdma_v4_4_2_ras_hw_ops,
2197 static void sdma_v4_4_2_set_ras_funcs(struct amdgpu_device *adev)
2199 adev->sdma.ras = &sdma_v4_4_2_ras;