drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c

   1 /*
   2  * Copyright 2023 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "amdgpu.h"
  26 #include "amdgpu_vcn.h"
  27 #include "amdgpu_pm.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "soc15_hw_ip.h"
  31 #include "vcn_v2_0.h"
  32
  33 #include "vcn/vcn_5_0_0_offset.h"
  34 #include "vcn/vcn_5_0_0_sh_mask.h"
  35 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
  36 #include "vcn_v5_0_0.h"
  37
  38 #include <drm/drm_drv.h>
  39
  40 static int amdgpu_ih_clientid_vcns[] = {
  41         SOC15_IH_CLIENTID_VCN,
  42         SOC15_IH_CLIENTID_VCN1
  43 };
  44
  45 static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev);
  46 static void vcn_v5_0_0_set_irq_funcs(struct amdgpu_device *adev);
  47 static int vcn_v5_0_0_set_powergating_state(void *handle,
  48                 enum amd_powergating_state state);
  49 static int vcn_v5_0_0_pause_dpg_mode(struct amdgpu_device *adev,
  50                 int inst_idx, struct dpg_pause_state *new_state);
  51 static void vcn_v5_0_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
  52
  53 /**
  54  * vcn_v5_0_0_early_init - set function pointers and load microcode
  55  *
  56  * @handle: amdgpu_device pointer
  57  *
  58  * Set ring and irq function pointers
  59  * Load microcode from filesystem
  60  */
  61 static int vcn_v5_0_0_early_init(void *handle)
  62 {
  63         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  64
  65         /* re-use enc ring as unified ring */
  66         adev->vcn.num_enc_rings = 1;
  67
  68         vcn_v5_0_0_set_unified_ring_funcs(adev);
  69         vcn_v5_0_0_set_irq_funcs(adev);
  70
  71         return amdgpu_vcn_early_init(adev);
  72 }
  73
  74 /**
  75  * vcn_v5_0_0_sw_init - sw init for VCN block
  76  *
  77  * @handle: amdgpu_device pointer
  78  *
  79  * Load firmware and sw initialization
  80  */
  81 static int vcn_v5_0_0_sw_init(void *handle)
  82 {
  83         struct amdgpu_ring *ring;
  84         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  85         int i, r;
  86
  87         r = amdgpu_vcn_sw_init(adev);
  88         if (r)
  89                 return r;
  90
  91         amdgpu_vcn_setup_ucode(adev);
  92
  93         r = amdgpu_vcn_resume(adev);
  94         if (r)
  95                 return r;
  96
  97         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
  98                 volatile struct amdgpu_vcn4_fw_shared *fw_shared;
  99
 100                 if (adev->vcn.harvest_config & (1 << i))
 101                         continue;
 102
 103                 atomic_set(&adev->vcn.inst[i].sched_score, 0);
 104
 105                 /* VCN UNIFIED TRAP */
 106                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 107                                 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
 108                 if (r)
 109                         return r;
 110
 111                 /* VCN POISON TRAP */
 112                 r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
 113                                 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst[i].irq);
 114                 if (r)
 115                         return r;
 116
 117                 ring = &adev->vcn.inst[i].ring_enc[0];
 118                 ring->use_doorbell = true;
 119                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
 120
 121                 ring->vm_hub = AMDGPU_MMHUB0(0);
 122                 sprintf(ring->name, "vcn_unified_%d", i);
 123
 124                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 125                                                 AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
 126                 if (r)
 127                         return r;
 128
 129                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 130                 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
 131                 fw_shared->sq.is_enabled = 1;
 132
 133                 if (amdgpu_vcnfw_log)
 134                         amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
 135         }
 136
 137         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 138                 adev->vcn.pause_dpg_mode = vcn_v5_0_0_pause_dpg_mode;
 139
 140         return 0;
 141 }
 142
 143 /**
 144  * vcn_v5_0_0_sw_fini - sw fini for VCN block
 145  *
 146  * @handle: amdgpu_device pointer
 147  *
 148  * VCN suspend and free up sw allocation
 149  */
 150 static int vcn_v5_0_0_sw_fini(void *handle)
 151 {
 152         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 153         int i, r, idx;
 154
 155         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 156                 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 157                         volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 158
 159                         if (adev->vcn.harvest_config & (1 << i))
 160                                 continue;
 161
 162                         fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 163                         fw_shared->present_flag_0 = 0;
 164                         fw_shared->sq.is_enabled = 0;
 165                 }
 166
 167                 drm_dev_exit(idx);
 168         }
 169
 170         r = amdgpu_vcn_suspend(adev);
 171         if (r)
 172                 return r;
 173
 174         r = amdgpu_vcn_sw_fini(adev);
 175
 176         return r;
 177 }
 178
 179 /**
 180  * vcn_v5_0_0_hw_init - start and test VCN block
 181  *
 182  * @handle: amdgpu_device pointer
 183  *
 184  * Initialize the hardware, boot up the VCPU and do some testing
 185  */
 186 static int vcn_v5_0_0_hw_init(void *handle)
 187 {
 188         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 189         struct amdgpu_ring *ring;
 190         int i, r;
 191
 192         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 193                 if (adev->vcn.harvest_config & (1 << i))
 194                         continue;
 195
 196                 ring = &adev->vcn.inst[i].ring_enc[0];
 197
 198                 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 199                         ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
 200
 201                 r = amdgpu_ring_test_helper(ring);
 202                 if (r)
 203                         goto done;
 204         }
 205
 206 done:
 207         if (!r)
 208                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 209                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 210
 211         return r;
 212 }
 213
 214 /**
 215  * vcn_v5_0_0_hw_fini - stop the hardware block
 216  *
 217  * @handle: amdgpu_device pointer
 218  *
 219  * Stop the VCN block, mark ring as not ready any more
 220  */
 221 static int vcn_v5_0_0_hw_fini(void *handle)
 222 {
 223         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 224         int i;
 225
 226         cancel_delayed_work_sync(&adev->vcn.idle_work);
 227
 228         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 229                 if (adev->vcn.harvest_config & (1 << i))
 230                         continue;
 231
 232                 amdgpu_irq_put(adev, &adev->vcn.inst[i].irq, 0);
 233         }
 234
 235         return 0;
 236 }
 237
 238 /**
 239  * vcn_v5_0_0_suspend - suspend VCN block
 240  *
 241  * @handle: amdgpu_device pointer
 242  *
 243  * HW fini and suspend VCN block
 244  */
 245 static int vcn_v5_0_0_suspend(void *handle)
 246 {
 247         int r;
 248         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 249
 250         r = vcn_v5_0_0_hw_fini(adev);
 251         if (r)
 252                 return r;
 253
 254         r = amdgpu_vcn_suspend(adev);
 255
 256         return r;
 257 }
 258
 259 /**
 260  * vcn_v5_0_0_resume - resume VCN block
 261  *
 262  * @handle: amdgpu_device pointer
 263  *
 264  * Resume firmware and hw init VCN block
 265  */
 266 static int vcn_v5_0_0_resume(void *handle)
 267 {
 268         int r;
 269         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 270
 271         r = amdgpu_vcn_resume(adev);
 272         if (r)
 273                 return r;
 274
 275         r = vcn_v5_0_0_hw_init(adev);
 276
 277         return r;
 278 }
 279
 280 /**
 281  * vcn_v5_0_0_mc_resume - memory controller programming
 282  *
 283  * @adev: amdgpu_device pointer
 284  * @inst: instance number
 285  *
 286  * Let the VCN memory controller know it's offsets
 287  */
 288 static void vcn_v5_0_0_mc_resume(struct amdgpu_device *adev, int inst)
 289 {
 290         uint32_t offset, size;
 291         const struct common_firmware_header *hdr;
 292
 293         hdr = (const struct common_firmware_header *)adev->vcn.fw[inst]->data;
 294         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 295
 296         /* cache window 0: fw */
 297         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 298                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 299                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
 300                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 301                         (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
 302                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
 303                 offset = 0;
 304         } else {
 305                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 306                         lower_32_bits(adev->vcn.inst[inst].gpu_addr));
 307                 WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 308                         upper_32_bits(adev->vcn.inst[inst].gpu_addr));
 309                 offset = size;
 310                 WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 311         }
 312         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
 313
 314         /* cache window 1: stack */
 315         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 316                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 317         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 318                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
 319         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
 320         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 321
 322         /* cache window 2: context */
 323         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 324                 lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 325         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 326                 upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 327         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
 328         WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 329
 330         /* non-cache window */
 331         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
 332                 lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 333         WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
 334                 upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
 335         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
 336         WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
 337                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
 338 }
 339
 340 /**
 341  * vcn_v5_0_0_mc_resume_dpg_mode - memory controller programming for dpg mode
 342  *
 343  * @adev: amdgpu_device pointer
 344  * @inst_idx: instance number index
 345  * @indirect: indirectly write sram
 346  *
 347  * Let the VCN memory controller know it's offsets with dpg mode
 348  */
 349 static void vcn_v5_0_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 350 {
 351         uint32_t offset, size;
 352         const struct common_firmware_header *hdr;
 353
 354         hdr = (const struct common_firmware_header *)adev->vcn.fw[inst_idx]->data;
 355         size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
 356
 357         /* cache window 0: fw */
 358         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 359                 if (!indirect) {
 360                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 361                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 362                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
 363                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 364                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 365                                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
 366                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 367                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 368                 } else {
 369                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 370                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
 371                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 372                                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
 373                         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 374                                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
 375                 }
 376                 offset = 0;
 377         } else {
 378                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 379                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
 380                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 381                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 382                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
 383                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
 384                 offset = size;
 385                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 386                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
 387                         AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
 388         }
 389
 390         if (!indirect)
 391                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 392                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
 393         else
 394                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 395                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
 396
 397         /* cache window 1: stack */
 398         if (!indirect) {
 399                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 400                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
 401                         lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 402                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 403                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
 404                         upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
 405                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 406                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 407         } else {
 408                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 409                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
 410                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 411                         VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
 412                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 413                         VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
 414         }
 415                 WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 416                         VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
 417
 418         /* cache window 2: context */
 419         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 420                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
 421                 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 422         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 423                 VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
 424                 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
 425         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 426                 VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
 427         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 428                 VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
 429
 430         /* non-cache window */
 431         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 432                 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
 433                 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 434         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 435                 VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
 436                 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
 437         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 438                 VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
 439         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 440                 VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
 441                 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
 442
 443         /* VCN global tiling registers */
 444         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 445                 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
 446
 447         return;
 448 }
 449
 450 /**
 451  * vcn_v5_0_0_disable_static_power_gating - disable VCN static power gating
 452  *
 453  * @adev: amdgpu_device pointer
 454  * @inst: instance number
 455  *
 456  * Disable static power gating for VCN block
 457  */
 458 static void vcn_v5_0_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
 459 {
 460         uint32_t data = 0;
 461
 462         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 463                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 464                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 465                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 466                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 467
 468                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 469                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 470                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 471                                 1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 472                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 473
 474                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 475                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 476                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 477                                 1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 478                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 479
 480                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 481                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 482                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 483                                 1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 484                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 485         } else {
 486                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 487                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 488                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 489                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 490
 491                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 492                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 493                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 494                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 495
 496                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 497                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 498                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 499                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 500
 501                 data = 1 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 502                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 503                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS, 0,
 504                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 505         }
 506
 507         data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 508         data &= ~0x103;
 509         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 510                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
 511                         UVD_POWER_STATUS__UVD_PG_EN_MASK;
 512
 513         WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 514         return;
 515 }
 516
 517 /**
 518  * vcn_v5_0_0_enable_static_power_gating - enable VCN static power gating
 519  *
 520  * @adev: amdgpu_device pointer
 521  * @inst: instance number
 522  *
 523  * Enable static power gating for VCN block
 524  */
 525 static void vcn_v5_0_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
 526 {
 527         uint32_t data;
 528
 529         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 530                 /* Before power off, this indicator has to be turned on */
 531                 data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
 532                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 533                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 534                 WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
 535
 536                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO5_PWR_CONFIG__SHIFT;
 537                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 538                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 539                                 1 << UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS__SHIFT,
 540                                 UVD_IPX_DLDO_STATUS__ONO5_PWR_STATUS_MASK);
 541
 542                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO4_PWR_CONFIG__SHIFT;
 543                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 544                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 545                                 1 << UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS__SHIFT,
 546                                 UVD_IPX_DLDO_STATUS__ONO4_PWR_STATUS_MASK);
 547
 548                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO3_PWR_CONFIG__SHIFT;
 549                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 550                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 551                                 1 << UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS__SHIFT,
 552                                 UVD_IPX_DLDO_STATUS__ONO3_PWR_STATUS_MASK);
 553
 554                 data = 2 << UVD_IPX_DLDO_CONFIG__ONO2_PWR_CONFIG__SHIFT;
 555                 WREG32_SOC15(VCN, inst, regUVD_IPX_DLDO_CONFIG, data);
 556                 SOC15_WAIT_ON_RREG(VCN, inst, regUVD_IPX_DLDO_STATUS,
 557                                 1 << UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS__SHIFT,
 558                                 UVD_IPX_DLDO_STATUS__ONO2_PWR_STATUS_MASK);
 559         }
 560         return;
 561 }
 562
 563 /**
 564  * vcn_v5_0_0_disable_clock_gating - disable VCN clock gating
 565  *
 566  * @adev: amdgpu_device pointer
 567  * @inst: instance number
 568  *
 569  * Disable clock gating for VCN block
 570  */
 571 static void vcn_v5_0_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
 572 {
 573         return;
 574 }
 575
 576 #if 0
 577 /**
 578  * vcn_v5_0_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
 579  *
 580  * @adev: amdgpu_device pointer
 581  * @sram_sel: sram select
 582  * @inst_idx: instance number index
 583  * @indirect: indirectly write sram
 584  *
 585  * Disable clock gating for VCN block with dpg mode
 586  */
 587 static void vcn_v5_0_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
 588         int inst_idx, uint8_t indirect)
 589 {
 590         return;
 591 }
 592 #endif
 593
 594 /**
 595  * vcn_v5_0_0_enable_clock_gating - enable VCN clock gating
 596  *
 597  * @adev: amdgpu_device pointer
 598  * @inst: instance number
 599  *
 600  * Enable clock gating for VCN block
 601  */
 602 static void vcn_v5_0_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
 603 {
 604         return;
 605 }
 606
 607 /**
 608  * vcn_v5_0_0_start_dpg_mode - VCN start with dpg mode
 609  *
 610  * @adev: amdgpu_device pointer
 611  * @inst_idx: instance number index
 612  * @indirect: indirectly write sram
 613  *
 614  * Start VCN block with dpg mode
 615  */
 616 static int vcn_v5_0_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
 617 {
 618         volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
 619         struct amdgpu_ring *ring;
 620         uint32_t tmp;
 621
 622         /* disable register anti-hang mechanism */
 623         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
 624                 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 625
 626         /* enable dynamic power gating mode */
 627         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
 628         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 629         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 630         WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
 631
 632         if (indirect)
 633                 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 634
 635         /* enable VCPU clock */
 636         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 637         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
 638         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 639                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 640
 641         /* disable master interrupt */
 642         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 643                 VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
 644
 645         /* setup regUVD_LMI_CTRL */
 646         tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 647                 UVD_LMI_CTRL__REQ_MODE_MASK |
 648                 UVD_LMI_CTRL__CRC_RESET_MASK |
 649                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 650                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 651                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
 652                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
 653                 0x00100000L);
 654         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 655                 VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
 656
 657         vcn_v5_0_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
 658
 659         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 660         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 661         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 662                 VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
 663
 664         /* enable LMI MC and UMC channels */
 665         tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
 666         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 667                 VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
 668
 669         /* enable master interrupt */
 670         WREG32_SOC24_DPG_MODE(inst_idx, SOC24_DPG_MODE_OFFSET(
 671                 VCN, inst_idx, regUVD_MASTINT_EN),
 672                 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
 673
 674         if (indirect)
 675                 amdgpu_vcn_psp_update_sram(adev, inst_idx, 0);
 676
 677         ring = &adev->vcn.inst[inst_idx].ring_enc[0];
 678
 679         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
 680         WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 681         WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
 682
 683         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
 684         tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
 685         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
 686         fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
 687         WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
 688         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
 689
 690         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
 691         WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
 692         ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
 693
 694         tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
 695         tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
 696         WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
 697         fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
 698
 699         WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
 700                 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
 701                 VCN_RB1_DB_CTRL__EN_MASK);
 702
 703         return 0;
 704 }
 705
 706 /**
 707  * vcn_v5_0_0_start - VCN start
 708  *
 709  * @adev: amdgpu_device pointer
 710  *
 711  * Start VCN block
 712  */
 713 static int vcn_v5_0_0_start(struct amdgpu_device *adev)
 714 {
 715         volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 716         struct amdgpu_ring *ring;
 717         uint32_t tmp;
 718         int i, j, k, r;
 719
 720         if (adev->pm.dpm_enabled)
 721                 amdgpu_dpm_enable_uvd(adev, true);
 722
 723         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 724                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 725
 726                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
 727                         r = vcn_v5_0_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
 728                         continue;
 729                 }
 730
 731                 /* disable VCN power gating */
 732                 vcn_v5_0_0_disable_static_power_gating(adev, i);
 733
 734                 /* set VCN status busy */
 735                 tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
 736                 WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
 737
 738                 /* enable VCPU clock */
 739                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 740                         UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
 741
 742                 /* disable master interrupt */
 743                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
 744                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 745
 746                 /* enable LMI MC and UMC channels */
 747                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
 748                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 749
 750                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 751                 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 752                 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 753                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 754
 755                 /* setup regUVD_LMI_CTRL */
 756                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
 757                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
 758                         UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 759                         UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
 760                         UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
 761                         UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
 762
 763                 vcn_v5_0_0_mc_resume(adev, i);
 764
 765                 /* VCN global tiling registers */
 766                 WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
 767                         adev->gfx.config.gb_addr_config);
 768
 769                 /* unblock VCPU register access */
 770                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
 771                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
 772
 773                 /* release VCPU reset to boot */
 774                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 775                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 776
 777                 for (j = 0; j < 10; ++j) {
 778                         uint32_t status;
 779
 780                         for (k = 0; k < 100; ++k) {
 781                                 status = RREG32_SOC15(VCN, i, regUVD_STATUS);
 782                                 if (status & 2)
 783                                         break;
 784                                 mdelay(10);
 785                                 if (amdgpu_emu_mode == 1)
 786                                         msleep(1);
 787                         }
 788
 789                         if (amdgpu_emu_mode == 1) {
 790                                 r = -1;
 791                                 if (status & 2) {
 792                                         r = 0;
 793                                         break;
 794                                 }
 795                         } else {
 796                                 r = 0;
 797                                 if (status & 2)
 798                                         break;
 799
 800                                 dev_err(adev->dev,
 801                                         "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
 802                                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 803                                                         UVD_VCPU_CNTL__BLK_RST_MASK,
 804                                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 805                                 mdelay(10);
 806                                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 807                                                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 808
 809                                 mdelay(10);
 810                                 r = -1;
 811                         }
 812                 }
 813
 814                 if (r) {
 815                         dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
 816                         return r;
 817                 }
 818
 819                 /* enable master interrupt */
 820                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
 821                                 UVD_MASTINT_EN__VCPU_EN_MASK,
 822                                 ~UVD_MASTINT_EN__VCPU_EN_MASK);
 823
 824                 /* clear the busy bit of VCN_STATUS */
 825                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
 826                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 827
 828                 ring = &adev->vcn.inst[i].ring_enc[0];
 829                 WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
 830                         ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
 831                         VCN_RB1_DB_CTRL__EN_MASK);
 832
 833                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
 834                 WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 835                 WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
 836
 837                 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
 838                 tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
 839                 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
 840                 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
 841                 WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
 842                 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
 843
 844                 tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
 845                 WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
 846                 ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
 847
 848                 tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
 849                 tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
 850                 WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
 851                 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
 852         }
 853
 854         return 0;
 855 }
 856
 857 /**
 858  * vcn_v5_0_0_stop_dpg_mode - VCN stop with dpg mode
 859  *
 860  * @adev: amdgpu_device pointer
 861  * @inst_idx: instance number index
 862  *
 863  * Stop VCN block with dpg mode
 864  */
 865 static void vcn_v5_0_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
 866 {
 867         struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
 868         uint32_t tmp;
 869
 870         vcn_v5_0_0_pause_dpg_mode(adev, inst_idx, &state);
 871
 872         /* Wait for power status to be 1 */
 873         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
 874                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 875
 876         /* wait for read ptr to be equal to write ptr */
 877         tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
 878         SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
 879
 880         /* disable dynamic power gating mode */
 881         WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
 882                 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
 883
 884         return;
 885 }
 886
 887 /**
 888  * vcn_v5_0_0_stop - VCN stop
 889  *
 890  * @adev: amdgpu_device pointer
 891  *
 892  * Stop VCN block
 893  */
 894 static int vcn_v5_0_0_stop(struct amdgpu_device *adev)
 895 {
 896         volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 897         uint32_t tmp;
 898         int i, r = 0;
 899
 900         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 901                 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 902                 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
 903
 904                 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
 905                         vcn_v5_0_0_stop_dpg_mode(adev, i);
 906                         continue;
 907                 }
 908
 909                 /* wait for vcn idle */
 910                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
 911                 if (r)
 912                         return r;
 913
 914                 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
 915                       UVD_LMI_STATUS__READ_CLEAN_MASK |
 916                       UVD_LMI_STATUS__WRITE_CLEAN_MASK |
 917                       UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
 918                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
 919                 if (r)
 920                         return r;
 921
 922                 /* disable LMI UMC channel */
 923                 tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
 924                 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
 925                 WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
 926                 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
 927                       UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
 928                 r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
 929                 if (r)
 930                         return r;
 931
 932                 /* block VCPU register access */
 933                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
 934                         UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
 935                         ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
 936
 937                 /* reset VCPU */
 938                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
 939                         UVD_VCPU_CNTL__BLK_RST_MASK,
 940                         ~UVD_VCPU_CNTL__BLK_RST_MASK);
 941
 942                 /* disable VCPU clock */
 943                 WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
 944                         ~(UVD_VCPU_CNTL__CLK_EN_MASK));
 945
 946                 /* apply soft reset */
 947                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 948                 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 949                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 950                 tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
 951                 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 952                 WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
 953
 954                 /* clear status */
 955                 WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
 956
 957                 /* enable VCN power gating */
 958                 vcn_v5_0_0_enable_static_power_gating(adev, i);
 959         }
 960
 961         if (adev->pm.dpm_enabled)
 962                 amdgpu_dpm_enable_uvd(adev, false);
 963
 964         return 0;
 965 }
 966
 967 /**
 968  * vcn_v5_0_0_pause_dpg_mode - VCN pause with dpg mode
 969  *
 970  * @adev: amdgpu_device pointer
 971  * @inst_idx: instance number index
 972  * @new_state: pause state
 973  *
 974  * Pause dpg mode for VCN block
 975  */
 976 static int vcn_v5_0_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
 977         struct dpg_pause_state *new_state)
 978 {
 979         uint32_t reg_data = 0;
 980         int ret_code;
 981
 982         /* pause/unpause if state is changed */
 983         if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
 984                 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
 985                         adev->vcn.inst[inst_idx].pause_state.fw_based,  new_state->fw_based);
 986                 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
 987                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
 988
 989                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
 990                         ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
 991                                         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
 992
 993                         if (!ret_code) {
 994                                 /* pause DPG */
 995                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
 996                                 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
 997
 998                                 /* wait for ACK */
 999                                 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1000                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1001                                         UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1002                         }
1003                 } else {
1004                         /* unpause dpg, no need to wait */
1005                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1006                         WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1007                 }
1008                 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1009         }
1010
1011         return 0;
1012 }
1013
1014 /**
1015  * vcn_v5_0_0_unified_ring_get_rptr - get unified read pointer
1016  *
1017  * @ring: amdgpu_ring pointer
1018  *
1019  * Returns the current hardware unified read pointer
1020  */
1021 static uint64_t vcn_v5_0_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1022 {
1023         struct amdgpu_device *adev = ring->adev;
1024
1025         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1026                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1027
1028         return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1029 }
1030
1031 /**
1032  * vcn_v5_0_0_unified_ring_get_wptr - get unified write pointer
1033  *
1034  * @ring: amdgpu_ring pointer
1035  *
1036  * Returns the current hardware unified write pointer
1037  */
1038 static uint64_t vcn_v5_0_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1039 {
1040         struct amdgpu_device *adev = ring->adev;
1041
1042         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1043                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1044
1045         if (ring->use_doorbell)
1046                 return *ring->wptr_cpu_addr;
1047         else
1048                 return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1049 }
1050
1051 /**
1052  * vcn_v5_0_0_unified_ring_set_wptr - set enc write pointer
1053  *
1054  * @ring: amdgpu_ring pointer
1055  *
1056  * Commits the enc write pointer to the hardware
1057  */
1058 static void vcn_v5_0_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1059 {
1060         struct amdgpu_device *adev = ring->adev;
1061
1062         if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1063                 DRM_ERROR("wrong ring id is identified in %s", __func__);
1064
1065         if (ring->use_doorbell) {
1066                 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1067                 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1068         } else {
1069                 WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1070         }
1071 }
1072
1073 static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs = {
1074         .type = AMDGPU_RING_TYPE_VCN_ENC,
1075         .align_mask = 0x3f,
1076         .nop = VCN_ENC_CMD_NO_OP,
1077         .get_rptr = vcn_v5_0_0_unified_ring_get_rptr,
1078         .get_wptr = vcn_v5_0_0_unified_ring_get_wptr,
1079         .set_wptr = vcn_v5_0_0_unified_ring_set_wptr,
1080         .emit_frame_size =
1081                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1082                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1083                 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1084                 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1085                 1, /* vcn_v2_0_enc_ring_insert_end */
1086         .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1087         .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1088         .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1089         .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1090         .test_ring = amdgpu_vcn_enc_ring_test_ring,
1091         .test_ib = amdgpu_vcn_unified_ring_test_ib,
1092         .insert_nop = amdgpu_ring_insert_nop,
1093         .insert_end = vcn_v2_0_enc_ring_insert_end,
1094         .pad_ib = amdgpu_ring_generic_pad_ib,
1095         .begin_use = amdgpu_vcn_ring_begin_use,
1096         .end_use = amdgpu_vcn_ring_end_use,
1097         .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1098         .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1099         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1100 };
1101
1102 /**
1103  * vcn_v5_0_0_set_unified_ring_funcs - set unified ring functions
1104  *
1105  * @adev: amdgpu_device pointer
1106  *
1107  * Set unified ring functions
1108  */
1109 static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev)
1110 {
1111         int i;
1112
1113         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1114                 if (adev->vcn.harvest_config & (1 << i))
1115                         continue;
1116
1117                 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v5_0_0_unified_ring_vm_funcs;
1118                 adev->vcn.inst[i].ring_enc[0].me = i;
1119
1120                 DRM_INFO("VCN(%d) encode/decode are enabled in VM mode\n", i);
1121         }
1122 }
1123
1124 /**
1125  * vcn_v5_0_0_is_idle - check VCN block is idle
1126  *
1127  * @handle: amdgpu_device pointer
1128  *
1129  * Check whether VCN block is idle
1130  */
1131 static bool vcn_v5_0_0_is_idle(void *handle)
1132 {
1133         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1134         int i, ret = 1;
1135
1136         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1137                 if (adev->vcn.harvest_config & (1 << i))
1138                         continue;
1139
1140                 ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1141         }
1142
1143         return ret;
1144 }
1145
1146 /**
1147  * vcn_v5_0_0_wait_for_idle - wait for VCN block idle
1148  *
1149  * @handle: amdgpu_device pointer
1150  *
1151  * Wait for VCN block idle
1152  */
1153 static int vcn_v5_0_0_wait_for_idle(void *handle)
1154 {
1155         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1156         int i, ret = 0;
1157
1158         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1159                 if (adev->vcn.harvest_config & (1 << i))
1160                         continue;
1161
1162                 ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1163                         UVD_STATUS__IDLE);
1164                 if (ret)
1165                         return ret;
1166         }
1167
1168         return ret;
1169 }
1170
1171 /**
1172  * vcn_v5_0_0_set_clockgating_state - set VCN block clockgating state
1173  *
1174  * @handle: amdgpu_device pointer
1175  * @state: clock gating state
1176  *
1177  * Set VCN block clockgating state
1178  */
1179 static int vcn_v5_0_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1180 {
1181         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1182         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1183         int i;
1184
1185         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1186                 if (adev->vcn.harvest_config & (1 << i))
1187                         continue;
1188
1189                 if (enable) {
1190                         if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1191                                 return -EBUSY;
1192                         vcn_v5_0_0_enable_clock_gating(adev, i);
1193                 } else {
1194                         vcn_v5_0_0_disable_clock_gating(adev, i);
1195                 }
1196         }
1197
1198         return 0;
1199 }
1200
1201 /**
1202  * vcn_v5_0_0_set_powergating_state - set VCN block powergating state
1203  *
1204  * @handle: amdgpu_device pointer
1205  * @state: power gating state
1206  *
1207  * Set VCN block powergating state
1208  */
1209 static int vcn_v5_0_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1210 {
1211         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1212         int ret;
1213
1214         if (state == adev->vcn.cur_state)
1215                 return 0;
1216
1217         if (state == AMD_PG_STATE_GATE)
1218                 ret = vcn_v5_0_0_stop(adev);
1219         else
1220                 ret = vcn_v5_0_0_start(adev);
1221
1222         if (!ret)
1223                 adev->vcn.cur_state = state;
1224
1225         return ret;
1226 }
1227
1228 /**
1229  * vcn_v5_0_0_set_interrupt_state - set VCN block interrupt state
1230  *
1231  * @adev: amdgpu_device pointer
1232  * @source: interrupt sources
1233  * @type: interrupt types
1234  * @state: interrupt states
1235  *
1236  * Set VCN block interrupt state
1237  */
1238 static int vcn_v5_0_0_set_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1239         unsigned type, enum amdgpu_interrupt_state state)
1240 {
1241         return 0;
1242 }
1243
1244 /**
1245  * vcn_v5_0_0_process_interrupt - process VCN block interrupt
1246  *
1247  * @adev: amdgpu_device pointer
1248  * @source: interrupt sources
1249  * @entry: interrupt entry from clients and sources
1250  *
1251  * Process VCN block interrupt
1252  */
1253 static int vcn_v5_0_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1254         struct amdgpu_iv_entry *entry)
1255 {
1256         uint32_t ip_instance;
1257
1258         switch (entry->client_id) {
1259         case SOC15_IH_CLIENTID_VCN:
1260                 ip_instance = 0;
1261                 break;
1262         case SOC15_IH_CLIENTID_VCN1:
1263                 ip_instance = 1;
1264                 break;
1265         default:
1266                 DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1267                 return 0;
1268         }
1269
1270         DRM_DEBUG("IH: VCN TRAP\n");
1271
1272         switch (entry->src_id) {
1273         case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1274                 amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1275                 break;
1276         case VCN_4_0__SRCID_UVD_POISON:
1277                 amdgpu_vcn_process_poison_irq(adev, source, entry);
1278                 break;
1279         default:
1280                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1281                           entry->src_id, entry->src_data[0]);
1282                 break;
1283         }
1284
1285         return 0;
1286 }
1287
1288 static const struct amdgpu_irq_src_funcs vcn_v5_0_0_irq_funcs = {
1289         .set = vcn_v5_0_0_set_interrupt_state,
1290         .process = vcn_v5_0_0_process_interrupt,
1291 };
1292
1293 /**
1294  * vcn_v5_0_0_set_irq_funcs - set VCN block interrupt irq functions
1295  *
1296  * @adev: amdgpu_device pointer
1297  *
1298  * Set VCN block interrupt irq functions
1299  */
1300 static void vcn_v5_0_0_set_irq_funcs(struct amdgpu_device *adev)
1301 {
1302         int i;
1303
1304         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1305                 if (adev->vcn.harvest_config & (1 << i))
1306                         continue;
1307
1308                 adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1309                 adev->vcn.inst[i].irq.funcs = &vcn_v5_0_0_irq_funcs;
1310         }
1311 }
1312
1313 static const struct amd_ip_funcs vcn_v5_0_0_ip_funcs = {
1314         .name = "vcn_v5_0_0",
1315         .early_init = vcn_v5_0_0_early_init,
1316         .late_init = NULL,
1317         .sw_init = vcn_v5_0_0_sw_init,
1318         .sw_fini = vcn_v5_0_0_sw_fini,
1319         .hw_init = vcn_v5_0_0_hw_init,
1320         .hw_fini = vcn_v5_0_0_hw_fini,
1321         .suspend = vcn_v5_0_0_suspend,
1322         .resume = vcn_v5_0_0_resume,
1323         .is_idle = vcn_v5_0_0_is_idle,
1324         .wait_for_idle = vcn_v5_0_0_wait_for_idle,
1325         .check_soft_reset = NULL,
1326         .pre_soft_reset = NULL,
1327         .soft_reset = NULL,
1328         .post_soft_reset = NULL,
1329         .set_clockgating_state = vcn_v5_0_0_set_clockgating_state,
1330         .set_powergating_state = vcn_v5_0_0_set_powergating_state,
1331 };
1332
1333 const struct amdgpu_ip_block_version vcn_v5_0_0_ip_block = {
1334         .type = AMD_IP_BLOCK_TYPE_VCN,
1335         .major = 5,
1336         .minor = 0,
1337         .rev = 0,
1338         .funcs = &vcn_v5_0_0_ip_funcs,
1339 };