drivers/gpu/drm/msm/adreno/a6xx_gpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
   3
   4
   5 #include "msm_gem.h"
   6 #include "msm_mmu.h"
   7 #include "msm_gpu_trace.h"
   8 #include "a6xx_gpu.h"
   9 #include "a6xx_gmu.xml.h"
  10
  11 #include <linux/bitfield.h>
  12 #include <linux/devfreq.h>
  13 #include <linux/pm_domain.h>
  14 #include <linux/soc/qcom/llcc-qcom.h>
  15
  16 #define GPU_PAS_ID 13
  17
  18 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
  19 {
  20         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  21         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
  22
  23         /* Check that the GMU is idle */
  24         if (!adreno_has_gmu_wrapper(adreno_gpu) && !a6xx_gmu_isidle(&a6xx_gpu->gmu))
  25                 return false;
  26
  27         /* Check tha the CX master is idle */
  28         if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
  29                         ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
  30                 return false;
  31
  32         return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
  33                 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
  34 }
  35
  36 static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
  37 {
  38         /* wait for CP to drain ringbuffer: */
  39         if (!adreno_idle(gpu, ring))
  40                 return false;
  41
  42         if (spin_until(_a6xx_check_idle(gpu))) {
  43                 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
  44                         gpu->name, __builtin_return_address(0),
  45                         gpu_read(gpu, REG_A6XX_RBBM_STATUS),
  46                         gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
  47                         gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
  48                         gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
  49                 return false;
  50         }
  51
  52         return true;
  53 }
  54
  55 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
  56 {
  57         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
  58         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
  59
  60         /* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */
  61         if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {
  62                 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
  63                 OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));
  64                 OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));
  65         }
  66 }
  67
  68 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
  69 {
  70         uint32_t wptr;
  71         unsigned long flags;
  72
  73         update_shadow_rptr(gpu, ring);
  74
  75         spin_lock_irqsave(&ring->preempt_lock, flags);
  76
  77         /* Copy the shadow to the actual register */
  78         ring->cur = ring->next;
  79
  80         /* Make sure to wrap wptr if we need to */
  81         wptr = get_wptr(ring);
  82
  83         spin_unlock_irqrestore(&ring->preempt_lock, flags);
  84
  85         /* Make sure everything is posted before making a decision */
  86         mb();
  87
  88         gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
  89 }
  90
  91 static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter,
  92                 u64 iova)
  93 {
  94         OUT_PKT7(ring, CP_REG_TO_MEM, 3);
  95         OUT_RING(ring, CP_REG_TO_MEM_0_REG(counter) |
  96                 CP_REG_TO_MEM_0_CNT(2) |
  97                 CP_REG_TO_MEM_0_64B);
  98         OUT_RING(ring, lower_32_bits(iova));
  99         OUT_RING(ring, upper_32_bits(iova));
 100 }
 101
 102 static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
 103                 struct msm_ringbuffer *ring, struct msm_file_private *ctx)
 104 {
 105         bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1;
 106         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 107         phys_addr_t ttbr;
 108         u32 asid;
 109         u64 memptr = rbmemptr(ring, ttbr0);
 110
 111         if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno)
 112                 return;
 113
 114         if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid))
 115                 return;
 116
 117         if (!sysprof) {
 118                 if (!adreno_is_a7xx(adreno_gpu)) {
 119                         /* Turn off protected mode to write to special registers */
 120                         OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 121                         OUT_RING(ring, 0);
 122                 }
 123
 124                 OUT_PKT4(ring, REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1);
 125                 OUT_RING(ring, 1);
 126         }
 127
 128         /* Execute the table update */
 129         OUT_PKT7(ring, CP_SMMU_TABLE_UPDATE, 4);
 130         OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr)));
 131
 132         OUT_RING(ring,
 133                 CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) |
 134                 CP_SMMU_TABLE_UPDATE_1_ASID(asid));
 135         OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(0));
 136         OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(0));
 137
 138         /*
 139          * Write the new TTBR0 to the memstore. This is good for debugging.
 140          */
 141         OUT_PKT7(ring, CP_MEM_WRITE, 4);
 142         OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr)));
 143         OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr)));
 144         OUT_RING(ring, lower_32_bits(ttbr));
 145         OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr));
 146
 147         /*
 148          * Sync both threads after switching pagetables and enable BR only
 149          * to make sure BV doesn't race ahead while BR is still switching
 150          * pagetables.
 151          */
 152         if (adreno_is_a7xx(&a6xx_gpu->base)) {
 153                 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 154                 OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
 155         }
 156
 157         /*
 158          * And finally, trigger a uche flush to be sure there isn't anything
 159          * lingering in that part of the GPU
 160          */
 161
 162         OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 163         OUT_RING(ring, CACHE_INVALIDATE);
 164
 165         if (!sysprof) {
 166                 /*
 167                  * Wait for SRAM clear after the pgtable update, so the
 168                  * two can happen in parallel:
 169                  */
 170                 OUT_PKT7(ring, CP_WAIT_REG_MEM, 6);
 171                 OUT_RING(ring, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ));
 172                 OUT_RING(ring, CP_WAIT_REG_MEM_1_POLL_ADDR_LO(
 173                                 REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS));
 174                 OUT_RING(ring, CP_WAIT_REG_MEM_2_POLL_ADDR_HI(0));
 175                 OUT_RING(ring, CP_WAIT_REG_MEM_3_REF(0x1));
 176                 OUT_RING(ring, CP_WAIT_REG_MEM_4_MASK(0x1));
 177                 OUT_RING(ring, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0));
 178
 179                 if (!adreno_is_a7xx(adreno_gpu)) {
 180                         /* Re-enable protected mode: */
 181                         OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
 182                         OUT_RING(ring, 1);
 183                 }
 184         }
 185 }
 186
 187 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 188 {
 189         unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
 190         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 191         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 192         struct msm_ringbuffer *ring = submit->ring;
 193         unsigned int i, ibs = 0;
 194
 195         a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
 196
 197         get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
 198                 rbmemptr_stats(ring, index, cpcycles_start));
 199
 200         /*
 201          * For PM4 the GMU register offsets are calculated from the base of the
 202          * GPU registers so we need to add 0x1a800 to the register value on A630
 203          * to get the right value from PM4.
 204          */
 205         get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 206                 rbmemptr_stats(ring, index, alwayson_start));
 207
 208         /* Invalidate CCU depth and color */
 209         OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 210         OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_DEPTH));
 211
 212         OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 213         OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_COLOR));
 214
 215         /* Submit the commands */
 216         for (i = 0; i < submit->nr_cmds; i++) {
 217                 switch (submit->cmd[i].type) {
 218                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 219                         break;
 220                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 221                         if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
 222                                 break;
 223                         fallthrough;
 224                 case MSM_SUBMIT_CMD_BUF:
 225                         OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
 226                         OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 227                         OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
 228                         OUT_RING(ring, submit->cmd[i].size);
 229                         ibs++;
 230                         break;
 231                 }
 232
 233                 /*
 234                  * Periodically update shadow-wptr if needed, so that we
 235                  * can see partial progress of submits with large # of
 236                  * cmds.. otherwise we could needlessly stall waiting for
 237                  * ringbuffer state, simply due to looking at a shadow
 238                  * rptr value that has not been updated
 239                  */
 240                 if ((ibs % 32) == 0)
 241                         update_shadow_rptr(gpu, ring);
 242         }
 243
 244         get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
 245                 rbmemptr_stats(ring, index, cpcycles_end));
 246         get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 247                 rbmemptr_stats(ring, index, alwayson_end));
 248
 249         /* Write the fence to the scratch register */
 250         OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
 251         OUT_RING(ring, submit->seqno);
 252
 253         /*
 254          * Execute a CACHE_FLUSH_TS event. This will ensure that the
 255          * timestamp is written to the memory and then triggers the interrupt
 256          */
 257         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 258         OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
 259                 CP_EVENT_WRITE_0_IRQ);
 260         OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
 261         OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
 262         OUT_RING(ring, submit->seqno);
 263
 264         trace_msm_gpu_submit_flush(submit,
 265                 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
 266
 267         a6xx_flush(gpu, ring);
 268 }
 269
 270 static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 271 {
 272         unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
 273         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 274         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 275         struct msm_ringbuffer *ring = submit->ring;
 276         unsigned int i, ibs = 0;
 277
 278         /*
 279          * Toggle concurrent binning for pagetable switch and set the thread to
 280          * BR since only it can execute the pagetable switch packets.
 281          */
 282         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 283         OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
 284
 285         a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
 286
 287         get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
 288                 rbmemptr_stats(ring, index, cpcycles_start));
 289         get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 290                 rbmemptr_stats(ring, index, alwayson_start));
 291
 292         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 293         OUT_RING(ring, CP_SET_THREAD_BOTH);
 294
 295         OUT_PKT7(ring, CP_SET_MARKER, 1);
 296         OUT_RING(ring, 0x101); /* IFPC disable */
 297
 298         OUT_PKT7(ring, CP_SET_MARKER, 1);
 299         OUT_RING(ring, 0x00d); /* IB1LIST start */
 300
 301         /* Submit the commands */
 302         for (i = 0; i < submit->nr_cmds; i++) {
 303                 switch (submit->cmd[i].type) {
 304                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 305                         break;
 306                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 307                         if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
 308                                 break;
 309                         fallthrough;
 310                 case MSM_SUBMIT_CMD_BUF:
 311                         OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
 312                         OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
 313                         OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
 314                         OUT_RING(ring, submit->cmd[i].size);
 315                         ibs++;
 316                         break;
 317                 }
 318
 319                 /*
 320                  * Periodically update shadow-wptr if needed, so that we
 321                  * can see partial progress of submits with large # of
 322                  * cmds.. otherwise we could needlessly stall waiting for
 323                  * ringbuffer state, simply due to looking at a shadow
 324                  * rptr value that has not been updated
 325                  */
 326                 if ((ibs % 32) == 0)
 327                         update_shadow_rptr(gpu, ring);
 328         }
 329
 330         OUT_PKT7(ring, CP_SET_MARKER, 1);
 331         OUT_RING(ring, 0x00e); /* IB1LIST end */
 332
 333         get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
 334                 rbmemptr_stats(ring, index, cpcycles_end));
 335         get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
 336                 rbmemptr_stats(ring, index, alwayson_end));
 337
 338         /* Write the fence to the scratch register */
 339         OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
 340         OUT_RING(ring, submit->seqno);
 341
 342         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 343         OUT_RING(ring, CP_SET_THREAD_BR);
 344
 345         OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 346         OUT_RING(ring, CCU_INVALIDATE_DEPTH);
 347
 348         OUT_PKT7(ring, CP_EVENT_WRITE, 1);
 349         OUT_RING(ring, CCU_INVALIDATE_COLOR);
 350
 351         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 352         OUT_RING(ring, CP_SET_THREAD_BV);
 353
 354         /*
 355          * Make sure the timestamp is committed once BV pipe is
 356          * completely done with this submission.
 357          */
 358         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 359         OUT_RING(ring, CACHE_CLEAN | BIT(27));
 360         OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
 361         OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
 362         OUT_RING(ring, submit->seqno);
 363
 364         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 365         OUT_RING(ring, CP_SET_THREAD_BR);
 366
 367         /*
 368          * This makes sure that BR doesn't race ahead and commit
 369          * timestamp to memstore while BV is still processing
 370          * this submission.
 371          */
 372         OUT_PKT7(ring, CP_WAIT_TIMESTAMP, 4);
 373         OUT_RING(ring, 0);
 374         OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
 375         OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
 376         OUT_RING(ring, submit->seqno);
 377
 378         /* write the ringbuffer timestamp */
 379         OUT_PKT7(ring, CP_EVENT_WRITE, 4);
 380         OUT_RING(ring, CACHE_CLEAN | CP_EVENT_WRITE_0_IRQ | BIT(27));
 381         OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
 382         OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
 383         OUT_RING(ring, submit->seqno);
 384
 385         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
 386         OUT_RING(ring, CP_SET_THREAD_BOTH);
 387
 388         OUT_PKT7(ring, CP_SET_MARKER, 1);
 389         OUT_RING(ring, 0x100); /* IFPC enable */
 390
 391         trace_msm_gpu_submit_flush(submit,
 392                 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
 393
 394         a6xx_flush(gpu, ring);
 395 }
 396
 397 const struct adreno_reglist a612_hwcg[] = {
 398         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
 399         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 400         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081},
 401         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
 402         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 403         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 404         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 405         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 406         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 407         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 408         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 409         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 410         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 411         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 412         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 413         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 414         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 415         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222},
 416         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 417         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
 418         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022},
 419         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 420         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 421         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 422         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 423         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 424         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
 425         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 426         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 427         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 428         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 429         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 430         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 431         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 432         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 433         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 434         {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
 435         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 436         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 437         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 438         {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
 439         {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
 440         {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
 441         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 442         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 443         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 444         {},
 445 };
 446
 447 /* For a615 family (a615, a616, a618 and a619) */
 448 const struct adreno_reglist a615_hwcg[] = {
 449         {REG_A6XX_RBBM_CLOCK_CNTL_SP0,  0x02222222},
 450         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 451         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 452         {REG_A6XX_RBBM_CLOCK_HYST_SP0,  0x0000F3CF},
 453         {REG_A6XX_RBBM_CLOCK_CNTL_TP0,  0x02222222},
 454         {REG_A6XX_RBBM_CLOCK_CNTL_TP1,  0x02222222},
 455         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 456         {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 457         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 458         {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
 459         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 460         {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
 461         {REG_A6XX_RBBM_CLOCK_HYST_TP0,  0x77777777},
 462         {REG_A6XX_RBBM_CLOCK_HYST_TP1,  0x77777777},
 463         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 464         {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 465         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 466         {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
 467         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 468         {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
 469         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 470         {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 471         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 472         {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 473         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 474         {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
 475         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 476         {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
 477         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE,  0x22222222},
 478         {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 479         {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 480         {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 481         {REG_A6XX_RBBM_CLOCK_HYST_UCHE,  0x00000004},
 482         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 483         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 484         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
 485         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002020},
 486         {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
 487         {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
 488         {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
 489         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
 490         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00},
 491         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00},
 492         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00},
 493         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
 494         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 495         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 496         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 497         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 498         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 499         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 500         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 501         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 502         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 503         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 504         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 505         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 506         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 507         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 508         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 509         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 510         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 511         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 512         {},
 513 };
 514
 515 const struct adreno_reglist a630_hwcg[] = {
 516         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
 517         {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
 518         {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
 519         {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
 520         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
 521         {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
 522         {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
 523         {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
 524         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 525         {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
 526         {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
 527         {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
 528         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
 529         {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
 530         {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
 531         {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
 532         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
 533         {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
 534         {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
 535         {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
 536         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 537         {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
 538         {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
 539         {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
 540         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 541         {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
 542         {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
 543         {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
 544         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 545         {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
 546         {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
 547         {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
 548         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 549         {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
 550         {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
 551         {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
 552         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 553         {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
 554         {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
 555         {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
 556         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 557         {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
 558         {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
 559         {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
 560         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 561         {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
 562         {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
 563         {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
 564         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 565         {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
 566         {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
 567         {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
 568         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 569         {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
 570         {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
 571         {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
 572         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 573         {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
 574         {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
 575         {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
 576         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 577         {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
 578         {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
 579         {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
 580         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 581         {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
 582         {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
 583         {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
 584         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 585         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 586         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 587         {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
 588         {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
 589         {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
 590         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
 591         {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
 592         {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
 593         {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
 594         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 595         {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
 596         {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
 597         {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
 598         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
 599         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
 600         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
 601         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
 602         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
 603         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 604         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 605         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 606         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 607         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 608         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 609         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 610         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 611         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 612         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 613         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 614         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 615         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 616         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 617         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 618         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 619         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 620         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 621         {},
 622 };
 623
 624 const struct adreno_reglist a640_hwcg[] = {
 625         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 626         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 627         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 628         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 629         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
 630         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 631         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 632         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 633         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 634         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 635         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 636         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 637         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 638         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 639         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 640         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 641         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 642         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
 643         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 644         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
 645         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022},
 646         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 647         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 648         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 649         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 650         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 651         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 652         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 653         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 654         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 655         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 656         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 657         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 658         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 659         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 660         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 661         {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
 662         {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
 663         {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
 664         {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
 665         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 666         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 667         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 668         {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
 669         {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
 670         {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
 671         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 672         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 673         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 674         {},
 675 };
 676
 677 const struct adreno_reglist a650_hwcg[] = {
 678         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 679         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 680         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 681         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 682         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
 683         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 684         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 685         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 686         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 687         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 688         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 689         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 690         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 691         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 692         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 693         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 694         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 695         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
 696         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 697         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
 698         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
 699         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 700         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 701         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 702         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 703         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 704         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 705         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 706         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 707         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 708         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 709         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 710         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 711         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 712         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 713         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 714         {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
 715         {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
 716         {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
 717         {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777},
 718         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 719         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 720         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 721         {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
 722         {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
 723         {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
 724         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 725         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 726         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 727         {},
 728 };
 729
 730 const struct adreno_reglist a660_hwcg[] = {
 731         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 732         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 733         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 734         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 735         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 736         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 737         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 738         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 739         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 740         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 741         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 742         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 743         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 744         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 745         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 746         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 747         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 748         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
 749         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 750         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
 751         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
 752         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 753         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 754         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 755         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 756         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 757         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 758         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 759         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 760         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 761         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 762         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 763         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 764         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 765         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 766         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 767         {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
 768         {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
 769         {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
 770         {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
 771         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 772         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 773         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 774         {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
 775         {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
 776         {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
 777         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 778         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 779         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 780         {},
 781 };
 782
 783 const struct adreno_reglist a690_hwcg[] = {
 784         {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
 785         {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
 786         {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
 787         {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
 788         {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
 789         {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
 790         {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
 791         {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
 792         {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
 793         {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
 794         {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
 795         {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
 796         {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
 797         {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
 798         {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
 799         {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
 800         {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
 801         {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
 802         {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
 803         {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
 804         {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
 805         {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
 806         {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
 807         {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
 808         {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
 809         {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
 810         {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
 811         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
 812         {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
 813         {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
 814         {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
 815         {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
 816         {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
 817         {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
 818         {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
 819         {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
 820         {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
 821         {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
 822         {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
 823         {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
 824         {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
 825         {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
 826         {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
 827         {REG_A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82},
 828         {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
 829         {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
 830         {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
 831         {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
 832         {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
 833         {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
 834         {REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x20200},
 835         {REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x10111},
 836         {REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x5555},
 837         {}
 838 };
 839
 840 const struct adreno_reglist a702_hwcg[] = {
 841         { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222 },
 842         { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220 },
 843         { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081 },
 844         { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf },
 845         { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222 },
 846         { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
 847         { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
 848         { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222 },
 849         { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
 850         { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
 851         { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
 852         { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
 853         { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
 854         { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
 855         { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
 856         { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
 857         { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
 858         { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222 },
 859         { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
 860         { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00 },
 861         { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022 },
 862         { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555 },
 863         { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
 864         { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044 },
 865         { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
 866         { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 },
 867         { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222 },
 868         { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 },
 869         { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
 870         { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 },
 871         { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 },
 872         { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
 873         { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
 874         { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
 875         { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
 876         { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
 877         { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
 878         { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
 879         { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 },
 880         { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 },
 881         { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
 882         { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
 883         { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
 884         { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
 885         { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
 886         { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
 887         { REG_A6XX_RBBM_CLOCK_CNTL_FCHE, 0x00000222 },
 888         { REG_A6XX_RBBM_CLOCK_DELAY_FCHE, 0x00000000 },
 889         { REG_A6XX_RBBM_CLOCK_HYST_FCHE, 0x00000000 },
 890         { REG_A6XX_RBBM_CLOCK_CNTL_GLC, 0x00222222 },
 891         { REG_A6XX_RBBM_CLOCK_DELAY_GLC, 0x00000000 },
 892         { REG_A6XX_RBBM_CLOCK_HYST_GLC, 0x00000000 },
 893         { REG_A6XX_RBBM_CLOCK_CNTL_MHUB, 0x00000002 },
 894         { REG_A6XX_RBBM_CLOCK_DELAY_MHUB, 0x00000000 },
 895         { REG_A6XX_RBBM_CLOCK_HYST_MHUB, 0x00000000 },
 896         {}
 897 };
 898
 899 const struct adreno_reglist a730_hwcg[] = {
 900         { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
 901         { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022222 },
 902         { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf },
 903         { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
 904         { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
 905         { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
 906         { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
 907         { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
 908         { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
 909         { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
 910         { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
 911         { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
 912         { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
 913         { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
 914         { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
 915         { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
 916         { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
 917         { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 },
 918         { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 },
 919         { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
 920         { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
 921         { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
 922         { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
 923         { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
 924         { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
 925         { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
 926         { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
 927         { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
 928         { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
 929         { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
 930         { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
 931         { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 },
 932         { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
 933         { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
 934         { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
 935         { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
 936         { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
 937         { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 },
 938         { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
 939         { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 },
 940         { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
 941         { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
 942         { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
 943         { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 },
 944         { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
 945         { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000223 },
 946         { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
 947         { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
 948         { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
 949         { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
 950         { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
 951         { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
 952         { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
 953         {},
 954 };
 955
 956 const struct adreno_reglist a740_hwcg[] = {
 957         { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
 958         { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x22022222 },
 959         { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x003cf3cf },
 960         { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
 961         { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
 962         { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
 963         { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
 964         { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
 965         { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
 966         { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
 967         { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
 968         { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
 969         { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
 970         { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
 971         { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
 972         { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
 973         { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
 974         { REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x00222222 },
 975         { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000444 },
 976         { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000222 },
 977         { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
 978         { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
 979         { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
 980         { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
 981         { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
 982         { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
 983         { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
 984         { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
 985         { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
 986         { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
 987         { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
 988         { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
 989         { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00222222 },
 990         { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
 991         { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
 992         { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
 993         { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
 994         { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
 995         { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000000 },
 996         { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
 997         { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00000000 },
 998         { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
 999         { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
1000         { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
1001         { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
1002         { REG_A7XX_RBBM_CLOCK_HYST2_VFD, 0x00000000 },
1003         { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000222 },
1004         { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
1005         { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
1006         { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
1007         { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
1008         { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
1009         { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
1010         { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
1011         {},
1012 };
1013
1014 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
1015 {
1016         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1017         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1018         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1019         const struct adreno_reglist *reg;
1020         unsigned int i;
1021         u32 val, clock_cntl_on, cgc_mode;
1022
1023         if (!(adreno_gpu->info->hwcg || adreno_is_a7xx(adreno_gpu)))
1024                 return;
1025
1026         if (adreno_is_a630(adreno_gpu))
1027                 clock_cntl_on = 0x8aa8aa02;
1028         else if (adreno_is_a610(adreno_gpu))
1029                 clock_cntl_on = 0xaaa8aa82;
1030         else if (adreno_is_a702(adreno_gpu))
1031                 clock_cntl_on = 0xaaaaaa82;
1032         else
1033                 clock_cntl_on = 0x8aa8aa82;
1034
1035         if (adreno_is_a7xx(adreno_gpu)) {
1036                 cgc_mode = adreno_is_a740_family(adreno_gpu) ? 0x20222 : 0x20000;
1037
1038                 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL,
1039                           state ? cgc_mode : 0);
1040                 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL,
1041                           state ? 0x10111 : 0);
1042                 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL,
1043                           state ? 0x5555 : 0);
1044         }
1045
1046         if (!adreno_gpu->info->hwcg) {
1047                 gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 1);
1048                 gpu_write(gpu, REG_A7XX_RBBM_CGC_GLOBAL_LOAD_CMD, state ? 1 : 0);
1049
1050                 if (state) {
1051                         gpu_write(gpu, REG_A7XX_RBBM_CGC_P2S_TRIG_CMD, 1);
1052
1053                         if (gpu_poll_timeout(gpu, REG_A7XX_RBBM_CGC_P2S_STATUS, val,
1054                                              val & A7XX_RBBM_CGC_P2S_STATUS_TXDONE, 1, 10)) {
1055                                 dev_err(&gpu->pdev->dev, "RBBM_CGC_P2S_STATUS TXDONE Poll failed\n");
1056                                 return;
1057                         }
1058
1059                         gpu_write(gpu, REG_A7XX_RBBM_CLOCK_CNTL_GLOBAL, 0);
1060                 }
1061
1062                 return;
1063         }
1064
1065         val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
1066
1067         /* Don't re-program the registers if they are already correct */
1068         if ((!state && !val) || (state && (val == clock_cntl_on)))
1069                 return;
1070
1071         /* Disable SP clock before programming HWCG registers */
1072         if (!adreno_is_a610_family(adreno_gpu) && !adreno_is_a7xx(adreno_gpu))
1073                 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
1074
1075         for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++)
1076                 gpu_write(gpu, reg->offset, state ? reg->value : 0);
1077
1078         /* Enable SP clock */
1079         if (!adreno_is_a610_family(adreno_gpu) && !adreno_is_a7xx(adreno_gpu))
1080                 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
1081
1082         gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0);
1083 }
1084
1085 /* For a615, a616, a618, a619, a630, a640 and a680 */
1086 static const u32 a6xx_protect[] = {
1087         A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1088         A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1089         A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1090         A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1091         A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1092         A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1093         A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1094         A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1095         A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1096         A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1097         A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1098         A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1099         A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1100         A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1101         A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1102         A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1103         A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1104         A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1105         A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1106         A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1107         A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1108         A6XX_PROTECT_NORDWR(0x09e70, 0x0001),
1109         A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1110         A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1111         A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1112         A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1113         A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1114         A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1115         A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1116         A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1117         A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1118         A6XX_PROTECT_NORDWR(0x11c00, 0x0000), /* note: infinite range */
1119 };
1120
1121 /* These are for a620 and a650 */
1122 static const u32 a650_protect[] = {
1123         A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1124         A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1125         A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1126         A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1127         A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1128         A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1129         A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1130         A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1131         A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1132         A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1133         A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1134         A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1135         A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1136         A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1137         A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1138         A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1139         A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1140         A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1141         A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1142         A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1143         A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1144         A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1145         A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1146         A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1147         A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1148         A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1149         A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1150         A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1151         A6XX_PROTECT_NORDWR(0x0b608, 0x0007),
1152         A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1153         A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1154         A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1155         A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1156         A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1157         A6XX_PROTECT_NORDWR(0x1a800, 0x1fff),
1158         A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1159         A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1160         A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1161         A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1162 };
1163
1164 /* These are for a635 and a660 */
1165 static const u32 a660_protect[] = {
1166         A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1167         A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1168         A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1169         A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1170         A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1171         A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1172         A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1173         A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1174         A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1175         A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1176         A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1177         A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1178         A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1179         A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1180         A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1181         A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1182         A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1183         A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1184         A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1185         A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1186         A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1187         A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1188         A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1189         A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1190         A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1191         A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1192         A6XX_PROTECT_NORDWR(0x0ae50, 0x012f),
1193         A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1194         A6XX_PROTECT_NORDWR(0x0b608, 0x0006),
1195         A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1196         A6XX_PROTECT_NORDWR(0x0be20, 0x015f),
1197         A6XX_PROTECT_NORDWR(0x0d000, 0x05ff),
1198         A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1199         A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1200         A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1201         A6XX_PROTECT_NORDWR(0x1a400, 0x1fff),
1202         A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1203         A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1204         A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1205         A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1206         A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1207 };
1208
1209 /* These are for a690 */
1210 static const u32 a690_protect[] = {
1211         A6XX_PROTECT_RDONLY(0x00000, 0x004ff),
1212         A6XX_PROTECT_RDONLY(0x00501, 0x00001),
1213         A6XX_PROTECT_RDONLY(0x0050b, 0x002f4),
1214         A6XX_PROTECT_NORDWR(0x0050e, 0x00000),
1215         A6XX_PROTECT_NORDWR(0x00510, 0x00000),
1216         A6XX_PROTECT_NORDWR(0x00534, 0x00000),
1217         A6XX_PROTECT_NORDWR(0x00800, 0x00082),
1218         A6XX_PROTECT_NORDWR(0x008a0, 0x00008),
1219         A6XX_PROTECT_NORDWR(0x008ab, 0x00024),
1220         A6XX_PROTECT_RDONLY(0x008de, 0x000ae),
1221         A6XX_PROTECT_NORDWR(0x00900, 0x0004d),
1222         A6XX_PROTECT_NORDWR(0x0098d, 0x00272),
1223         A6XX_PROTECT_NORDWR(0x00e00, 0x00001),
1224         A6XX_PROTECT_NORDWR(0x00e03, 0x0000c),
1225         A6XX_PROTECT_NORDWR(0x03c00, 0x000c3),
1226         A6XX_PROTECT_RDONLY(0x03cc4, 0x01fff),
1227         A6XX_PROTECT_NORDWR(0x08630, 0x001cf),
1228         A6XX_PROTECT_NORDWR(0x08e00, 0x00000),
1229         A6XX_PROTECT_NORDWR(0x08e08, 0x00007),
1230         A6XX_PROTECT_NORDWR(0x08e50, 0x0001f),
1231         A6XX_PROTECT_NORDWR(0x08e80, 0x0027f),
1232         A6XX_PROTECT_NORDWR(0x09624, 0x001db),
1233         A6XX_PROTECT_NORDWR(0x09e60, 0x00011),
1234         A6XX_PROTECT_NORDWR(0x09e78, 0x00187),
1235         A6XX_PROTECT_NORDWR(0x0a630, 0x001cf),
1236         A6XX_PROTECT_NORDWR(0x0ae02, 0x00000),
1237         A6XX_PROTECT_NORDWR(0x0ae50, 0x0012f),
1238         A6XX_PROTECT_NORDWR(0x0b604, 0x00000),
1239         A6XX_PROTECT_NORDWR(0x0b608, 0x00006),
1240         A6XX_PROTECT_NORDWR(0x0be02, 0x00001),
1241         A6XX_PROTECT_NORDWR(0x0be20, 0x0015f),
1242         A6XX_PROTECT_NORDWR(0x0d000, 0x005ff),
1243         A6XX_PROTECT_NORDWR(0x0f000, 0x00bff),
1244         A6XX_PROTECT_RDONLY(0x0fc00, 0x01fff),
1245         A6XX_PROTECT_NORDWR(0x11c00, 0x00000), /*note: infiite range */
1246 };
1247
1248 static const u32 a730_protect[] = {
1249         A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1250         A6XX_PROTECT_RDONLY(0x0050b, 0x0058),
1251         A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1252         A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1253         A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1254         A6XX_PROTECT_RDONLY(0x005fb, 0x009d),
1255         A6XX_PROTECT_NORDWR(0x00699, 0x01e9),
1256         A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1257         A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1258         /* 0x008d0-0x008dd are unprotected on purpose for tools like perfetto */
1259         A6XX_PROTECT_RDONLY(0x008de, 0x0154),
1260         A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1261         A6XX_PROTECT_NORDWR(0x0098d, 0x00b2),
1262         A6XX_PROTECT_NORDWR(0x00a41, 0x01be),
1263         A6XX_PROTECT_NORDWR(0x00df0, 0x0001),
1264         A6XX_PROTECT_NORDWR(0x00e01, 0x0000),
1265         A6XX_PROTECT_NORDWR(0x00e07, 0x0008),
1266         A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1267         A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1268         A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1269         A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1270         A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1271         A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1272         A6XX_PROTECT_NORDWR(0x08e80, 0x0280),
1273         A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1274         A6XX_PROTECT_NORDWR(0x09e40, 0x0000),
1275         A6XX_PROTECT_NORDWR(0x09e64, 0x000d),
1276         A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1277         A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1278         A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1279         A6XX_PROTECT_NORDWR(0x0ae50, 0x000f),
1280         A6XX_PROTECT_NORDWR(0x0ae66, 0x0003),
1281         A6XX_PROTECT_NORDWR(0x0ae6f, 0x0003),
1282         A6XX_PROTECT_NORDWR(0x0b604, 0x0003),
1283         A6XX_PROTECT_NORDWR(0x0ec00, 0x0fff),
1284         A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1285         A6XX_PROTECT_NORDWR(0x18400, 0x0053),
1286         A6XX_PROTECT_RDONLY(0x18454, 0x0004),
1287         A6XX_PROTECT_NORDWR(0x18459, 0x1fff),
1288         A6XX_PROTECT_NORDWR(0x1a459, 0x1fff),
1289         A6XX_PROTECT_NORDWR(0x1c459, 0x1fff),
1290         A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1291         A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1292         A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1293         A6XX_PROTECT_NORDWR(0x1f878, 0x002a),
1294         /* CP_PROTECT_REG[44, 46] are left untouched! */
1295         0,
1296         0,
1297         0,
1298         A6XX_PROTECT_NORDWR(0x1f8c0, 0x00000),
1299 };
1300
1301 static void a6xx_set_cp_protect(struct msm_gpu *gpu)
1302 {
1303         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1304         const u32 *regs = a6xx_protect;
1305         unsigned i, count, count_max;
1306
1307         if (adreno_is_a650(adreno_gpu) || adreno_is_a702(adreno_gpu)) {
1308                 regs = a650_protect;
1309                 count = ARRAY_SIZE(a650_protect);
1310                 count_max = 48;
1311                 BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
1312         } else if (adreno_is_a690(adreno_gpu)) {
1313                 regs = a690_protect;
1314                 count = ARRAY_SIZE(a690_protect);
1315                 count_max = 48;
1316                 BUILD_BUG_ON(ARRAY_SIZE(a690_protect) > 48);
1317         } else if (adreno_is_a660_family(adreno_gpu)) {
1318                 regs = a660_protect;
1319                 count = ARRAY_SIZE(a660_protect);
1320                 count_max = 48;
1321                 BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48);
1322         } else if (adreno_is_a730(adreno_gpu) ||
1323                    adreno_is_a740(adreno_gpu) ||
1324                    adreno_is_a750(adreno_gpu)) {
1325                 regs = a730_protect;
1326                 count = ARRAY_SIZE(a730_protect);
1327                 count_max = 48;
1328                 BUILD_BUG_ON(ARRAY_SIZE(a730_protect) > 48);
1329         } else {
1330                 regs = a6xx_protect;
1331                 count = ARRAY_SIZE(a6xx_protect);
1332                 count_max = 32;
1333                 BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
1334         }
1335
1336         /*
1337          * Enable access protection to privileged registers, fault on an access
1338          * protect violation and select the last span to protect from the start
1339          * address all the way to the end of the register address space
1340          */
1341         gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL,
1342                   A6XX_CP_PROTECT_CNTL_ACCESS_PROT_EN |
1343                   A6XX_CP_PROTECT_CNTL_ACCESS_FAULT_ON_VIOL_EN |
1344                   A6XX_CP_PROTECT_CNTL_LAST_SPAN_INF_RANGE);
1345
1346         for (i = 0; i < count - 1; i++) {
1347                 /* Intentionally skip writing to some registers */
1348                 if (regs[i])
1349                         gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]);
1350         }
1351         /* last CP_PROTECT to have "infinite" length on the last entry */
1352         gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]);
1353 }
1354
1355 static void a6xx_calc_ubwc_config(struct adreno_gpu *gpu)
1356 {
1357         /* Unknown, introduced with A650 family, related to UBWC mode/ver 4 */
1358         gpu->ubwc_config.rgb565_predicator = 0;
1359         /* Unknown, introduced with A650 family */
1360         gpu->ubwc_config.uavflagprd_inv = 0;
1361         /* Whether the minimum access length is 64 bits */
1362         gpu->ubwc_config.min_acc_len = 0;
1363         /* Entirely magic, per-GPU-gen value */
1364         gpu->ubwc_config.ubwc_mode = 0;
1365         /*
1366          * The Highest Bank Bit value represents the bit of the highest DDR bank.
1367          * This should ideally use DRAM type detection.
1368          */
1369         gpu->ubwc_config.highest_bank_bit = 15;
1370
1371         if (adreno_is_a610(gpu)) {
1372                 gpu->ubwc_config.highest_bank_bit = 13;
1373                 gpu->ubwc_config.min_acc_len = 1;
1374                 gpu->ubwc_config.ubwc_mode = 1;
1375         }
1376
1377         if (adreno_is_a618(gpu))
1378                 gpu->ubwc_config.highest_bank_bit = 14;
1379
1380         if (adreno_is_a619_holi(gpu))
1381                 gpu->ubwc_config.highest_bank_bit = 13;
1382
1383         if (adreno_is_a640_family(gpu))
1384                 gpu->ubwc_config.amsbc = 1;
1385
1386         if (adreno_is_a650(gpu) ||
1387             adreno_is_a660(gpu) ||
1388             adreno_is_a690(gpu) ||
1389             adreno_is_a730(gpu) ||
1390             adreno_is_a740_family(gpu)) {
1391                 /* TODO: get ddr type from bootloader and use 2 for LPDDR4 */
1392                 gpu->ubwc_config.highest_bank_bit = 16;
1393                 gpu->ubwc_config.amsbc = 1;
1394                 gpu->ubwc_config.rgb565_predicator = 1;
1395                 gpu->ubwc_config.uavflagprd_inv = 2;
1396         }
1397
1398         if (adreno_is_7c3(gpu)) {
1399                 gpu->ubwc_config.highest_bank_bit = 14;
1400                 gpu->ubwc_config.amsbc = 1;
1401                 gpu->ubwc_config.rgb565_predicator = 1;
1402                 gpu->ubwc_config.uavflagprd_inv = 2;
1403         }
1404
1405         if (adreno_is_a702(gpu)) {
1406                 gpu->ubwc_config.highest_bank_bit = 14;
1407                 gpu->ubwc_config.min_acc_len = 1;
1408                 gpu->ubwc_config.ubwc_mode = 2;
1409         }
1410 }
1411
1412 static void a6xx_set_ubwc_config(struct msm_gpu *gpu)
1413 {
1414         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1415         /*
1416          * We subtract 13 from the highest bank bit (13 is the minimum value
1417          * allowed by hw) and write the lowest two bits of the remaining value
1418          * as hbb_lo and the one above it as hbb_hi to the hardware.
1419          */
1420         BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13);
1421         u32 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13;
1422         u32 hbb_hi = hbb >> 2;
1423         u32 hbb_lo = hbb & 3;
1424
1425         gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL,
1426                   adreno_gpu->ubwc_config.rgb565_predicator << 11 |
1427                   hbb_hi << 10 | adreno_gpu->ubwc_config.amsbc << 4 |
1428                   adreno_gpu->ubwc_config.min_acc_len << 3 |
1429                   hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1430
1431         gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, hbb_hi << 4 |
1432                   adreno_gpu->ubwc_config.min_acc_len << 3 |
1433                   hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1434
1435         gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, hbb_hi << 10 |
1436                   adreno_gpu->ubwc_config.uavflagprd_inv << 4 |
1437                   adreno_gpu->ubwc_config.min_acc_len << 3 |
1438                   hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1439
1440         if (adreno_is_a7xx(adreno_gpu))
1441                 gpu_write(gpu, REG_A7XX_GRAS_NC_MODE_CNTL,
1442                           FIELD_PREP(GENMASK(8, 5), hbb_lo));
1443
1444         gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL,
1445                   adreno_gpu->ubwc_config.min_acc_len << 23 | hbb_lo << 21);
1446 }
1447
1448 static int a6xx_cp_init(struct msm_gpu *gpu)
1449 {
1450         struct msm_ringbuffer *ring = gpu->rb[0];
1451
1452         OUT_PKT7(ring, CP_ME_INIT, 8);
1453
1454         OUT_RING(ring, 0x0000002f);
1455
1456         /* Enable multiple hardware contexts */
1457         OUT_RING(ring, 0x00000003);
1458
1459         /* Enable error detection */
1460         OUT_RING(ring, 0x20000000);
1461
1462         /* Don't enable header dump */
1463         OUT_RING(ring, 0x00000000);
1464         OUT_RING(ring, 0x00000000);
1465
1466         /* No workarounds enabled */
1467         OUT_RING(ring, 0x00000000);
1468
1469         /* Pad rest of the cmds with 0's */
1470         OUT_RING(ring, 0x00000000);
1471         OUT_RING(ring, 0x00000000);
1472
1473         a6xx_flush(gpu, ring);
1474         return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1475 }
1476
1477 static int a7xx_cp_init(struct msm_gpu *gpu)
1478 {
1479         struct msm_ringbuffer *ring = gpu->rb[0];
1480         u32 mask;
1481
1482         /* Disable concurrent binning before sending CP init */
1483         OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
1484         OUT_RING(ring, BIT(27));
1485
1486         OUT_PKT7(ring, CP_ME_INIT, 7);
1487
1488         /* Use multiple HW contexts */
1489         mask = BIT(0);
1490
1491         /* Enable error detection */
1492         mask |= BIT(1);
1493
1494         /* Set default reset state */
1495         mask |= BIT(3);
1496
1497         /* Disable save/restore of performance counters across preemption */
1498         mask |= BIT(6);
1499
1500         /* Enable the register init list with the spinlock */
1501         mask |= BIT(8);
1502
1503         OUT_RING(ring, mask);
1504
1505         /* Enable multiple hardware contexts */
1506         OUT_RING(ring, 0x00000003);
1507
1508         /* Enable error detection */
1509         OUT_RING(ring, 0x20000000);
1510
1511         /* Operation mode mask */
1512         OUT_RING(ring, 0x00000002);
1513
1514         /* *Don't* send a power up reg list for concurrent binning (TODO) */
1515         /* Lo address */
1516         OUT_RING(ring, 0x00000000);
1517         /* Hi address */
1518         OUT_RING(ring, 0x00000000);
1519         /* BIT(31) set => read the regs from the list */
1520         OUT_RING(ring, 0x00000000);
1521
1522         a6xx_flush(gpu, ring);
1523         return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1524 }
1525
1526 /*
1527  * Check that the microcode version is new enough to include several key
1528  * security fixes. Return true if the ucode is safe.
1529  */
1530 static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,
1531                 struct drm_gem_object *obj)
1532 {
1533         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1534         struct msm_gpu *gpu = &adreno_gpu->base;
1535         const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE];
1536         u32 *buf = msm_gem_get_vaddr(obj);
1537         bool ret = false;
1538
1539         if (IS_ERR(buf))
1540                 return false;
1541
1542         /* A7xx is safe! */
1543         if (adreno_is_a7xx(adreno_gpu) || adreno_is_a702(adreno_gpu))
1544                 return true;
1545
1546         /*
1547          * Targets up to a640 (a618, a630 and a640) need to check for a
1548          * microcode version that is patched to support the whereami opcode or
1549          * one that is new enough to include it by default.
1550          *
1551          * a650 tier targets don't need whereami but still need to be
1552          * equal to or newer than 0.95 for other security fixes
1553          *
1554          * a660 targets have all the critical security fixes from the start
1555          */
1556         if (!strcmp(sqe_name, "a630_sqe.fw")) {
1557                 /*
1558                  * If the lowest nibble is 0xa that is an indication that this
1559                  * microcode has been patched. The actual version is in dword
1560                  * [3] but we only care about the patchlevel which is the lowest
1561                  * nibble of dword [3]
1562                  *
1563                  * Otherwise check that the firmware is greater than or equal
1564                  * to 1.90 which was the first version that had this fix built
1565                  * in
1566                  */
1567                 if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) ||
1568                         (buf[0] & 0xfff) >= 0x190) {
1569                         a6xx_gpu->has_whereami = true;
1570                         ret = true;
1571                         goto out;
1572                 }
1573
1574                 DRM_DEV_ERROR(&gpu->pdev->dev,
1575                         "a630 SQE ucode is too old. Have version %x need at least %x\n",
1576                         buf[0] & 0xfff, 0x190);
1577         } else if (!strcmp(sqe_name, "a650_sqe.fw")) {
1578                 if ((buf[0] & 0xfff) >= 0x095) {
1579                         ret = true;
1580                         goto out;
1581                 }
1582
1583                 DRM_DEV_ERROR(&gpu->pdev->dev,
1584                         "a650 SQE ucode is too old. Have version %x need at least %x\n",
1585                         buf[0] & 0xfff, 0x095);
1586         } else if (!strcmp(sqe_name, "a660_sqe.fw")) {
1587                 ret = true;
1588         } else {
1589                 DRM_DEV_ERROR(&gpu->pdev->dev,
1590                         "unknown GPU, add it to a6xx_ucode_check_version()!!\n");
1591         }
1592 out:
1593         msm_gem_put_vaddr(obj);
1594         return ret;
1595 }
1596
1597 static int a6xx_ucode_load(struct msm_gpu *gpu)
1598 {
1599         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1600         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1601
1602         if (!a6xx_gpu->sqe_bo) {
1603                 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
1604                         adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
1605
1606                 if (IS_ERR(a6xx_gpu->sqe_bo)) {
1607                         int ret = PTR_ERR(a6xx_gpu->sqe_bo);
1608
1609                         a6xx_gpu->sqe_bo = NULL;
1610                         DRM_DEV_ERROR(&gpu->pdev->dev,
1611                                 "Could not allocate SQE ucode: %d\n", ret);
1612
1613                         return ret;
1614                 }
1615
1616                 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");
1617                 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) {
1618                         msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
1619                         drm_gem_object_put(a6xx_gpu->sqe_bo);
1620
1621                         a6xx_gpu->sqe_bo = NULL;
1622                         return -EPERM;
1623                 }
1624         }
1625
1626         /*
1627          * Expanded APRIV and targets that support WHERE_AM_I both need a
1628          * privileged buffer to store the RPTR shadow
1629          */
1630         if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) &&
1631             !a6xx_gpu->shadow_bo) {
1632                 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
1633                                                       sizeof(u32) * gpu->nr_rings,
1634                                                       MSM_BO_WC | MSM_BO_MAP_PRIV,
1635                                                       gpu->aspace, &a6xx_gpu->shadow_bo,
1636                                                       &a6xx_gpu->shadow_iova);
1637
1638                 if (IS_ERR(a6xx_gpu->shadow))
1639                         return PTR_ERR(a6xx_gpu->shadow);
1640
1641                 msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
1642         }
1643
1644         return 0;
1645 }
1646
1647 static int a6xx_zap_shader_init(struct msm_gpu *gpu)
1648 {
1649         static bool loaded;
1650         int ret;
1651
1652         if (loaded)
1653                 return 0;
1654
1655         ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
1656
1657         loaded = !ret;
1658         return ret;
1659 }
1660
1661 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1662                        A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1663                        A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1664                        A6XX_RBBM_INT_0_MASK_CP_IB2 | \
1665                        A6XX_RBBM_INT_0_MASK_CP_IB1 | \
1666                        A6XX_RBBM_INT_0_MASK_CP_RB | \
1667                        A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1668                        A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1669                        A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1670                        A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1671                        A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
1672
1673 #define A7XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1674                        A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1675                        A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \
1676                        A6XX_RBBM_INT_0_MASK_CP_SW | \
1677                        A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1678                        A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \
1679                        A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \
1680                        A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1681                        A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1682                        A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1683                        A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1684                        A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \
1685                        A6XX_RBBM_INT_0_MASK_TSBWRITEERROR)
1686
1687 #define A7XX_APRIV_MASK (A6XX_CP_APRIV_CNTL_ICACHE | \
1688                          A6XX_CP_APRIV_CNTL_RBFETCH | \
1689                          A6XX_CP_APRIV_CNTL_RBPRIVLEVEL | \
1690                          A6XX_CP_APRIV_CNTL_RBRPWB)
1691
1692 #define A7XX_BR_APRIVMASK (A7XX_APRIV_MASK | \
1693                            A6XX_CP_APRIV_CNTL_CDREAD | \
1694                            A6XX_CP_APRIV_CNTL_CDWRITE)
1695
1696 static int hw_init(struct msm_gpu *gpu)
1697 {
1698         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1699         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1700         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1701         u64 gmem_range_min;
1702         int ret;
1703
1704         if (!adreno_has_gmu_wrapper(adreno_gpu)) {
1705                 /* Make sure the GMU keeps the GPU on while we set it up */
1706                 ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1707                 if (ret)
1708                         return ret;
1709         }
1710
1711         /* Clear GBIF halt in case GX domain was not collapsed */
1712         if (adreno_is_a619_holi(adreno_gpu)) {
1713                 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1714                 gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, 0);
1715                 /* Let's make extra sure that the GPU can access the memory.. */
1716                 mb();
1717         } else if (a6xx_has_gbif(adreno_gpu)) {
1718                 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1719                 gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0);
1720                 /* Let's make extra sure that the GPU can access the memory.. */
1721                 mb();
1722         }
1723
1724         /* Some GPUs are stubborn and take their sweet time to unhalt GBIF! */
1725         if (adreno_is_a7xx(adreno_gpu) && a6xx_has_gbif(adreno_gpu))
1726                 spin_until(!gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK));
1727
1728         gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
1729
1730         if (adreno_is_a619_holi(adreno_gpu))
1731                 a6xx_sptprac_enable(gmu);
1732
1733         /*
1734          * Disable the trusted memory range - we don't actually supported secure
1735          * memory rendering at this point in time and we don't want to block off
1736          * part of the virtual memory space.
1737          */
1738         gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000);
1739         gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
1740
1741         if (!adreno_is_a7xx(adreno_gpu)) {
1742                 /* Turn on 64 bit addressing for all blocks */
1743                 gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);
1744                 gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);
1745                 gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);
1746                 gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);
1747                 gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);
1748                 gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);
1749                 gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);
1750                 gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);
1751                 gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);
1752                 gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);
1753                 gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);
1754                 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
1755         }
1756
1757         /* enable hardware clockgating */
1758         a6xx_set_hwcg(gpu, true);
1759
1760         /* VBIF/GBIF start*/
1761         if (adreno_is_a610_family(adreno_gpu) ||
1762             adreno_is_a640_family(adreno_gpu) ||
1763             adreno_is_a650_family(adreno_gpu) ||
1764             adreno_is_a7xx(adreno_gpu)) {
1765                 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
1766                 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
1767                 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
1768                 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
1769                 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL,
1770                           adreno_is_a7xx(adreno_gpu) ? 0x2120212 : 0x3);
1771         } else {
1772                 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
1773         }
1774
1775         if (adreno_is_a630(adreno_gpu))
1776                 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
1777
1778         if (adreno_is_a7xx(adreno_gpu))
1779                 gpu_write(gpu, REG_A6XX_UCHE_GBIF_GX_CONFIG, 0x10240e0);
1780
1781         /* Make all blocks contribute to the GPU BUSY perf counter */
1782         gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
1783
1784         /* Disable L2 bypass in the UCHE */
1785         if (adreno_is_a7xx(adreno_gpu)) {
1786                 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1787                 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1788         } else {
1789                 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, 0x0001ffffffffffc0llu);
1790                 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1791                 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1792         }
1793
1794         if (!(adreno_is_a650_family(adreno_gpu) ||
1795               adreno_is_a702(adreno_gpu) ||
1796               adreno_is_a730(adreno_gpu))) {
1797                 gmem_range_min = adreno_is_a740_family(adreno_gpu) ? SZ_16M : SZ_1M;
1798
1799                 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
1800                 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN, gmem_range_min);
1801
1802                 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX,
1803                         gmem_range_min + adreno_gpu->info->gmem - 1);
1804         }
1805
1806         if (adreno_is_a7xx(adreno_gpu))
1807                 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, BIT(23));
1808         else {
1809                 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
1810                 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
1811         }
1812
1813         if (adreno_is_a640_family(adreno_gpu) || adreno_is_a650_family(adreno_gpu)) {
1814                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
1815                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1816         } else if (adreno_is_a610_family(adreno_gpu)) {
1817                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060);
1818                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16);
1819         } else if (!adreno_is_a7xx(adreno_gpu)) {
1820                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
1821                 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1822         }
1823
1824         if (adreno_is_a660_family(adreno_gpu))
1825                 gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020);
1826
1827         /* Setting the mem pool size */
1828         if (adreno_is_a610(adreno_gpu)) {
1829                 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 48);
1830                 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 47);
1831         } else if (adreno_is_a702(adreno_gpu)) {
1832                 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 64);
1833                 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 63);
1834         } else if (!adreno_is_a7xx(adreno_gpu))
1835                 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
1836
1837         /* Setting the primFifo thresholds default values,
1838          * and vccCacheSkipDis=1 bit (0x200) for A640 and newer
1839         */
1840         if (adreno_is_a702(adreno_gpu))
1841                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x0000c000);
1842         else if (adreno_is_a690(adreno_gpu))
1843                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00800200);
1844         else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1845                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1846         else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu))
1847                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200);
1848         else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1849                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1850         else if (adreno_is_a619(adreno_gpu))
1851                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00018000);
1852         else if (adreno_is_a610(adreno_gpu))
1853                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00080000);
1854         else if (!adreno_is_a7xx(adreno_gpu))
1855                 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000);
1856
1857         /* Set the AHB default slave response to "ERROR" */
1858         gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
1859
1860         /* Turn on performance counters */
1861         gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
1862
1863         if (adreno_is_a7xx(adreno_gpu)) {
1864                 /* Turn on the IFPC counter (countable 4 on XOCLK4) */
1865                 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1,
1866                           FIELD_PREP(GENMASK(7, 0), 0x4));
1867         }
1868
1869         /* Select CP0 to always count cycles */
1870         gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT);
1871
1872         a6xx_set_ubwc_config(gpu);
1873
1874         /* Enable fault detection */
1875         if (adreno_is_a730(adreno_gpu) ||
1876             adreno_is_a740_family(adreno_gpu))
1877                 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0xcfffff);
1878         else if (adreno_is_a690(adreno_gpu))
1879                 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x4fffff);
1880         else if (adreno_is_a619(adreno_gpu))
1881                 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3fffff);
1882         else if (adreno_is_a610(adreno_gpu) || adreno_is_a702(adreno_gpu))
1883                 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3ffff);
1884         else
1885                 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x1fffff);
1886
1887         gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, BIT(7) | 0x1);
1888
1889         /* Set weights for bicubic filtering */
1890         if (adreno_is_a650_family(adreno_gpu)) {
1891                 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
1892                 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
1893                         0x3fe05ff4);
1894                 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
1895                         0x3fa0ebee);
1896                 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
1897                         0x3f5193ed);
1898                 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
1899                         0x3f0243f0);
1900         }
1901
1902         /* Set up the CX GMU counter 0 to count busy ticks */
1903         gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000);
1904
1905         /* Enable the power counter */
1906         gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, BIT(5));
1907         gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
1908
1909         /* Protect registers from the CP */
1910         a6xx_set_cp_protect(gpu);
1911
1912         if (adreno_is_a660_family(adreno_gpu)) {
1913                 if (adreno_is_a690(adreno_gpu))
1914                         gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x00028801);
1915                 else
1916                         gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1);
1917                 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0);
1918         } else if (adreno_is_a702(adreno_gpu)) {
1919                 /* Something to do with the HLSQ cluster */
1920                 gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, BIT(24));
1921         }
1922
1923         if (adreno_is_a690(adreno_gpu))
1924                 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x90);
1925         /* Set dualQ + disable afull for A660 GPU */
1926         else if (adreno_is_a660(adreno_gpu))
1927                 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906);
1928         else if (adreno_is_a7xx(adreno_gpu))
1929                 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG,
1930                           FIELD_PREP(GENMASK(19, 16), 6) |
1931                           FIELD_PREP(GENMASK(15, 12), 6) |
1932                           FIELD_PREP(GENMASK(11, 8), 9) |
1933                           BIT(3) | BIT(2) |
1934                           FIELD_PREP(GENMASK(1, 0), 2));
1935
1936         /* Enable expanded apriv for targets that support it */
1937         if (gpu->hw_apriv) {
1938                 if (adreno_is_a7xx(adreno_gpu)) {
1939                         gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1940                                   A7XX_BR_APRIVMASK);
1941                         gpu_write(gpu, REG_A7XX_CP_BV_APRIV_CNTL,
1942                                   A7XX_APRIV_MASK);
1943                         gpu_write(gpu, REG_A7XX_CP_LPAC_APRIV_CNTL,
1944                                   A7XX_APRIV_MASK);
1945                 } else
1946                         gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1947                                   BIT(6) | BIT(5) | BIT(3) | BIT(2) | BIT(1));
1948         }
1949
1950         /* Enable interrupts */
1951         gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK,
1952                   adreno_is_a7xx(adreno_gpu) ? A7XX_INT_MASK : A6XX_INT_MASK);
1953
1954         ret = adreno_hw_init(gpu);
1955         if (ret)
1956                 goto out;
1957
1958         gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
1959
1960         /* Set the ringbuffer address */
1961         gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova);
1962
1963         /* Targets that support extended APRIV can use the RPTR shadow from
1964          * hardware but all the other ones need to disable the feature. Targets
1965          * that support the WHERE_AM_I opcode can use that instead
1966          */
1967         if (adreno_gpu->base.hw_apriv)
1968                 gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
1969         else
1970                 gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
1971                         MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
1972
1973         /* Configure the RPTR shadow if needed: */
1974         if (a6xx_gpu->shadow_bo) {
1975                 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR,
1976                         shadowptr(a6xx_gpu, gpu->rb[0]));
1977         }
1978
1979         /* ..which means "always" on A7xx, also for BV shadow */
1980         if (adreno_is_a7xx(adreno_gpu)) {
1981                 gpu_write64(gpu, REG_A7XX_CP_BV_RB_RPTR_ADDR,
1982                             rbmemptr(gpu->rb[0], bv_fence));
1983         }
1984
1985         /* Always come up on rb 0 */
1986         a6xx_gpu->cur_ring = gpu->rb[0];
1987
1988         gpu->cur_ctx_seqno = 0;
1989
1990         /* Enable the SQE_to start the CP engine */
1991         gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
1992
1993         ret = adreno_is_a7xx(adreno_gpu) ? a7xx_cp_init(gpu) : a6xx_cp_init(gpu);
1994         if (ret)
1995                 goto out;
1996
1997         /*
1998          * Try to load a zap shader into the secure world. If successful
1999          * we can use the CP to switch out of secure mode. If not then we
2000          * have no resource but to try to switch ourselves out manually. If we
2001          * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
2002          * be blocked and a permissions violation will soon follow.
2003          */
2004         ret = a6xx_zap_shader_init(gpu);
2005         if (!ret) {
2006                 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
2007                 OUT_RING(gpu->rb[0], 0x00000000);
2008
2009                 a6xx_flush(gpu, gpu->rb[0]);
2010                 if (!a6xx_idle(gpu, gpu->rb[0]))
2011                         return -EINVAL;
2012         } else if (ret == -ENODEV) {
2013                 /*
2014                  * This device does not use zap shader (but print a warning
2015                  * just in case someone got their dt wrong.. hopefully they
2016                  * have a debug UART to realize the error of their ways...
2017                  * if you mess this up you are about to crash horribly)
2018                  */
2019                 dev_warn_once(gpu->dev->dev,
2020                         "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
2021                 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
2022                 ret = 0;
2023         } else {
2024                 return ret;
2025         }
2026
2027 out:
2028         if (adreno_has_gmu_wrapper(adreno_gpu))
2029                 return ret;
2030         /*
2031          * Tell the GMU that we are done touching the GPU and it can start power
2032          * management
2033          */
2034         a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
2035
2036         if (a6xx_gpu->gmu.legacy) {
2037                 /* Take the GMU out of its special boot mode */
2038                 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
2039         }
2040
2041         return ret;
2042 }
2043
2044 static int a6xx_hw_init(struct msm_gpu *gpu)
2045 {
2046         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2047         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2048         int ret;
2049
2050         mutex_lock(&a6xx_gpu->gmu.lock);
2051         ret = hw_init(gpu);
2052         mutex_unlock(&a6xx_gpu->gmu.lock);
2053
2054         return ret;
2055 }
2056
2057 static void a6xx_dump(struct msm_gpu *gpu)
2058 {
2059         DRM_DEV_INFO(&gpu->pdev->dev, "status:   %08x\n",
2060                         gpu_read(gpu, REG_A6XX_RBBM_STATUS));
2061         adreno_dump(gpu);
2062 }
2063
2064 static void a6xx_recover(struct msm_gpu *gpu)
2065 {
2066         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2067         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2068         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2069         int i, active_submits;
2070
2071         adreno_dump_info(gpu);
2072
2073         for (i = 0; i < 8; i++)
2074                 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
2075                         gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
2076
2077         if (hang_debug)
2078                 a6xx_dump(gpu);
2079
2080         /*
2081          * To handle recovery specific sequences during the rpm suspend we are
2082          * about to trigger
2083          */
2084         a6xx_gpu->hung = true;
2085
2086         /* Halt SQE first */
2087         gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3);
2088
2089         pm_runtime_dont_use_autosuspend(&gpu->pdev->dev);
2090
2091         /* active_submit won't change until we make a submission */
2092         mutex_lock(&gpu->active_lock);
2093         active_submits = gpu->active_submits;
2094
2095         /*
2096          * Temporarily clear active_submits count to silence a WARN() in the
2097          * runtime suspend cb
2098          */
2099         gpu->active_submits = 0;
2100
2101         if (adreno_has_gmu_wrapper(adreno_gpu)) {
2102                 /* Drain the outstanding traffic on memory buses */
2103                 a6xx_bus_clear_pending_transactions(adreno_gpu, true);
2104
2105                 /* Reset the GPU to a clean state */
2106                 a6xx_gpu_sw_reset(gpu, true);
2107                 a6xx_gpu_sw_reset(gpu, false);
2108         }
2109
2110         reinit_completion(&gmu->pd_gate);
2111         dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb);
2112         dev_pm_genpd_synced_poweroff(gmu->cxpd);
2113
2114         /* Drop the rpm refcount from active submits */
2115         if (active_submits)
2116                 pm_runtime_put(&gpu->pdev->dev);
2117
2118         /* And the final one from recover worker */
2119         pm_runtime_put_sync(&gpu->pdev->dev);
2120
2121         if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000)))
2122                 DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n");
2123
2124         dev_pm_genpd_remove_notifier(gmu->cxpd);
2125
2126         pm_runtime_use_autosuspend(&gpu->pdev->dev);
2127
2128         if (active_submits)
2129                 pm_runtime_get(&gpu->pdev->dev);
2130
2131         pm_runtime_get_sync(&gpu->pdev->dev);
2132
2133         gpu->active_submits = active_submits;
2134         mutex_unlock(&gpu->active_lock);
2135
2136         msm_gpu_hw_init(gpu);
2137         a6xx_gpu->hung = false;
2138 }
2139
2140 static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
2141 {
2142         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2143         static const char *uche_clients[7] = {
2144                 "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ",
2145         };
2146         u32 val;
2147
2148         if (adreno_is_a7xx(adreno_gpu)) {
2149                 if (mid != 1 && mid != 2 && mid != 3 && mid != 8)
2150                         return "UNKNOWN";
2151         } else {
2152                 if (mid < 1 || mid > 3)
2153                         return "UNKNOWN";
2154         }
2155
2156         /*
2157          * The source of the data depends on the mid ID read from FSYNR1.
2158          * and the client ID read from the UCHE block
2159          */
2160         val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF);
2161
2162         if (adreno_is_a7xx(adreno_gpu)) {
2163                 /* Bit 3 for mid=3 indicates BR or BV */
2164                 static const char *uche_clients_a7xx[16] = {
2165                         "BR_VFD", "BR_SP", "BR_VSC", "BR_VPC",
2166                         "BR_HLSQ", "BR_PC", "BR_LRZ", "BR_TP",
2167                         "BV_VFD", "BV_SP", "BV_VSC", "BV_VPC",
2168                         "BV_HLSQ", "BV_PC", "BV_LRZ", "BV_TP",
2169                 };
2170
2171                 /* LPAC has the same clients as BR and BV, but because it is
2172                  * compute-only some of them do not exist and there are holes
2173                  * in the array.
2174                  */
2175                 static const char *uche_clients_lpac_a7xx[8] = {
2176                         "-", "LPAC_SP", "-", "-",
2177                         "LPAC_HLSQ", "-", "-", "LPAC_TP",
2178                 };
2179
2180                 val &= GENMASK(6, 0);
2181
2182                 /* mid=3 refers to BR or BV */
2183                 if (mid == 3) {
2184                         if (val < ARRAY_SIZE(uche_clients_a7xx))
2185                                 return uche_clients_a7xx[val];
2186                         else
2187                                 return "UCHE";
2188                 }
2189
2190                 /* mid=8 refers to LPAC */
2191                 if (mid == 8) {
2192                         if (val < ARRAY_SIZE(uche_clients_lpac_a7xx))
2193                                 return uche_clients_lpac_a7xx[val];
2194                         else
2195                                 return "UCHE_LPAC";
2196                 }
2197
2198                 /* mid=2 is a catchall for everything else in LPAC */
2199                 if (mid == 2)
2200                         return "UCHE_LPAC";
2201
2202                 /* mid=1 is a catchall for everything else in BR/BV */
2203                 return "UCHE";
2204         } else if (adreno_is_a660_family(adreno_gpu)) {
2205                 static const char *uche_clients_a660[8] = {
2206                         "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ", "TP",
2207                 };
2208
2209                 static const char *uche_clients_a660_not[8] = {
2210                         "not VFD", "not SP", "not VSC", "not VPC",
2211                         "not HLSQ", "not PC", "not LRZ", "not TP",
2212                 };
2213
2214                 val &= GENMASK(6, 0);
2215
2216                 if (mid == 3 && val < ARRAY_SIZE(uche_clients_a660))
2217                         return uche_clients_a660[val];
2218
2219                 if (mid == 1 && val < ARRAY_SIZE(uche_clients_a660_not))
2220                         return uche_clients_a660_not[val];
2221
2222                 return "UCHE";
2223         } else {
2224                 /* mid = 3 is most precise and refers to only one block per client */
2225                 if (mid == 3)
2226                         return uche_clients[val & 7];
2227
2228                 /* For mid=2 the source is TP or VFD except when the client id is 0 */
2229                 if (mid == 2)
2230                         return ((val & 7) == 0) ? "TP" : "TP|VFD";
2231
2232                 /* For mid=1 just return "UCHE" as a catchall for everything else */
2233                 return "UCHE";
2234         }
2235 }
2236
2237 static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
2238 {
2239         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2240
2241         if (id == 0)
2242                 return "CP";
2243         else if (id == 4)
2244                 return "CCU";
2245         else if (id == 6)
2246                 return "CDP Prefetch";
2247         else if (id == 7)
2248                 return "GMU";
2249         else if (id == 5 && adreno_is_a7xx(adreno_gpu))
2250                 return "Flag cache";
2251
2252         return a6xx_uche_fault_block(gpu, id);
2253 }
2254
2255 static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
2256 {
2257         struct msm_gpu *gpu = arg;
2258         struct adreno_smmu_fault_info *info = data;
2259         const char *block = "unknown";
2260
2261         u32 scratch[] = {
2262                         gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
2263                         gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
2264                         gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
2265                         gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)),
2266         };
2267
2268         if (info)
2269                 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
2270
2271         return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
2272 }
2273
2274 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
2275 {
2276         u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
2277
2278         if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
2279                 u32 val;
2280
2281                 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
2282                 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
2283                 dev_err_ratelimited(&gpu->pdev->dev,
2284                         "CP | opcode error | possible opcode=0x%8.8X\n",
2285                         val);
2286         }
2287
2288         if (status & A6XX_CP_INT_CP_UCODE_ERROR)
2289                 dev_err_ratelimited(&gpu->pdev->dev,
2290                         "CP ucode error interrupt\n");
2291
2292         if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
2293                 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
2294                         gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
2295
2296         if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
2297                 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
2298
2299                 dev_err_ratelimited(&gpu->pdev->dev,
2300                         "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
2301                         val & (1 << 20) ? "READ" : "WRITE",
2302                         (val & 0x3ffff), val);
2303         }
2304
2305         if (status & A6XX_CP_INT_CP_AHB_ERROR && !adreno_is_a7xx(to_adreno_gpu(gpu)))
2306                 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
2307
2308         if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
2309                 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
2310
2311         if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
2312                 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
2313
2314 }
2315
2316 static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
2317 {
2318         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2319         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2320         struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
2321
2322         /*
2323          * If stalled on SMMU fault, we could trip the GPU's hang detection,
2324          * but the fault handler will trigger the devcore dump, and we want
2325          * to otherwise resume normally rather than killing the submit, so
2326          * just bail.
2327          */
2328         if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT)
2329                 return;
2330
2331         /*
2332          * Force the GPU to stay on until after we finish
2333          * collecting information
2334          */
2335         if (!adreno_has_gmu_wrapper(adreno_gpu))
2336                 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
2337
2338         DRM_DEV_ERROR(&gpu->pdev->dev,
2339                 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
2340                 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
2341                 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
2342                 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
2343                 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
2344                 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2345                 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2346                 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2347                 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
2348
2349         /* Turn off the hangcheck timer to keep it from bothering us */
2350         del_timer(&gpu->hangcheck_timer);
2351
2352         kthread_queue_work(gpu->worker, &gpu->recover_work);
2353 }
2354
2355 static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
2356 {
2357         struct msm_drm_private *priv = gpu->dev->dev_private;
2358         u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
2359
2360         gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
2361
2362         if (priv->disable_err_irq)
2363                 status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
2364
2365         if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
2366                 a6xx_fault_detect_irq(gpu);
2367
2368         if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
2369                 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
2370
2371         if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
2372                 a6xx_cp_hw_err_irq(gpu);
2373
2374         if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
2375                 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
2376
2377         if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
2378                 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
2379
2380         if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
2381                 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
2382
2383         if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
2384                 msm_gpu_retire(gpu);
2385
2386         return IRQ_HANDLED;
2387 }
2388
2389 static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)
2390 {
2391         llcc_slice_deactivate(a6xx_gpu->llc_slice);
2392         llcc_slice_deactivate(a6xx_gpu->htw_llc_slice);
2393 }
2394
2395 static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2396 {
2397         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2398         struct msm_gpu *gpu = &adreno_gpu->base;
2399         u32 cntl1_regval = 0;
2400
2401         if (IS_ERR(a6xx_gpu->llc_mmio))
2402                 return;
2403
2404         if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
2405                 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
2406
2407                 gpu_scid &= 0x1f;
2408                 cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
2409                                (gpu_scid << 15) | (gpu_scid << 20);
2410
2411                 /* On A660, the SCID programming for UCHE traffic is done in
2412                  * A6XX_GBIF_SCACHE_CNTL0[14:10]
2413                  */
2414                 if (adreno_is_a660_family(adreno_gpu))
2415                         gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
2416                                 (1 << 8), (gpu_scid << 10) | (1 << 8));
2417         }
2418
2419         /*
2420          * For targets with a MMU500, activate the slice but don't program the
2421          * register.  The XBL will take care of that.
2422          */
2423         if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) {
2424                 if (!a6xx_gpu->have_mmu500) {
2425                         u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice);
2426
2427                         gpuhtw_scid &= 0x1f;
2428                         cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid);
2429                 }
2430         }
2431
2432         if (!cntl1_regval)
2433                 return;
2434
2435         /*
2436          * Program the slice IDs for the various GPU blocks and GPU MMU
2437          * pagetables
2438          */
2439         if (!a6xx_gpu->have_mmu500) {
2440                 a6xx_llc_write(a6xx_gpu,
2441                         REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval);
2442
2443                 /*
2444                  * Program cacheability overrides to not allocate cache
2445                  * lines on a write miss
2446                  */
2447                 a6xx_llc_rmw(a6xx_gpu,
2448                         REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03);
2449                 return;
2450         }
2451
2452         gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
2453 }
2454
2455 static void a7xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2456 {
2457         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2458         struct msm_gpu *gpu = &adreno_gpu->base;
2459
2460         if (IS_ERR(a6xx_gpu->llc_mmio))
2461                 return;
2462
2463         if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
2464                 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
2465
2466                 gpu_scid &= GENMASK(4, 0);
2467
2468                 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1,
2469                           FIELD_PREP(GENMASK(29, 25), gpu_scid) |
2470                           FIELD_PREP(GENMASK(24, 20), gpu_scid) |
2471                           FIELD_PREP(GENMASK(19, 15), gpu_scid) |
2472                           FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2473                           FIELD_PREP(GENMASK(9, 5), gpu_scid) |
2474                           FIELD_PREP(GENMASK(4, 0), gpu_scid));
2475
2476                 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0,
2477                           FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2478                           BIT(8));
2479         }
2480
2481         llcc_slice_activate(a6xx_gpu->htw_llc_slice);
2482 }
2483
2484 static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
2485 {
2486         /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2487         if (adreno_has_gmu_wrapper(&a6xx_gpu->base))
2488                 return;
2489
2490         llcc_slice_putd(a6xx_gpu->llc_slice);
2491         llcc_slice_putd(a6xx_gpu->htw_llc_slice);
2492 }
2493
2494 static void a6xx_llc_slices_init(struct platform_device *pdev,
2495                 struct a6xx_gpu *a6xx_gpu, bool is_a7xx)
2496 {
2497         struct device_node *phandle;
2498
2499         /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2500         if (adreno_has_gmu_wrapper(&a6xx_gpu->base))
2501                 return;
2502
2503         /*
2504          * There is a different programming path for A6xx targets with an
2505          * mmu500 attached, so detect if that is the case
2506          */
2507         phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0);
2508         a6xx_gpu->have_mmu500 = (phandle &&
2509                 of_device_is_compatible(phandle, "arm,mmu-500"));
2510         of_node_put(phandle);
2511
2512         if (is_a7xx || !a6xx_gpu->have_mmu500)
2513                 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem");
2514         else
2515                 a6xx_gpu->llc_mmio = NULL;
2516
2517         a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);
2518         a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);
2519
2520         if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
2521                 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL);
2522 }
2523
2524 #define GBIF_CLIENT_HALT_MASK           BIT(0)
2525 #define GBIF_ARB_HALT_MASK              BIT(1)
2526 #define VBIF_XIN_HALT_CTRL0_MASK        GENMASK(3, 0)
2527 #define VBIF_RESET_ACK_MASK             0xF0
2528 #define GPR0_GBIF_HALT_REQUEST          0x1E0
2529
2530 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off)
2531 {
2532         struct msm_gpu *gpu = &adreno_gpu->base;
2533
2534         if (adreno_is_a619_holi(adreno_gpu)) {
2535                 gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, GPR0_GBIF_HALT_REQUEST);
2536                 spin_until((gpu_read(gpu, REG_A6XX_RBBM_VBIF_GX_RESET_STATUS) &
2537                                 (VBIF_RESET_ACK_MASK)) == VBIF_RESET_ACK_MASK);
2538         } else if (!a6xx_has_gbif(adreno_gpu)) {
2539                 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, VBIF_XIN_HALT_CTRL0_MASK);
2540                 spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
2541                                 (VBIF_XIN_HALT_CTRL0_MASK)) == VBIF_XIN_HALT_CTRL0_MASK);
2542                 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
2543
2544                 return;
2545         }
2546
2547         if (gx_off) {
2548                 /* Halt the gx side of GBIF */
2549                 gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1);
2550                 spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1);
2551         }
2552
2553         /* Halt new client requests on GBIF */
2554         gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
2555         spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
2556                         (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
2557
2558         /* Halt all AXI requests on GBIF */
2559         gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
2560         spin_until((gpu_read(gpu,  REG_A6XX_GBIF_HALT_ACK) &
2561                         (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
2562
2563         /* The GBIF halt needs to be explicitly cleared */
2564         gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
2565 }
2566
2567 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert)
2568 {
2569         /* 11nm chips (e.g. ones with A610) have hw issues with the reset line! */
2570         if (adreno_is_a610(to_adreno_gpu(gpu)))
2571                 return;
2572
2573         gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, assert);
2574         /* Perform a bogus read and add a brief delay to ensure ordering. */
2575         gpu_read(gpu, REG_A6XX_RBBM_SW_RESET_CMD);
2576         udelay(1);
2577
2578         /* The reset line needs to be asserted for at least 100 us */
2579         if (assert)
2580                 udelay(100);
2581 }
2582
2583 static int a6xx_gmu_pm_resume(struct msm_gpu *gpu)
2584 {
2585         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2586         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2587         int ret;
2588
2589         gpu->needs_hw_init = true;
2590
2591         trace_msm_gpu_resume(0);
2592
2593         mutex_lock(&a6xx_gpu->gmu.lock);
2594         ret = a6xx_gmu_resume(a6xx_gpu);
2595         mutex_unlock(&a6xx_gpu->gmu.lock);
2596         if (ret)
2597                 return ret;
2598
2599         msm_devfreq_resume(gpu);
2600
2601         adreno_is_a7xx(adreno_gpu) ? a7xx_llc_activate(a6xx_gpu) : a6xx_llc_activate(a6xx_gpu);
2602
2603         return ret;
2604 }
2605
2606 static int a6xx_pm_resume(struct msm_gpu *gpu)
2607 {
2608         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2609         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2610         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2611         unsigned long freq = gpu->fast_rate;
2612         struct dev_pm_opp *opp;
2613         int ret;
2614
2615         gpu->needs_hw_init = true;
2616
2617         trace_msm_gpu_resume(0);
2618
2619         mutex_lock(&a6xx_gpu->gmu.lock);
2620
2621         opp = dev_pm_opp_find_freq_ceil(&gpu->pdev->dev, &freq);
2622         if (IS_ERR(opp)) {
2623                 ret = PTR_ERR(opp);
2624                 goto err_set_opp;
2625         }
2626         dev_pm_opp_put(opp);
2627
2628         /* Set the core clock and bus bw, having VDD scaling in mind */
2629         dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
2630
2631         pm_runtime_resume_and_get(gmu->dev);
2632         pm_runtime_resume_and_get(gmu->gxpd);
2633
2634         ret = clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);
2635         if (ret)
2636                 goto err_bulk_clk;
2637
2638         if (adreno_is_a619_holi(adreno_gpu))
2639                 a6xx_sptprac_enable(gmu);
2640
2641         /* If anything goes south, tear the GPU down piece by piece.. */
2642         if (ret) {
2643 err_bulk_clk:
2644                 pm_runtime_put(gmu->gxpd);
2645                 pm_runtime_put(gmu->dev);
2646                 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
2647         }
2648 err_set_opp:
2649         mutex_unlock(&a6xx_gpu->gmu.lock);
2650
2651         if (!ret)
2652                 msm_devfreq_resume(gpu);
2653
2654         return ret;
2655 }
2656
2657 static int a6xx_gmu_pm_suspend(struct msm_gpu *gpu)
2658 {
2659         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2660         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2661         int i, ret;
2662
2663         trace_msm_gpu_suspend(0);
2664
2665         a6xx_llc_deactivate(a6xx_gpu);
2666
2667         msm_devfreq_suspend(gpu);
2668
2669         mutex_lock(&a6xx_gpu->gmu.lock);
2670         ret = a6xx_gmu_stop(a6xx_gpu);
2671         mutex_unlock(&a6xx_gpu->gmu.lock);
2672         if (ret)
2673                 return ret;
2674
2675         if (a6xx_gpu->shadow_bo)
2676                 for (i = 0; i < gpu->nr_rings; i++)
2677                         a6xx_gpu->shadow[i] = 0;
2678
2679         gpu->suspend_count++;
2680
2681         return 0;
2682 }
2683
2684 static int a6xx_pm_suspend(struct msm_gpu *gpu)
2685 {
2686         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2687         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2688         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2689         int i;
2690
2691         trace_msm_gpu_suspend(0);
2692
2693         msm_devfreq_suspend(gpu);
2694
2695         mutex_lock(&a6xx_gpu->gmu.lock);
2696
2697         /* Drain the outstanding traffic on memory buses */
2698         a6xx_bus_clear_pending_transactions(adreno_gpu, true);
2699
2700         if (adreno_is_a619_holi(adreno_gpu))
2701                 a6xx_sptprac_disable(gmu);
2702
2703         clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks);
2704
2705         pm_runtime_put_sync(gmu->gxpd);
2706         dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
2707         pm_runtime_put_sync(gmu->dev);
2708
2709         mutex_unlock(&a6xx_gpu->gmu.lock);
2710
2711         if (a6xx_gpu->shadow_bo)
2712                 for (i = 0; i < gpu->nr_rings; i++)
2713                         a6xx_gpu->shadow[i] = 0;
2714
2715         gpu->suspend_count++;
2716
2717         return 0;
2718 }
2719
2720 static int a6xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2721 {
2722         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2723         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2724
2725         mutex_lock(&a6xx_gpu->gmu.lock);
2726
2727         /* Force the GPU power on so we can read this register */
2728         a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
2729
2730         *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2731
2732         a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
2733
2734         mutex_unlock(&a6xx_gpu->gmu.lock);
2735
2736         return 0;
2737 }
2738
2739 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2740 {
2741         *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2742         return 0;
2743 }
2744
2745 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
2746 {
2747         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2748         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2749
2750         return a6xx_gpu->cur_ring;
2751 }
2752
2753 static void a6xx_destroy(struct msm_gpu *gpu)
2754 {
2755         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2756         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2757
2758         if (a6xx_gpu->sqe_bo) {
2759                 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
2760                 drm_gem_object_put(a6xx_gpu->sqe_bo);
2761         }
2762
2763         if (a6xx_gpu->shadow_bo) {
2764                 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);
2765                 drm_gem_object_put(a6xx_gpu->shadow_bo);
2766         }
2767
2768         a6xx_llc_slices_destroy(a6xx_gpu);
2769
2770         a6xx_gmu_remove(a6xx_gpu);
2771
2772         adreno_gpu_cleanup(adreno_gpu);
2773
2774         kfree(a6xx_gpu);
2775 }
2776
2777 static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
2778 {
2779         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2780         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2781         u64 busy_cycles;
2782
2783         /* 19.2MHz */
2784         *out_sample_rate = 19200000;
2785
2786         busy_cycles = gmu_read64(&a6xx_gpu->gmu,
2787                         REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
2788                         REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
2789
2790         return busy_cycles;
2791 }
2792
2793 static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
2794                               bool suspended)
2795 {
2796         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2797         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2798
2799         mutex_lock(&a6xx_gpu->gmu.lock);
2800         a6xx_gmu_set_freq(gpu, opp, suspended);
2801         mutex_unlock(&a6xx_gpu->gmu.lock);
2802 }
2803
2804 static struct msm_gem_address_space *
2805 a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
2806 {
2807         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2808         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2809         unsigned long quirks = 0;
2810
2811         /*
2812          * This allows GPU to set the bus attributes required to use system
2813          * cache on behalf of the iommu page table walker.
2814          */
2815         if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice) &&
2816             !device_iommu_capable(&pdev->dev, IOMMU_CAP_CACHE_COHERENCY))
2817                 quirks |= IO_PGTABLE_QUIRK_ARM_OUTER_WBWA;
2818
2819         return adreno_iommu_create_address_space(gpu, pdev, quirks);
2820 }
2821
2822 static struct msm_gem_address_space *
2823 a6xx_create_private_address_space(struct msm_gpu *gpu)
2824 {
2825         struct msm_mmu *mmu;
2826
2827         mmu = msm_iommu_pagetable_create(gpu->aspace->mmu);
2828
2829         if (IS_ERR(mmu))
2830                 return ERR_CAST(mmu);
2831
2832         return msm_gem_address_space_create(mmu,
2833                 "gpu", 0x100000000ULL,
2834                 adreno_private_address_space_size(gpu));
2835 }
2836
2837 static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2838 {
2839         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2840         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2841
2842         if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)
2843                 return a6xx_gpu->shadow[ring->id];
2844
2845         return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR);
2846 }
2847
2848 static bool a6xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2849 {
2850         struct msm_cp_state cp_state = {
2851                 .ib1_base = gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2852                 .ib2_base = gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2853                 .ib1_rem  = gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2854                 .ib2_rem  = gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE),
2855         };
2856         bool progress;
2857
2858         /*
2859          * Adjust the remaining data to account for what has already been
2860          * fetched from memory, but not yet consumed by the SQE.
2861          *
2862          * This is not *technically* correct, the amount buffered could
2863          * exceed the IB size due to hw prefetching ahead, but:
2864          *
2865          * (1) We aren't trying to find the exact position, just whether
2866          *     progress has been made
2867          * (2) The CP_REG_TO_MEM at the end of a submit should be enough
2868          *     to prevent prefetching into an unrelated submit.  (And
2869          *     either way, at some point the ROQ will be full.)
2870          */
2871         cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB1) >> 16;
2872         cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB2) >> 16;
2873
2874         progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state));
2875
2876         ring->last_cp_state = cp_state;
2877
2878         return progress;
2879 }
2880
2881 static u32 fuse_to_supp_hw(const struct adreno_info *info, u32 fuse)
2882 {
2883         if (!info->speedbins)
2884                 return UINT_MAX;
2885
2886         for (int i = 0; info->speedbins[i].fuse != SHRT_MAX; i++)
2887                 if (info->speedbins[i].fuse == fuse)
2888                         return BIT(info->speedbins[i].speedbin);
2889
2890         return UINT_MAX;
2891 }
2892
2893 static int a6xx_set_supported_hw(struct device *dev, const struct adreno_info *info)
2894 {
2895         u32 supp_hw;
2896         u32 speedbin;
2897         int ret;
2898
2899         ret = adreno_read_speedbin(dev, &speedbin);
2900         /*
2901          * -ENOENT means that the platform doesn't support speedbin which is
2902          * fine
2903          */
2904         if (ret == -ENOENT) {
2905                 return 0;
2906         } else if (ret) {
2907                 dev_err_probe(dev, ret,
2908                               "failed to read speed-bin. Some OPPs may not be supported by hardware\n");
2909                 return ret;
2910         }
2911
2912         supp_hw = fuse_to_supp_hw(info, speedbin);
2913
2914         if (supp_hw == UINT_MAX) {
2915                 DRM_DEV_ERROR(dev,
2916                         "missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n",
2917                         speedbin);
2918                 supp_hw = BIT(0); /* Default */
2919         }
2920
2921         ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1);
2922         if (ret)
2923                 return ret;
2924
2925         return 0;
2926 }
2927
2928 static const struct adreno_gpu_funcs funcs = {
2929         .base = {
2930                 .get_param = adreno_get_param,
2931                 .set_param = adreno_set_param,
2932                 .hw_init = a6xx_hw_init,
2933                 .ucode_load = a6xx_ucode_load,
2934                 .pm_suspend = a6xx_gmu_pm_suspend,
2935                 .pm_resume = a6xx_gmu_pm_resume,
2936                 .recover = a6xx_recover,
2937                 .submit = a6xx_submit,
2938                 .active_ring = a6xx_active_ring,
2939                 .irq = a6xx_irq,
2940                 .destroy = a6xx_destroy,
2941 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2942                 .show = a6xx_show,
2943 #endif
2944                 .gpu_busy = a6xx_gpu_busy,
2945                 .gpu_get_freq = a6xx_gmu_get_freq,
2946                 .gpu_set_freq = a6xx_gpu_set_freq,
2947 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2948                 .gpu_state_get = a6xx_gpu_state_get,
2949                 .gpu_state_put = a6xx_gpu_state_put,
2950 #endif
2951                 .create_address_space = a6xx_create_address_space,
2952                 .create_private_address_space = a6xx_create_private_address_space,
2953                 .get_rptr = a6xx_get_rptr,
2954                 .progress = a6xx_progress,
2955         },
2956         .get_timestamp = a6xx_gmu_get_timestamp,
2957 };
2958
2959 static const struct adreno_gpu_funcs funcs_gmuwrapper = {
2960         .base = {
2961                 .get_param = adreno_get_param,
2962                 .set_param = adreno_set_param,
2963                 .hw_init = a6xx_hw_init,
2964                 .ucode_load = a6xx_ucode_load,
2965                 .pm_suspend = a6xx_pm_suspend,
2966                 .pm_resume = a6xx_pm_resume,
2967                 .recover = a6xx_recover,
2968                 .submit = a6xx_submit,
2969                 .active_ring = a6xx_active_ring,
2970                 .irq = a6xx_irq,
2971                 .destroy = a6xx_destroy,
2972 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2973                 .show = a6xx_show,
2974 #endif
2975                 .gpu_busy = a6xx_gpu_busy,
2976 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2977                 .gpu_state_get = a6xx_gpu_state_get,
2978                 .gpu_state_put = a6xx_gpu_state_put,
2979 #endif
2980                 .create_address_space = a6xx_create_address_space,
2981                 .create_private_address_space = a6xx_create_private_address_space,
2982                 .get_rptr = a6xx_get_rptr,
2983                 .progress = a6xx_progress,
2984         },
2985         .get_timestamp = a6xx_get_timestamp,
2986 };
2987
2988 static const struct adreno_gpu_funcs funcs_a7xx = {
2989         .base = {
2990                 .get_param = adreno_get_param,
2991                 .set_param = adreno_set_param,
2992                 .hw_init = a6xx_hw_init,
2993                 .ucode_load = a6xx_ucode_load,
2994                 .pm_suspend = a6xx_gmu_pm_suspend,
2995                 .pm_resume = a6xx_gmu_pm_resume,
2996                 .recover = a6xx_recover,
2997                 .submit = a7xx_submit,
2998                 .active_ring = a6xx_active_ring,
2999                 .irq = a6xx_irq,
3000                 .destroy = a6xx_destroy,
3001 #if defined(CONFIG_DRM_MSM_GPU_STATE)
3002                 .show = a6xx_show,
3003 #endif
3004                 .gpu_busy = a6xx_gpu_busy,
3005                 .gpu_get_freq = a6xx_gmu_get_freq,
3006                 .gpu_set_freq = a6xx_gpu_set_freq,
3007 #if defined(CONFIG_DRM_MSM_GPU_STATE)
3008                 .gpu_state_get = a6xx_gpu_state_get,
3009                 .gpu_state_put = a6xx_gpu_state_put,
3010 #endif
3011                 .create_address_space = a6xx_create_address_space,
3012                 .create_private_address_space = a6xx_create_private_address_space,
3013                 .get_rptr = a6xx_get_rptr,
3014                 .progress = a6xx_progress,
3015         },
3016         .get_timestamp = a6xx_gmu_get_timestamp,
3017 };
3018
3019 struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
3020 {
3021         struct msm_drm_private *priv = dev->dev_private;
3022         struct platform_device *pdev = priv->gpu_pdev;
3023         struct adreno_platform_config *config = pdev->dev.platform_data;
3024         struct device_node *node;
3025         struct a6xx_gpu *a6xx_gpu;
3026         struct adreno_gpu *adreno_gpu;
3027         struct msm_gpu *gpu;
3028         bool is_a7xx;
3029         int ret;
3030
3031         a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
3032         if (!a6xx_gpu)
3033                 return ERR_PTR(-ENOMEM);
3034
3035         adreno_gpu = &a6xx_gpu->base;
3036         gpu = &adreno_gpu->base;
3037
3038         mutex_init(&a6xx_gpu->gmu.lock);
3039
3040         adreno_gpu->registers = NULL;
3041
3042         /* Check if there is a GMU phandle and set it up */
3043         node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
3044         /* FIXME: How do we gracefully handle this? */
3045         BUG_ON(!node);
3046
3047         adreno_gpu->gmu_is_wrapper = of_device_is_compatible(node, "qcom,adreno-gmu-wrapper");
3048
3049         adreno_gpu->base.hw_apriv =
3050                 !!(config->info->quirks & ADRENO_QUIRK_HAS_HW_APRIV);
3051
3052         /* gpu->info only gets assigned in adreno_gpu_init() */
3053         is_a7xx = config->info->family == ADRENO_7XX_GEN1 ||
3054                   config->info->family == ADRENO_7XX_GEN2 ||
3055                   config->info->family == ADRENO_7XX_GEN3;
3056
3057         a6xx_llc_slices_init(pdev, a6xx_gpu, is_a7xx);
3058
3059         ret = a6xx_set_supported_hw(&pdev->dev, config->info);
3060         if (ret) {
3061                 a6xx_destroy(&(a6xx_gpu->base.base));
3062                 return ERR_PTR(ret);
3063         }
3064
3065         if (is_a7xx)
3066                 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_a7xx, 1);
3067         else if (adreno_has_gmu_wrapper(adreno_gpu))
3068                 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_gmuwrapper, 1);
3069         else
3070                 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
3071         if (ret) {
3072                 a6xx_destroy(&(a6xx_gpu->base.base));
3073                 return ERR_PTR(ret);
3074         }
3075
3076         /*
3077          * For now only clamp to idle freq for devices where this is known not
3078          * to cause power supply issues:
3079          */
3080         if (adreno_is_a618(adreno_gpu) || adreno_is_7c3(adreno_gpu))
3081                 priv->gpu_clamp_to_idle = true;
3082
3083         if (adreno_has_gmu_wrapper(adreno_gpu))
3084                 ret = a6xx_gmu_wrapper_init(a6xx_gpu, node);
3085         else
3086                 ret = a6xx_gmu_init(a6xx_gpu, node);
3087         of_node_put(node);
3088         if (ret) {
3089                 a6xx_destroy(&(a6xx_gpu->base.base));
3090                 return ERR_PTR(ret);
3091         }
3092
3093         if (gpu->aspace)
3094                 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
3095                                 a6xx_fault_handler);
3096
3097         a6xx_calc_ubwc_config(adreno_gpu);
3098
3099         return gpu;
3100 }