drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45
  46 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
  47 MODULE_FIRMWARE("radeon/bonaire_me.bin");
  48 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
  49 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
  50 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
  51 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
  52 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
  53 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
  54
  55 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  56 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  57 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  58 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  59 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  60 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  61 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  62 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  63 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  64
  65 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
  66 MODULE_FIRMWARE("radeon/hawaii_me.bin");
  67 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
  68 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
  69 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
  70 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
  71 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
  72 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
  73
  74 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  75 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  76 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  77 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  78 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  79 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  80
  81 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
  82 MODULE_FIRMWARE("radeon/kaveri_me.bin");
  83 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
  84 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
  85 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
  86 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
  87 MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
  88
  89 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  90 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  91 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  92 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  93 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  94 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  95
  96 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
  97 MODULE_FIRMWARE("radeon/kabini_me.bin");
  98 MODULE_FIRMWARE("radeon/kabini_ce.bin");
  99 MODULE_FIRMWARE("radeon/kabini_mec.bin");
 100 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
 101 MODULE_FIRMWARE("radeon/kabini_sdma.bin");
 102
 103 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
 104 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
 105 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
 106 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
 107 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
 108 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
 109
 110 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
 111 MODULE_FIRMWARE("radeon/mullins_me.bin");
 112 MODULE_FIRMWARE("radeon/mullins_ce.bin");
 113 MODULE_FIRMWARE("radeon/mullins_mec.bin");
 114 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
 115 MODULE_FIRMWARE("radeon/mullins_sdma.bin");
 116
 117 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 118 extern void r600_ih_ring_fini(struct radeon_device *rdev);
 119 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 120 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 121 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 122 extern void sumo_rlc_fini(struct radeon_device *rdev);
 123 extern int sumo_rlc_init(struct radeon_device *rdev);
 124 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
 125 extern void si_rlc_reset(struct radeon_device *rdev);
 126 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 127 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 128 extern int cik_sdma_resume(struct radeon_device *rdev);
 129 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 130 extern void cik_sdma_fini(struct radeon_device *rdev);
 131 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 132 static void cik_rlc_stop(struct radeon_device *rdev);
 133 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 134 static void cik_program_aspm(struct radeon_device *rdev);
 135 static void cik_init_pg(struct radeon_device *rdev);
 136 static void cik_init_cg(struct radeon_device *rdev);
 137 static void cik_fini_pg(struct radeon_device *rdev);
 138 static void cik_fini_cg(struct radeon_device *rdev);
 139 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
 140                                           bool enable);
 141
 142 /* get temperature in millidegrees */
 143 int ci_get_temp(struct radeon_device *rdev)
 144 {
 145         u32 temp;
 146         int actual_temp = 0;
 147
 148         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 149                 CTF_TEMP_SHIFT;
 150
 151         if (temp & 0x200)
 152                 actual_temp = 255;
 153         else
 154                 actual_temp = temp & 0x1ff;
 155
 156         actual_temp = actual_temp * 1000;
 157
 158         return actual_temp;
 159 }
 160
 161 /* get temperature in millidegrees */
 162 int kv_get_temp(struct radeon_device *rdev)
 163 {
 164         u32 temp;
 165         int actual_temp = 0;
 166
 167         temp = RREG32_SMC(0xC0300E0C);
 168
 169         if (temp)
 170                 actual_temp = (temp / 8) - 49;
 171         else
 172                 actual_temp = 0;
 173
 174         actual_temp = actual_temp * 1000;
 175
 176         return actual_temp;
 177 }
 178
 179 /*
 180  * Indirect registers accessor
 181  */
 182 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 183 {
 184         unsigned long flags;
 185         u32 r;
 186
 187         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 188         WREG32(PCIE_INDEX, reg);
 189         (void)RREG32(PCIE_INDEX);
 190         r = RREG32(PCIE_DATA);
 191         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 192         return r;
 193 }
 194
 195 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 196 {
 197         unsigned long flags;
 198
 199         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 200         WREG32(PCIE_INDEX, reg);
 201         (void)RREG32(PCIE_INDEX);
 202         WREG32(PCIE_DATA, v);
 203         (void)RREG32(PCIE_DATA);
 204         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 205 }
 206
 207 static const u32 spectre_rlc_save_restore_register_list[] =
 208 {
 209         (0x0e00 << 16) | (0xc12c >> 2),
 210         0x00000000,
 211         (0x0e00 << 16) | (0xc140 >> 2),
 212         0x00000000,
 213         (0x0e00 << 16) | (0xc150 >> 2),
 214         0x00000000,
 215         (0x0e00 << 16) | (0xc15c >> 2),
 216         0x00000000,
 217         (0x0e00 << 16) | (0xc168 >> 2),
 218         0x00000000,
 219         (0x0e00 << 16) | (0xc170 >> 2),
 220         0x00000000,
 221         (0x0e00 << 16) | (0xc178 >> 2),
 222         0x00000000,
 223         (0x0e00 << 16) | (0xc204 >> 2),
 224         0x00000000,
 225         (0x0e00 << 16) | (0xc2b4 >> 2),
 226         0x00000000,
 227         (0x0e00 << 16) | (0xc2b8 >> 2),
 228         0x00000000,
 229         (0x0e00 << 16) | (0xc2bc >> 2),
 230         0x00000000,
 231         (0x0e00 << 16) | (0xc2c0 >> 2),
 232         0x00000000,
 233         (0x0e00 << 16) | (0x8228 >> 2),
 234         0x00000000,
 235         (0x0e00 << 16) | (0x829c >> 2),
 236         0x00000000,
 237         (0x0e00 << 16) | (0x869c >> 2),
 238         0x00000000,
 239         (0x0600 << 16) | (0x98f4 >> 2),
 240         0x00000000,
 241         (0x0e00 << 16) | (0x98f8 >> 2),
 242         0x00000000,
 243         (0x0e00 << 16) | (0x9900 >> 2),
 244         0x00000000,
 245         (0x0e00 << 16) | (0xc260 >> 2),
 246         0x00000000,
 247         (0x0e00 << 16) | (0x90e8 >> 2),
 248         0x00000000,
 249         (0x0e00 << 16) | (0x3c000 >> 2),
 250         0x00000000,
 251         (0x0e00 << 16) | (0x3c00c >> 2),
 252         0x00000000,
 253         (0x0e00 << 16) | (0x8c1c >> 2),
 254         0x00000000,
 255         (0x0e00 << 16) | (0x9700 >> 2),
 256         0x00000000,
 257         (0x0e00 << 16) | (0xcd20 >> 2),
 258         0x00000000,
 259         (0x4e00 << 16) | (0xcd20 >> 2),
 260         0x00000000,
 261         (0x5e00 << 16) | (0xcd20 >> 2),
 262         0x00000000,
 263         (0x6e00 << 16) | (0xcd20 >> 2),
 264         0x00000000,
 265         (0x7e00 << 16) | (0xcd20 >> 2),
 266         0x00000000,
 267         (0x8e00 << 16) | (0xcd20 >> 2),
 268         0x00000000,
 269         (0x9e00 << 16) | (0xcd20 >> 2),
 270         0x00000000,
 271         (0xae00 << 16) | (0xcd20 >> 2),
 272         0x00000000,
 273         (0xbe00 << 16) | (0xcd20 >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0x89bc >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0x8900 >> 2),
 278         0x00000000,
 279         0x3,
 280         (0x0e00 << 16) | (0xc130 >> 2),
 281         0x00000000,
 282         (0x0e00 << 16) | (0xc134 >> 2),
 283         0x00000000,
 284         (0x0e00 << 16) | (0xc1fc >> 2),
 285         0x00000000,
 286         (0x0e00 << 16) | (0xc208 >> 2),
 287         0x00000000,
 288         (0x0e00 << 16) | (0xc264 >> 2),
 289         0x00000000,
 290         (0x0e00 << 16) | (0xc268 >> 2),
 291         0x00000000,
 292         (0x0e00 << 16) | (0xc26c >> 2),
 293         0x00000000,
 294         (0x0e00 << 16) | (0xc270 >> 2),
 295         0x00000000,
 296         (0x0e00 << 16) | (0xc274 >> 2),
 297         0x00000000,
 298         (0x0e00 << 16) | (0xc278 >> 2),
 299         0x00000000,
 300         (0x0e00 << 16) | (0xc27c >> 2),
 301         0x00000000,
 302         (0x0e00 << 16) | (0xc280 >> 2),
 303         0x00000000,
 304         (0x0e00 << 16) | (0xc284 >> 2),
 305         0x00000000,
 306         (0x0e00 << 16) | (0xc288 >> 2),
 307         0x00000000,
 308         (0x0e00 << 16) | (0xc28c >> 2),
 309         0x00000000,
 310         (0x0e00 << 16) | (0xc290 >> 2),
 311         0x00000000,
 312         (0x0e00 << 16) | (0xc294 >> 2),
 313         0x00000000,
 314         (0x0e00 << 16) | (0xc298 >> 2),
 315         0x00000000,
 316         (0x0e00 << 16) | (0xc29c >> 2),
 317         0x00000000,
 318         (0x0e00 << 16) | (0xc2a0 >> 2),
 319         0x00000000,
 320         (0x0e00 << 16) | (0xc2a4 >> 2),
 321         0x00000000,
 322         (0x0e00 << 16) | (0xc2a8 >> 2),
 323         0x00000000,
 324         (0x0e00 << 16) | (0xc2ac  >> 2),
 325         0x00000000,
 326         (0x0e00 << 16) | (0xc2b0 >> 2),
 327         0x00000000,
 328         (0x0e00 << 16) | (0x301d0 >> 2),
 329         0x00000000,
 330         (0x0e00 << 16) | (0x30238 >> 2),
 331         0x00000000,
 332         (0x0e00 << 16) | (0x30250 >> 2),
 333         0x00000000,
 334         (0x0e00 << 16) | (0x30254 >> 2),
 335         0x00000000,
 336         (0x0e00 << 16) | (0x30258 >> 2),
 337         0x00000000,
 338         (0x0e00 << 16) | (0x3025c >> 2),
 339         0x00000000,
 340         (0x4e00 << 16) | (0xc900 >> 2),
 341         0x00000000,
 342         (0x5e00 << 16) | (0xc900 >> 2),
 343         0x00000000,
 344         (0x6e00 << 16) | (0xc900 >> 2),
 345         0x00000000,
 346         (0x7e00 << 16) | (0xc900 >> 2),
 347         0x00000000,
 348         (0x8e00 << 16) | (0xc900 >> 2),
 349         0x00000000,
 350         (0x9e00 << 16) | (0xc900 >> 2),
 351         0x00000000,
 352         (0xae00 << 16) | (0xc900 >> 2),
 353         0x00000000,
 354         (0xbe00 << 16) | (0xc900 >> 2),
 355         0x00000000,
 356         (0x4e00 << 16) | (0xc904 >> 2),
 357         0x00000000,
 358         (0x5e00 << 16) | (0xc904 >> 2),
 359         0x00000000,
 360         (0x6e00 << 16) | (0xc904 >> 2),
 361         0x00000000,
 362         (0x7e00 << 16) | (0xc904 >> 2),
 363         0x00000000,
 364         (0x8e00 << 16) | (0xc904 >> 2),
 365         0x00000000,
 366         (0x9e00 << 16) | (0xc904 >> 2),
 367         0x00000000,
 368         (0xae00 << 16) | (0xc904 >> 2),
 369         0x00000000,
 370         (0xbe00 << 16) | (0xc904 >> 2),
 371         0x00000000,
 372         (0x4e00 << 16) | (0xc908 >> 2),
 373         0x00000000,
 374         (0x5e00 << 16) | (0xc908 >> 2),
 375         0x00000000,
 376         (0x6e00 << 16) | (0xc908 >> 2),
 377         0x00000000,
 378         (0x7e00 << 16) | (0xc908 >> 2),
 379         0x00000000,
 380         (0x8e00 << 16) | (0xc908 >> 2),
 381         0x00000000,
 382         (0x9e00 << 16) | (0xc908 >> 2),
 383         0x00000000,
 384         (0xae00 << 16) | (0xc908 >> 2),
 385         0x00000000,
 386         (0xbe00 << 16) | (0xc908 >> 2),
 387         0x00000000,
 388         (0x4e00 << 16) | (0xc90c >> 2),
 389         0x00000000,
 390         (0x5e00 << 16) | (0xc90c >> 2),
 391         0x00000000,
 392         (0x6e00 << 16) | (0xc90c >> 2),
 393         0x00000000,
 394         (0x7e00 << 16) | (0xc90c >> 2),
 395         0x00000000,
 396         (0x8e00 << 16) | (0xc90c >> 2),
 397         0x00000000,
 398         (0x9e00 << 16) | (0xc90c >> 2),
 399         0x00000000,
 400         (0xae00 << 16) | (0xc90c >> 2),
 401         0x00000000,
 402         (0xbe00 << 16) | (0xc90c >> 2),
 403         0x00000000,
 404         (0x4e00 << 16) | (0xc910 >> 2),
 405         0x00000000,
 406         (0x5e00 << 16) | (0xc910 >> 2),
 407         0x00000000,
 408         (0x6e00 << 16) | (0xc910 >> 2),
 409         0x00000000,
 410         (0x7e00 << 16) | (0xc910 >> 2),
 411         0x00000000,
 412         (0x8e00 << 16) | (0xc910 >> 2),
 413         0x00000000,
 414         (0x9e00 << 16) | (0xc910 >> 2),
 415         0x00000000,
 416         (0xae00 << 16) | (0xc910 >> 2),
 417         0x00000000,
 418         (0xbe00 << 16) | (0xc910 >> 2),
 419         0x00000000,
 420         (0x0e00 << 16) | (0xc99c >> 2),
 421         0x00000000,
 422         (0x0e00 << 16) | (0x9834 >> 2),
 423         0x00000000,
 424         (0x0000 << 16) | (0x30f00 >> 2),
 425         0x00000000,
 426         (0x0001 << 16) | (0x30f00 >> 2),
 427         0x00000000,
 428         (0x0000 << 16) | (0x30f04 >> 2),
 429         0x00000000,
 430         (0x0001 << 16) | (0x30f04 >> 2),
 431         0x00000000,
 432         (0x0000 << 16) | (0x30f08 >> 2),
 433         0x00000000,
 434         (0x0001 << 16) | (0x30f08 >> 2),
 435         0x00000000,
 436         (0x0000 << 16) | (0x30f0c >> 2),
 437         0x00000000,
 438         (0x0001 << 16) | (0x30f0c >> 2),
 439         0x00000000,
 440         (0x0600 << 16) | (0x9b7c >> 2),
 441         0x00000000,
 442         (0x0e00 << 16) | (0x8a14 >> 2),
 443         0x00000000,
 444         (0x0e00 << 16) | (0x8a18 >> 2),
 445         0x00000000,
 446         (0x0600 << 16) | (0x30a00 >> 2),
 447         0x00000000,
 448         (0x0e00 << 16) | (0x8bf0 >> 2),
 449         0x00000000,
 450         (0x0e00 << 16) | (0x8bcc >> 2),
 451         0x00000000,
 452         (0x0e00 << 16) | (0x8b24 >> 2),
 453         0x00000000,
 454         (0x0e00 << 16) | (0x30a04 >> 2),
 455         0x00000000,
 456         (0x0600 << 16) | (0x30a10 >> 2),
 457         0x00000000,
 458         (0x0600 << 16) | (0x30a14 >> 2),
 459         0x00000000,
 460         (0x0600 << 16) | (0x30a18 >> 2),
 461         0x00000000,
 462         (0x0600 << 16) | (0x30a2c >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0xc700 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0xc704 >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0xc708 >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0xc768 >> 2),
 471         0x00000000,
 472         (0x0400 << 16) | (0xc770 >> 2),
 473         0x00000000,
 474         (0x0400 << 16) | (0xc774 >> 2),
 475         0x00000000,
 476         (0x0400 << 16) | (0xc778 >> 2),
 477         0x00000000,
 478         (0x0400 << 16) | (0xc77c >> 2),
 479         0x00000000,
 480         (0x0400 << 16) | (0xc780 >> 2),
 481         0x00000000,
 482         (0x0400 << 16) | (0xc784 >> 2),
 483         0x00000000,
 484         (0x0400 << 16) | (0xc788 >> 2),
 485         0x00000000,
 486         (0x0400 << 16) | (0xc78c >> 2),
 487         0x00000000,
 488         (0x0400 << 16) | (0xc798 >> 2),
 489         0x00000000,
 490         (0x0400 << 16) | (0xc79c >> 2),
 491         0x00000000,
 492         (0x0400 << 16) | (0xc7a0 >> 2),
 493         0x00000000,
 494         (0x0400 << 16) | (0xc7a4 >> 2),
 495         0x00000000,
 496         (0x0400 << 16) | (0xc7a8 >> 2),
 497         0x00000000,
 498         (0x0400 << 16) | (0xc7ac >> 2),
 499         0x00000000,
 500         (0x0400 << 16) | (0xc7b0 >> 2),
 501         0x00000000,
 502         (0x0400 << 16) | (0xc7b4 >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0x9100 >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0x3c010 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0x92a8 >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0x92ac >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0x92b4 >> 2),
 513         0x00000000,
 514         (0x0e00 << 16) | (0x92b8 >> 2),
 515         0x00000000,
 516         (0x0e00 << 16) | (0x92bc >> 2),
 517         0x00000000,
 518         (0x0e00 << 16) | (0x92c0 >> 2),
 519         0x00000000,
 520         (0x0e00 << 16) | (0x92c4 >> 2),
 521         0x00000000,
 522         (0x0e00 << 16) | (0x92c8 >> 2),
 523         0x00000000,
 524         (0x0e00 << 16) | (0x92cc >> 2),
 525         0x00000000,
 526         (0x0e00 << 16) | (0x92d0 >> 2),
 527         0x00000000,
 528         (0x0e00 << 16) | (0x8c00 >> 2),
 529         0x00000000,
 530         (0x0e00 << 16) | (0x8c04 >> 2),
 531         0x00000000,
 532         (0x0e00 << 16) | (0x8c20 >> 2),
 533         0x00000000,
 534         (0x0e00 << 16) | (0x8c38 >> 2),
 535         0x00000000,
 536         (0x0e00 << 16) | (0x8c3c >> 2),
 537         0x00000000,
 538         (0x0e00 << 16) | (0xae00 >> 2),
 539         0x00000000,
 540         (0x0e00 << 16) | (0x9604 >> 2),
 541         0x00000000,
 542         (0x0e00 << 16) | (0xac08 >> 2),
 543         0x00000000,
 544         (0x0e00 << 16) | (0xac0c >> 2),
 545         0x00000000,
 546         (0x0e00 << 16) | (0xac10 >> 2),
 547         0x00000000,
 548         (0x0e00 << 16) | (0xac14 >> 2),
 549         0x00000000,
 550         (0x0e00 << 16) | (0xac58 >> 2),
 551         0x00000000,
 552         (0x0e00 << 16) | (0xac68 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0xac6c >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0xac70 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0xac74 >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0xac78 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0xac7c >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0xac80 >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0xac84 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0xac88 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0xac8c >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x970c >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x9714 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x9718 >> 2),
 577         0x00000000,
 578         (0x0e00 << 16) | (0x971c >> 2),
 579         0x00000000,
 580         (0x0e00 << 16) | (0x31068 >> 2),
 581         0x00000000,
 582         (0x4e00 << 16) | (0x31068 >> 2),
 583         0x00000000,
 584         (0x5e00 << 16) | (0x31068 >> 2),
 585         0x00000000,
 586         (0x6e00 << 16) | (0x31068 >> 2),
 587         0x00000000,
 588         (0x7e00 << 16) | (0x31068 >> 2),
 589         0x00000000,
 590         (0x8e00 << 16) | (0x31068 >> 2),
 591         0x00000000,
 592         (0x9e00 << 16) | (0x31068 >> 2),
 593         0x00000000,
 594         (0xae00 << 16) | (0x31068 >> 2),
 595         0x00000000,
 596         (0xbe00 << 16) | (0x31068 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xcd10 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xcd14 >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0x88b0 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0x88b4 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0x88b8 >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0x88bc >> 2),
 609         0x00000000,
 610         (0x0400 << 16) | (0x89c0 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0x88c4 >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0x88c8 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0x88d0 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0x88d4 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0x88d8 >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0x8980 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x30938 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x3093c >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x30940 >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x89a0 >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x30900 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x30904 >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0x89b4 >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x3c210 >> 2),
 639         0x00000000,
 640         (0x0e00 << 16) | (0x3c214 >> 2),
 641         0x00000000,
 642         (0x0e00 << 16) | (0x3c218 >> 2),
 643         0x00000000,
 644         (0x0e00 << 16) | (0x8904 >> 2),
 645         0x00000000,
 646         0x5,
 647         (0x0e00 << 16) | (0x8c28 >> 2),
 648         (0x0e00 << 16) | (0x8c2c >> 2),
 649         (0x0e00 << 16) | (0x8c30 >> 2),
 650         (0x0e00 << 16) | (0x8c34 >> 2),
 651         (0x0e00 << 16) | (0x9600 >> 2),
 652 };
 653
 654 static const u32 kalindi_rlc_save_restore_register_list[] =
 655 {
 656         (0x0e00 << 16) | (0xc12c >> 2),
 657         0x00000000,
 658         (0x0e00 << 16) | (0xc140 >> 2),
 659         0x00000000,
 660         (0x0e00 << 16) | (0xc150 >> 2),
 661         0x00000000,
 662         (0x0e00 << 16) | (0xc15c >> 2),
 663         0x00000000,
 664         (0x0e00 << 16) | (0xc168 >> 2),
 665         0x00000000,
 666         (0x0e00 << 16) | (0xc170 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0xc204 >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0xc2b4 >> 2),
 671         0x00000000,
 672         (0x0e00 << 16) | (0xc2b8 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0xc2bc >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0xc2c0 >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0x8228 >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0x829c >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0x869c >> 2),
 683         0x00000000,
 684         (0x0600 << 16) | (0x98f4 >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0x98f8 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0x9900 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0xc260 >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0x90e8 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0x3c000 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0x3c00c >> 2),
 697         0x00000000,
 698         (0x0e00 << 16) | (0x8c1c >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0x9700 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0xcd20 >> 2),
 703         0x00000000,
 704         (0x4e00 << 16) | (0xcd20 >> 2),
 705         0x00000000,
 706         (0x5e00 << 16) | (0xcd20 >> 2),
 707         0x00000000,
 708         (0x6e00 << 16) | (0xcd20 >> 2),
 709         0x00000000,
 710         (0x7e00 << 16) | (0xcd20 >> 2),
 711         0x00000000,
 712         (0x0e00 << 16) | (0x89bc >> 2),
 713         0x00000000,
 714         (0x0e00 << 16) | (0x8900 >> 2),
 715         0x00000000,
 716         0x3,
 717         (0x0e00 << 16) | (0xc130 >> 2),
 718         0x00000000,
 719         (0x0e00 << 16) | (0xc134 >> 2),
 720         0x00000000,
 721         (0x0e00 << 16) | (0xc1fc >> 2),
 722         0x00000000,
 723         (0x0e00 << 16) | (0xc208 >> 2),
 724         0x00000000,
 725         (0x0e00 << 16) | (0xc264 >> 2),
 726         0x00000000,
 727         (0x0e00 << 16) | (0xc268 >> 2),
 728         0x00000000,
 729         (0x0e00 << 16) | (0xc26c >> 2),
 730         0x00000000,
 731         (0x0e00 << 16) | (0xc270 >> 2),
 732         0x00000000,
 733         (0x0e00 << 16) | (0xc274 >> 2),
 734         0x00000000,
 735         (0x0e00 << 16) | (0xc28c >> 2),
 736         0x00000000,
 737         (0x0e00 << 16) | (0xc290 >> 2),
 738         0x00000000,
 739         (0x0e00 << 16) | (0xc294 >> 2),
 740         0x00000000,
 741         (0x0e00 << 16) | (0xc298 >> 2),
 742         0x00000000,
 743         (0x0e00 << 16) | (0xc2a0 >> 2),
 744         0x00000000,
 745         (0x0e00 << 16) | (0xc2a4 >> 2),
 746         0x00000000,
 747         (0x0e00 << 16) | (0xc2a8 >> 2),
 748         0x00000000,
 749         (0x0e00 << 16) | (0xc2ac >> 2),
 750         0x00000000,
 751         (0x0e00 << 16) | (0x301d0 >> 2),
 752         0x00000000,
 753         (0x0e00 << 16) | (0x30238 >> 2),
 754         0x00000000,
 755         (0x0e00 << 16) | (0x30250 >> 2),
 756         0x00000000,
 757         (0x0e00 << 16) | (0x30254 >> 2),
 758         0x00000000,
 759         (0x0e00 << 16) | (0x30258 >> 2),
 760         0x00000000,
 761         (0x0e00 << 16) | (0x3025c >> 2),
 762         0x00000000,
 763         (0x4e00 << 16) | (0xc900 >> 2),
 764         0x00000000,
 765         (0x5e00 << 16) | (0xc900 >> 2),
 766         0x00000000,
 767         (0x6e00 << 16) | (0xc900 >> 2),
 768         0x00000000,
 769         (0x7e00 << 16) | (0xc900 >> 2),
 770         0x00000000,
 771         (0x4e00 << 16) | (0xc904 >> 2),
 772         0x00000000,
 773         (0x5e00 << 16) | (0xc904 >> 2),
 774         0x00000000,
 775         (0x6e00 << 16) | (0xc904 >> 2),
 776         0x00000000,
 777         (0x7e00 << 16) | (0xc904 >> 2),
 778         0x00000000,
 779         (0x4e00 << 16) | (0xc908 >> 2),
 780         0x00000000,
 781         (0x5e00 << 16) | (0xc908 >> 2),
 782         0x00000000,
 783         (0x6e00 << 16) | (0xc908 >> 2),
 784         0x00000000,
 785         (0x7e00 << 16) | (0xc908 >> 2),
 786         0x00000000,
 787         (0x4e00 << 16) | (0xc90c >> 2),
 788         0x00000000,
 789         (0x5e00 << 16) | (0xc90c >> 2),
 790         0x00000000,
 791         (0x6e00 << 16) | (0xc90c >> 2),
 792         0x00000000,
 793         (0x7e00 << 16) | (0xc90c >> 2),
 794         0x00000000,
 795         (0x4e00 << 16) | (0xc910 >> 2),
 796         0x00000000,
 797         (0x5e00 << 16) | (0xc910 >> 2),
 798         0x00000000,
 799         (0x6e00 << 16) | (0xc910 >> 2),
 800         0x00000000,
 801         (0x7e00 << 16) | (0xc910 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0xc99c >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0x9834 >> 2),
 806         0x00000000,
 807         (0x0000 << 16) | (0x30f00 >> 2),
 808         0x00000000,
 809         (0x0000 << 16) | (0x30f04 >> 2),
 810         0x00000000,
 811         (0x0000 << 16) | (0x30f08 >> 2),
 812         0x00000000,
 813         (0x0000 << 16) | (0x30f0c >> 2),
 814         0x00000000,
 815         (0x0600 << 16) | (0x9b7c >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0x8a14 >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0x8a18 >> 2),
 820         0x00000000,
 821         (0x0600 << 16) | (0x30a00 >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0x8bf0 >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0x8bcc >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0x8b24 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0x30a04 >> 2),
 830         0x00000000,
 831         (0x0600 << 16) | (0x30a10 >> 2),
 832         0x00000000,
 833         (0x0600 << 16) | (0x30a14 >> 2),
 834         0x00000000,
 835         (0x0600 << 16) | (0x30a18 >> 2),
 836         0x00000000,
 837         (0x0600 << 16) | (0x30a2c >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0xc700 >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0xc704 >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0xc708 >> 2),
 844         0x00000000,
 845         (0x0e00 << 16) | (0xc768 >> 2),
 846         0x00000000,
 847         (0x0400 << 16) | (0xc770 >> 2),
 848         0x00000000,
 849         (0x0400 << 16) | (0xc774 >> 2),
 850         0x00000000,
 851         (0x0400 << 16) | (0xc798 >> 2),
 852         0x00000000,
 853         (0x0400 << 16) | (0xc79c >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0x9100 >> 2),
 856         0x00000000,
 857         (0x0e00 << 16) | (0x3c010 >> 2),
 858         0x00000000,
 859         (0x0e00 << 16) | (0x8c00 >> 2),
 860         0x00000000,
 861         (0x0e00 << 16) | (0x8c04 >> 2),
 862         0x00000000,
 863         (0x0e00 << 16) | (0x8c20 >> 2),
 864         0x00000000,
 865         (0x0e00 << 16) | (0x8c38 >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x8c3c >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0xae00 >> 2),
 870         0x00000000,
 871         (0x0e00 << 16) | (0x9604 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0xac08 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0xac0c >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0xac10 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0xac14 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0xac58 >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0xac68 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0xac6c >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0xac70 >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0xac74 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0xac78 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0xac7c >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0xac80 >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0xac84 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0xac88 >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0xac8c >> 2),
 902         0x00000000,
 903         (0x0e00 << 16) | (0x970c >> 2),
 904         0x00000000,
 905         (0x0e00 << 16) | (0x9714 >> 2),
 906         0x00000000,
 907         (0x0e00 << 16) | (0x9718 >> 2),
 908         0x00000000,
 909         (0x0e00 << 16) | (0x971c >> 2),
 910         0x00000000,
 911         (0x0e00 << 16) | (0x31068 >> 2),
 912         0x00000000,
 913         (0x4e00 << 16) | (0x31068 >> 2),
 914         0x00000000,
 915         (0x5e00 << 16) | (0x31068 >> 2),
 916         0x00000000,
 917         (0x6e00 << 16) | (0x31068 >> 2),
 918         0x00000000,
 919         (0x7e00 << 16) | (0x31068 >> 2),
 920         0x00000000,
 921         (0x0e00 << 16) | (0xcd10 >> 2),
 922         0x00000000,
 923         (0x0e00 << 16) | (0xcd14 >> 2),
 924         0x00000000,
 925         (0x0e00 << 16) | (0x88b0 >> 2),
 926         0x00000000,
 927         (0x0e00 << 16) | (0x88b4 >> 2),
 928         0x00000000,
 929         (0x0e00 << 16) | (0x88b8 >> 2),
 930         0x00000000,
 931         (0x0e00 << 16) | (0x88bc >> 2),
 932         0x00000000,
 933         (0x0400 << 16) | (0x89c0 >> 2),
 934         0x00000000,
 935         (0x0e00 << 16) | (0x88c4 >> 2),
 936         0x00000000,
 937         (0x0e00 << 16) | (0x88c8 >> 2),
 938         0x00000000,
 939         (0x0e00 << 16) | (0x88d0 >> 2),
 940         0x00000000,
 941         (0x0e00 << 16) | (0x88d4 >> 2),
 942         0x00000000,
 943         (0x0e00 << 16) | (0x88d8 >> 2),
 944         0x00000000,
 945         (0x0e00 << 16) | (0x8980 >> 2),
 946         0x00000000,
 947         (0x0e00 << 16) | (0x30938 >> 2),
 948         0x00000000,
 949         (0x0e00 << 16) | (0x3093c >> 2),
 950         0x00000000,
 951         (0x0e00 << 16) | (0x30940 >> 2),
 952         0x00000000,
 953         (0x0e00 << 16) | (0x89a0 >> 2),
 954         0x00000000,
 955         (0x0e00 << 16) | (0x30900 >> 2),
 956         0x00000000,
 957         (0x0e00 << 16) | (0x30904 >> 2),
 958         0x00000000,
 959         (0x0e00 << 16) | (0x89b4 >> 2),
 960         0x00000000,
 961         (0x0e00 << 16) | (0x3e1fc >> 2),
 962         0x00000000,
 963         (0x0e00 << 16) | (0x3c210 >> 2),
 964         0x00000000,
 965         (0x0e00 << 16) | (0x3c214 >> 2),
 966         0x00000000,
 967         (0x0e00 << 16) | (0x3c218 >> 2),
 968         0x00000000,
 969         (0x0e00 << 16) | (0x8904 >> 2),
 970         0x00000000,
 971         0x5,
 972         (0x0e00 << 16) | (0x8c28 >> 2),
 973         (0x0e00 << 16) | (0x8c2c >> 2),
 974         (0x0e00 << 16) | (0x8c30 >> 2),
 975         (0x0e00 << 16) | (0x8c34 >> 2),
 976         (0x0e00 << 16) | (0x9600 >> 2),
 977 };
 978
 979 static const u32 bonaire_golden_spm_registers[] =
 980 {
 981         0x30800, 0xe0ffffff, 0xe0000000
 982 };
 983
 984 static const u32 bonaire_golden_common_registers[] =
 985 {
 986         0xc770, 0xffffffff, 0x00000800,
 987         0xc774, 0xffffffff, 0x00000800,
 988         0xc798, 0xffffffff, 0x00007fbf,
 989         0xc79c, 0xffffffff, 0x00007faf
 990 };
 991
 992 static const u32 bonaire_golden_registers[] =
 993 {
 994         0x3354, 0x00000333, 0x00000333,
 995         0x3350, 0x000c0fc0, 0x00040200,
 996         0x9a10, 0x00010000, 0x00058208,
 997         0x3c000, 0xffff1fff, 0x00140000,
 998         0x3c200, 0xfdfc0fff, 0x00000100,
 999         0x3c234, 0x40000000, 0x40000200,
1000         0x9830, 0xffffffff, 0x00000000,
1001         0x9834, 0xf00fffff, 0x00000400,
1002         0x9838, 0x0002021c, 0x00020200,
1003         0xc78, 0x00000080, 0x00000000,
1004         0x5bb0, 0x000000f0, 0x00000070,
1005         0x5bc0, 0xf0311fff, 0x80300000,
1006         0x98f8, 0x73773777, 0x12010001,
1007         0x350c, 0x00810000, 0x408af000,
1008         0x7030, 0x31000111, 0x00000011,
1009         0x2f48, 0x73773777, 0x12010001,
1010         0x220c, 0x00007fb6, 0x0021a1b1,
1011         0x2210, 0x00007fb6, 0x002021b1,
1012         0x2180, 0x00007fb6, 0x00002191,
1013         0x2218, 0x00007fb6, 0x002121b1,
1014         0x221c, 0x00007fb6, 0x002021b1,
1015         0x21dc, 0x00007fb6, 0x00002191,
1016         0x21e0, 0x00007fb6, 0x00002191,
1017         0x3628, 0x0000003f, 0x0000000a,
1018         0x362c, 0x0000003f, 0x0000000a,
1019         0x2ae4, 0x00073ffe, 0x000022a2,
1020         0x240c, 0x000007ff, 0x00000000,
1021         0x8a14, 0xf000003f, 0x00000007,
1022         0x8bf0, 0x00002001, 0x00000001,
1023         0x8b24, 0xffffffff, 0x00ffffff,
1024         0x30a04, 0x0000ff0f, 0x00000000,
1025         0x28a4c, 0x07ffffff, 0x06000000,
1026         0x4d8, 0x00000fff, 0x00000100,
1027         0x3e78, 0x00000001, 0x00000002,
1028         0x9100, 0x03000000, 0x0362c688,
1029         0x8c00, 0x000000ff, 0x00000001,
1030         0xe40, 0x00001fff, 0x00001fff,
1031         0x9060, 0x0000007f, 0x00000020,
1032         0x9508, 0x00010000, 0x00010000,
1033         0xac14, 0x000003ff, 0x000000f3,
1034         0xac0c, 0xffffffff, 0x00001032
1035 };
1036
1037 static const u32 bonaire_mgcg_cgcg_init[] =
1038 {
1039         0xc420, 0xffffffff, 0xfffffffc,
1040         0x30800, 0xffffffff, 0xe0000000,
1041         0x3c2a0, 0xffffffff, 0x00000100,
1042         0x3c208, 0xffffffff, 0x00000100,
1043         0x3c2c0, 0xffffffff, 0xc0000100,
1044         0x3c2c8, 0xffffffff, 0xc0000100,
1045         0x3c2c4, 0xffffffff, 0xc0000100,
1046         0x55e4, 0xffffffff, 0x00600100,
1047         0x3c280, 0xffffffff, 0x00000100,
1048         0x3c214, 0xffffffff, 0x06000100,
1049         0x3c220, 0xffffffff, 0x00000100,
1050         0x3c218, 0xffffffff, 0x06000100,
1051         0x3c204, 0xffffffff, 0x00000100,
1052         0x3c2e0, 0xffffffff, 0x00000100,
1053         0x3c224, 0xffffffff, 0x00000100,
1054         0x3c200, 0xffffffff, 0x00000100,
1055         0x3c230, 0xffffffff, 0x00000100,
1056         0x3c234, 0xffffffff, 0x00000100,
1057         0x3c250, 0xffffffff, 0x00000100,
1058         0x3c254, 0xffffffff, 0x00000100,
1059         0x3c258, 0xffffffff, 0x00000100,
1060         0x3c25c, 0xffffffff, 0x00000100,
1061         0x3c260, 0xffffffff, 0x00000100,
1062         0x3c27c, 0xffffffff, 0x00000100,
1063         0x3c278, 0xffffffff, 0x00000100,
1064         0x3c210, 0xffffffff, 0x06000100,
1065         0x3c290, 0xffffffff, 0x00000100,
1066         0x3c274, 0xffffffff, 0x00000100,
1067         0x3c2b4, 0xffffffff, 0x00000100,
1068         0x3c2b0, 0xffffffff, 0x00000100,
1069         0x3c270, 0xffffffff, 0x00000100,
1070         0x30800, 0xffffffff, 0xe0000000,
1071         0x3c020, 0xffffffff, 0x00010000,
1072         0x3c024, 0xffffffff, 0x00030002,
1073         0x3c028, 0xffffffff, 0x00040007,
1074         0x3c02c, 0xffffffff, 0x00060005,
1075         0x3c030, 0xffffffff, 0x00090008,
1076         0x3c034, 0xffffffff, 0x00010000,
1077         0x3c038, 0xffffffff, 0x00030002,
1078         0x3c03c, 0xffffffff, 0x00040007,
1079         0x3c040, 0xffffffff, 0x00060005,
1080         0x3c044, 0xffffffff, 0x00090008,
1081         0x3c048, 0xffffffff, 0x00010000,
1082         0x3c04c, 0xffffffff, 0x00030002,
1083         0x3c050, 0xffffffff, 0x00040007,
1084         0x3c054, 0xffffffff, 0x00060005,
1085         0x3c058, 0xffffffff, 0x00090008,
1086         0x3c05c, 0xffffffff, 0x00010000,
1087         0x3c060, 0xffffffff, 0x00030002,
1088         0x3c064, 0xffffffff, 0x00040007,
1089         0x3c068, 0xffffffff, 0x00060005,
1090         0x3c06c, 0xffffffff, 0x00090008,
1091         0x3c070, 0xffffffff, 0x00010000,
1092         0x3c074, 0xffffffff, 0x00030002,
1093         0x3c078, 0xffffffff, 0x00040007,
1094         0x3c07c, 0xffffffff, 0x00060005,
1095         0x3c080, 0xffffffff, 0x00090008,
1096         0x3c084, 0xffffffff, 0x00010000,
1097         0x3c088, 0xffffffff, 0x00030002,
1098         0x3c08c, 0xffffffff, 0x00040007,
1099         0x3c090, 0xffffffff, 0x00060005,
1100         0x3c094, 0xffffffff, 0x00090008,
1101         0x3c098, 0xffffffff, 0x00010000,
1102         0x3c09c, 0xffffffff, 0x00030002,
1103         0x3c0a0, 0xffffffff, 0x00040007,
1104         0x3c0a4, 0xffffffff, 0x00060005,
1105         0x3c0a8, 0xffffffff, 0x00090008,
1106         0x3c000, 0xffffffff, 0x96e00200,
1107         0x8708, 0xffffffff, 0x00900100,
1108         0xc424, 0xffffffff, 0x0020003f,
1109         0x38, 0xffffffff, 0x0140001c,
1110         0x3c, 0x000f0000, 0x000f0000,
1111         0x220, 0xffffffff, 0xC060000C,
1112         0x224, 0xc0000fff, 0x00000100,
1113         0xf90, 0xffffffff, 0x00000100,
1114         0xf98, 0x00000101, 0x00000000,
1115         0x20a8, 0xffffffff, 0x00000104,
1116         0x55e4, 0xff000fff, 0x00000100,
1117         0x30cc, 0xc0000fff, 0x00000104,
1118         0xc1e4, 0x00000001, 0x00000001,
1119         0xd00c, 0xff000ff0, 0x00000100,
1120         0xd80c, 0xff000ff0, 0x00000100
1121 };
1122
1123 static const u32 spectre_golden_spm_registers[] =
1124 {
1125         0x30800, 0xe0ffffff, 0xe0000000
1126 };
1127
1128 static const u32 spectre_golden_common_registers[] =
1129 {
1130         0xc770, 0xffffffff, 0x00000800,
1131         0xc774, 0xffffffff, 0x00000800,
1132         0xc798, 0xffffffff, 0x00007fbf,
1133         0xc79c, 0xffffffff, 0x00007faf
1134 };
1135
1136 static const u32 spectre_golden_registers[] =
1137 {
1138         0x3c000, 0xffff1fff, 0x96940200,
1139         0x3c00c, 0xffff0001, 0xff000000,
1140         0x3c200, 0xfffc0fff, 0x00000100,
1141         0x6ed8, 0x00010101, 0x00010000,
1142         0x9834, 0xf00fffff, 0x00000400,
1143         0x9838, 0xfffffffc, 0x00020200,
1144         0x5bb0, 0x000000f0, 0x00000070,
1145         0x5bc0, 0xf0311fff, 0x80300000,
1146         0x98f8, 0x73773777, 0x12010001,
1147         0x9b7c, 0x00ff0000, 0x00fc0000,
1148         0x2f48, 0x73773777, 0x12010001,
1149         0x8a14, 0xf000003f, 0x00000007,
1150         0x8b24, 0xffffffff, 0x00ffffff,
1151         0x28350, 0x3f3f3fff, 0x00000082,
1152         0x28354, 0x0000003f, 0x00000000,
1153         0x3e78, 0x00000001, 0x00000002,
1154         0x913c, 0xffff03df, 0x00000004,
1155         0xc768, 0x00000008, 0x00000008,
1156         0x8c00, 0x000008ff, 0x00000800,
1157         0x9508, 0x00010000, 0x00010000,
1158         0xac0c, 0xffffffff, 0x54763210,
1159         0x214f8, 0x01ff01ff, 0x00000002,
1160         0x21498, 0x007ff800, 0x00200000,
1161         0x2015c, 0xffffffff, 0x00000f40,
1162         0x30934, 0xffffffff, 0x00000001
1163 };
1164
1165 static const u32 spectre_mgcg_cgcg_init[] =
1166 {
1167         0xc420, 0xffffffff, 0xfffffffc,
1168         0x30800, 0xffffffff, 0xe0000000,
1169         0x3c2a0, 0xffffffff, 0x00000100,
1170         0x3c208, 0xffffffff, 0x00000100,
1171         0x3c2c0, 0xffffffff, 0x00000100,
1172         0x3c2c8, 0xffffffff, 0x00000100,
1173         0x3c2c4, 0xffffffff, 0x00000100,
1174         0x55e4, 0xffffffff, 0x00600100,
1175         0x3c280, 0xffffffff, 0x00000100,
1176         0x3c214, 0xffffffff, 0x06000100,
1177         0x3c220, 0xffffffff, 0x00000100,
1178         0x3c218, 0xffffffff, 0x06000100,
1179         0x3c204, 0xffffffff, 0x00000100,
1180         0x3c2e0, 0xffffffff, 0x00000100,
1181         0x3c224, 0xffffffff, 0x00000100,
1182         0x3c200, 0xffffffff, 0x00000100,
1183         0x3c230, 0xffffffff, 0x00000100,
1184         0x3c234, 0xffffffff, 0x00000100,
1185         0x3c250, 0xffffffff, 0x00000100,
1186         0x3c254, 0xffffffff, 0x00000100,
1187         0x3c258, 0xffffffff, 0x00000100,
1188         0x3c25c, 0xffffffff, 0x00000100,
1189         0x3c260, 0xffffffff, 0x00000100,
1190         0x3c27c, 0xffffffff, 0x00000100,
1191         0x3c278, 0xffffffff, 0x00000100,
1192         0x3c210, 0xffffffff, 0x06000100,
1193         0x3c290, 0xffffffff, 0x00000100,
1194         0x3c274, 0xffffffff, 0x00000100,
1195         0x3c2b4, 0xffffffff, 0x00000100,
1196         0x3c2b0, 0xffffffff, 0x00000100,
1197         0x3c270, 0xffffffff, 0x00000100,
1198         0x30800, 0xffffffff, 0xe0000000,
1199         0x3c020, 0xffffffff, 0x00010000,
1200         0x3c024, 0xffffffff, 0x00030002,
1201         0x3c028, 0xffffffff, 0x00040007,
1202         0x3c02c, 0xffffffff, 0x00060005,
1203         0x3c030, 0xffffffff, 0x00090008,
1204         0x3c034, 0xffffffff, 0x00010000,
1205         0x3c038, 0xffffffff, 0x00030002,
1206         0x3c03c, 0xffffffff, 0x00040007,
1207         0x3c040, 0xffffffff, 0x00060005,
1208         0x3c044, 0xffffffff, 0x00090008,
1209         0x3c048, 0xffffffff, 0x00010000,
1210         0x3c04c, 0xffffffff, 0x00030002,
1211         0x3c050, 0xffffffff, 0x00040007,
1212         0x3c054, 0xffffffff, 0x00060005,
1213         0x3c058, 0xffffffff, 0x00090008,
1214         0x3c05c, 0xffffffff, 0x00010000,
1215         0x3c060, 0xffffffff, 0x00030002,
1216         0x3c064, 0xffffffff, 0x00040007,
1217         0x3c068, 0xffffffff, 0x00060005,
1218         0x3c06c, 0xffffffff, 0x00090008,
1219         0x3c070, 0xffffffff, 0x00010000,
1220         0x3c074, 0xffffffff, 0x00030002,
1221         0x3c078, 0xffffffff, 0x00040007,
1222         0x3c07c, 0xffffffff, 0x00060005,
1223         0x3c080, 0xffffffff, 0x00090008,
1224         0x3c084, 0xffffffff, 0x00010000,
1225         0x3c088, 0xffffffff, 0x00030002,
1226         0x3c08c, 0xffffffff, 0x00040007,
1227         0x3c090, 0xffffffff, 0x00060005,
1228         0x3c094, 0xffffffff, 0x00090008,
1229         0x3c098, 0xffffffff, 0x00010000,
1230         0x3c09c, 0xffffffff, 0x00030002,
1231         0x3c0a0, 0xffffffff, 0x00040007,
1232         0x3c0a4, 0xffffffff, 0x00060005,
1233         0x3c0a8, 0xffffffff, 0x00090008,
1234         0x3c0ac, 0xffffffff, 0x00010000,
1235         0x3c0b0, 0xffffffff, 0x00030002,
1236         0x3c0b4, 0xffffffff, 0x00040007,
1237         0x3c0b8, 0xffffffff, 0x00060005,
1238         0x3c0bc, 0xffffffff, 0x00090008,
1239         0x3c000, 0xffffffff, 0x96e00200,
1240         0x8708, 0xffffffff, 0x00900100,
1241         0xc424, 0xffffffff, 0x0020003f,
1242         0x38, 0xffffffff, 0x0140001c,
1243         0x3c, 0x000f0000, 0x000f0000,
1244         0x220, 0xffffffff, 0xC060000C,
1245         0x224, 0xc0000fff, 0x00000100,
1246         0xf90, 0xffffffff, 0x00000100,
1247         0xf98, 0x00000101, 0x00000000,
1248         0x20a8, 0xffffffff, 0x00000104,
1249         0x55e4, 0xff000fff, 0x00000100,
1250         0x30cc, 0xc0000fff, 0x00000104,
1251         0xc1e4, 0x00000001, 0x00000001,
1252         0xd00c, 0xff000ff0, 0x00000100,
1253         0xd80c, 0xff000ff0, 0x00000100
1254 };
1255
1256 static const u32 kalindi_golden_spm_registers[] =
1257 {
1258         0x30800, 0xe0ffffff, 0xe0000000
1259 };
1260
1261 static const u32 kalindi_golden_common_registers[] =
1262 {
1263         0xc770, 0xffffffff, 0x00000800,
1264         0xc774, 0xffffffff, 0x00000800,
1265         0xc798, 0xffffffff, 0x00007fbf,
1266         0xc79c, 0xffffffff, 0x00007faf
1267 };
1268
1269 static const u32 kalindi_golden_registers[] =
1270 {
1271         0x3c000, 0xffffdfff, 0x6e944040,
1272         0x55e4, 0xff607fff, 0xfc000100,
1273         0x3c220, 0xff000fff, 0x00000100,
1274         0x3c224, 0xff000fff, 0x00000100,
1275         0x3c200, 0xfffc0fff, 0x00000100,
1276         0x6ed8, 0x00010101, 0x00010000,
1277         0x9830, 0xffffffff, 0x00000000,
1278         0x9834, 0xf00fffff, 0x00000400,
1279         0x5bb0, 0x000000f0, 0x00000070,
1280         0x5bc0, 0xf0311fff, 0x80300000,
1281         0x98f8, 0x73773777, 0x12010001,
1282         0x98fc, 0xffffffff, 0x00000010,
1283         0x9b7c, 0x00ff0000, 0x00fc0000,
1284         0x8030, 0x00001f0f, 0x0000100a,
1285         0x2f48, 0x73773777, 0x12010001,
1286         0x2408, 0x000fffff, 0x000c007f,
1287         0x8a14, 0xf000003f, 0x00000007,
1288         0x8b24, 0x3fff3fff, 0x00ffcfff,
1289         0x30a04, 0x0000ff0f, 0x00000000,
1290         0x28a4c, 0x07ffffff, 0x06000000,
1291         0x4d8, 0x00000fff, 0x00000100,
1292         0x3e78, 0x00000001, 0x00000002,
1293         0xc768, 0x00000008, 0x00000008,
1294         0x8c00, 0x000000ff, 0x00000003,
1295         0x214f8, 0x01ff01ff, 0x00000002,
1296         0x21498, 0x007ff800, 0x00200000,
1297         0x2015c, 0xffffffff, 0x00000f40,
1298         0x88c4, 0x001f3ae3, 0x00000082,
1299         0x88d4, 0x0000001f, 0x00000010,
1300         0x30934, 0xffffffff, 0x00000000
1301 };
1302
1303 static const u32 kalindi_mgcg_cgcg_init[] =
1304 {
1305         0xc420, 0xffffffff, 0xfffffffc,
1306         0x30800, 0xffffffff, 0xe0000000,
1307         0x3c2a0, 0xffffffff, 0x00000100,
1308         0x3c208, 0xffffffff, 0x00000100,
1309         0x3c2c0, 0xffffffff, 0x00000100,
1310         0x3c2c8, 0xffffffff, 0x00000100,
1311         0x3c2c4, 0xffffffff, 0x00000100,
1312         0x55e4, 0xffffffff, 0x00600100,
1313         0x3c280, 0xffffffff, 0x00000100,
1314         0x3c214, 0xffffffff, 0x06000100,
1315         0x3c220, 0xffffffff, 0x00000100,
1316         0x3c218, 0xffffffff, 0x06000100,
1317         0x3c204, 0xffffffff, 0x00000100,
1318         0x3c2e0, 0xffffffff, 0x00000100,
1319         0x3c224, 0xffffffff, 0x00000100,
1320         0x3c200, 0xffffffff, 0x00000100,
1321         0x3c230, 0xffffffff, 0x00000100,
1322         0x3c234, 0xffffffff, 0x00000100,
1323         0x3c250, 0xffffffff, 0x00000100,
1324         0x3c254, 0xffffffff, 0x00000100,
1325         0x3c258, 0xffffffff, 0x00000100,
1326         0x3c25c, 0xffffffff, 0x00000100,
1327         0x3c260, 0xffffffff, 0x00000100,
1328         0x3c27c, 0xffffffff, 0x00000100,
1329         0x3c278, 0xffffffff, 0x00000100,
1330         0x3c210, 0xffffffff, 0x06000100,
1331         0x3c290, 0xffffffff, 0x00000100,
1332         0x3c274, 0xffffffff, 0x00000100,
1333         0x3c2b4, 0xffffffff, 0x00000100,
1334         0x3c2b0, 0xffffffff, 0x00000100,
1335         0x3c270, 0xffffffff, 0x00000100,
1336         0x30800, 0xffffffff, 0xe0000000,
1337         0x3c020, 0xffffffff, 0x00010000,
1338         0x3c024, 0xffffffff, 0x00030002,
1339         0x3c028, 0xffffffff, 0x00040007,
1340         0x3c02c, 0xffffffff, 0x00060005,
1341         0x3c030, 0xffffffff, 0x00090008,
1342         0x3c034, 0xffffffff, 0x00010000,
1343         0x3c038, 0xffffffff, 0x00030002,
1344         0x3c03c, 0xffffffff, 0x00040007,
1345         0x3c040, 0xffffffff, 0x00060005,
1346         0x3c044, 0xffffffff, 0x00090008,
1347         0x3c000, 0xffffffff, 0x96e00200,
1348         0x8708, 0xffffffff, 0x00900100,
1349         0xc424, 0xffffffff, 0x0020003f,
1350         0x38, 0xffffffff, 0x0140001c,
1351         0x3c, 0x000f0000, 0x000f0000,
1352         0x220, 0xffffffff, 0xC060000C,
1353         0x224, 0xc0000fff, 0x00000100,
1354         0x20a8, 0xffffffff, 0x00000104,
1355         0x55e4, 0xff000fff, 0x00000100,
1356         0x30cc, 0xc0000fff, 0x00000104,
1357         0xc1e4, 0x00000001, 0x00000001,
1358         0xd00c, 0xff000ff0, 0x00000100,
1359         0xd80c, 0xff000ff0, 0x00000100
1360 };
1361
1362 static const u32 hawaii_golden_spm_registers[] =
1363 {
1364         0x30800, 0xe0ffffff, 0xe0000000
1365 };
1366
1367 static const u32 hawaii_golden_common_registers[] =
1368 {
1369         0x30800, 0xffffffff, 0xe0000000,
1370         0x28350, 0xffffffff, 0x3a00161a,
1371         0x28354, 0xffffffff, 0x0000002e,
1372         0x9a10, 0xffffffff, 0x00018208,
1373         0x98f8, 0xffffffff, 0x12011003
1374 };
1375
1376 static const u32 hawaii_golden_registers[] =
1377 {
1378         0x3354, 0x00000333, 0x00000333,
1379         0x9a10, 0x00010000, 0x00058208,
1380         0x9830, 0xffffffff, 0x00000000,
1381         0x9834, 0xf00fffff, 0x00000400,
1382         0x9838, 0x0002021c, 0x00020200,
1383         0xc78, 0x00000080, 0x00000000,
1384         0x5bb0, 0x000000f0, 0x00000070,
1385         0x5bc0, 0xf0311fff, 0x80300000,
1386         0x350c, 0x00810000, 0x408af000,
1387         0x7030, 0x31000111, 0x00000011,
1388         0x2f48, 0x73773777, 0x12010001,
1389         0x2120, 0x0000007f, 0x0000001b,
1390         0x21dc, 0x00007fb6, 0x00002191,
1391         0x3628, 0x0000003f, 0x0000000a,
1392         0x362c, 0x0000003f, 0x0000000a,
1393         0x2ae4, 0x00073ffe, 0x000022a2,
1394         0x240c, 0x000007ff, 0x00000000,
1395         0x8bf0, 0x00002001, 0x00000001,
1396         0x8b24, 0xffffffff, 0x00ffffff,
1397         0x30a04, 0x0000ff0f, 0x00000000,
1398         0x28a4c, 0x07ffffff, 0x06000000,
1399         0x3e78, 0x00000001, 0x00000002,
1400         0xc768, 0x00000008, 0x00000008,
1401         0xc770, 0x00000f00, 0x00000800,
1402         0xc774, 0x00000f00, 0x00000800,
1403         0xc798, 0x00ffffff, 0x00ff7fbf,
1404         0xc79c, 0x00ffffff, 0x00ff7faf,
1405         0x8c00, 0x000000ff, 0x00000800,
1406         0xe40, 0x00001fff, 0x00001fff,
1407         0x9060, 0x0000007f, 0x00000020,
1408         0x9508, 0x00010000, 0x00010000,
1409         0xae00, 0x00100000, 0x000ff07c,
1410         0xac14, 0x000003ff, 0x0000000f,
1411         0xac10, 0xffffffff, 0x7564fdec,
1412         0xac0c, 0xffffffff, 0x3120b9a8,
1413         0xac08, 0x20000000, 0x0f9c0000
1414 };
1415
1416 static const u32 hawaii_mgcg_cgcg_init[] =
1417 {
1418         0xc420, 0xffffffff, 0xfffffffd,
1419         0x30800, 0xffffffff, 0xe0000000,
1420         0x3c2a0, 0xffffffff, 0x00000100,
1421         0x3c208, 0xffffffff, 0x00000100,
1422         0x3c2c0, 0xffffffff, 0x00000100,
1423         0x3c2c8, 0xffffffff, 0x00000100,
1424         0x3c2c4, 0xffffffff, 0x00000100,
1425         0x55e4, 0xffffffff, 0x00200100,
1426         0x3c280, 0xffffffff, 0x00000100,
1427         0x3c214, 0xffffffff, 0x06000100,
1428         0x3c220, 0xffffffff, 0x00000100,
1429         0x3c218, 0xffffffff, 0x06000100,
1430         0x3c204, 0xffffffff, 0x00000100,
1431         0x3c2e0, 0xffffffff, 0x00000100,
1432         0x3c224, 0xffffffff, 0x00000100,
1433         0x3c200, 0xffffffff, 0x00000100,
1434         0x3c230, 0xffffffff, 0x00000100,
1435         0x3c234, 0xffffffff, 0x00000100,
1436         0x3c250, 0xffffffff, 0x00000100,
1437         0x3c254, 0xffffffff, 0x00000100,
1438         0x3c258, 0xffffffff, 0x00000100,
1439         0x3c25c, 0xffffffff, 0x00000100,
1440         0x3c260, 0xffffffff, 0x00000100,
1441         0x3c27c, 0xffffffff, 0x00000100,
1442         0x3c278, 0xffffffff, 0x00000100,
1443         0x3c210, 0xffffffff, 0x06000100,
1444         0x3c290, 0xffffffff, 0x00000100,
1445         0x3c274, 0xffffffff, 0x00000100,
1446         0x3c2b4, 0xffffffff, 0x00000100,
1447         0x3c2b0, 0xffffffff, 0x00000100,
1448         0x3c270, 0xffffffff, 0x00000100,
1449         0x30800, 0xffffffff, 0xe0000000,
1450         0x3c020, 0xffffffff, 0x00010000,
1451         0x3c024, 0xffffffff, 0x00030002,
1452         0x3c028, 0xffffffff, 0x00040007,
1453         0x3c02c, 0xffffffff, 0x00060005,
1454         0x3c030, 0xffffffff, 0x00090008,
1455         0x3c034, 0xffffffff, 0x00010000,
1456         0x3c038, 0xffffffff, 0x00030002,
1457         0x3c03c, 0xffffffff, 0x00040007,
1458         0x3c040, 0xffffffff, 0x00060005,
1459         0x3c044, 0xffffffff, 0x00090008,
1460         0x3c048, 0xffffffff, 0x00010000,
1461         0x3c04c, 0xffffffff, 0x00030002,
1462         0x3c050, 0xffffffff, 0x00040007,
1463         0x3c054, 0xffffffff, 0x00060005,
1464         0x3c058, 0xffffffff, 0x00090008,
1465         0x3c05c, 0xffffffff, 0x00010000,
1466         0x3c060, 0xffffffff, 0x00030002,
1467         0x3c064, 0xffffffff, 0x00040007,
1468         0x3c068, 0xffffffff, 0x00060005,
1469         0x3c06c, 0xffffffff, 0x00090008,
1470         0x3c070, 0xffffffff, 0x00010000,
1471         0x3c074, 0xffffffff, 0x00030002,
1472         0x3c078, 0xffffffff, 0x00040007,
1473         0x3c07c, 0xffffffff, 0x00060005,
1474         0x3c080, 0xffffffff, 0x00090008,
1475         0x3c084, 0xffffffff, 0x00010000,
1476         0x3c088, 0xffffffff, 0x00030002,
1477         0x3c08c, 0xffffffff, 0x00040007,
1478         0x3c090, 0xffffffff, 0x00060005,
1479         0x3c094, 0xffffffff, 0x00090008,
1480         0x3c098, 0xffffffff, 0x00010000,
1481         0x3c09c, 0xffffffff, 0x00030002,
1482         0x3c0a0, 0xffffffff, 0x00040007,
1483         0x3c0a4, 0xffffffff, 0x00060005,
1484         0x3c0a8, 0xffffffff, 0x00090008,
1485         0x3c0ac, 0xffffffff, 0x00010000,
1486         0x3c0b0, 0xffffffff, 0x00030002,
1487         0x3c0b4, 0xffffffff, 0x00040007,
1488         0x3c0b8, 0xffffffff, 0x00060005,
1489         0x3c0bc, 0xffffffff, 0x00090008,
1490         0x3c0c0, 0xffffffff, 0x00010000,
1491         0x3c0c4, 0xffffffff, 0x00030002,
1492         0x3c0c8, 0xffffffff, 0x00040007,
1493         0x3c0cc, 0xffffffff, 0x00060005,
1494         0x3c0d0, 0xffffffff, 0x00090008,
1495         0x3c0d4, 0xffffffff, 0x00010000,
1496         0x3c0d8, 0xffffffff, 0x00030002,
1497         0x3c0dc, 0xffffffff, 0x00040007,
1498         0x3c0e0, 0xffffffff, 0x00060005,
1499         0x3c0e4, 0xffffffff, 0x00090008,
1500         0x3c0e8, 0xffffffff, 0x00010000,
1501         0x3c0ec, 0xffffffff, 0x00030002,
1502         0x3c0f0, 0xffffffff, 0x00040007,
1503         0x3c0f4, 0xffffffff, 0x00060005,
1504         0x3c0f8, 0xffffffff, 0x00090008,
1505         0xc318, 0xffffffff, 0x00020200,
1506         0x3350, 0xffffffff, 0x00000200,
1507         0x15c0, 0xffffffff, 0x00000400,
1508         0x55e8, 0xffffffff, 0x00000000,
1509         0x2f50, 0xffffffff, 0x00000902,
1510         0x3c000, 0xffffffff, 0x96940200,
1511         0x8708, 0xffffffff, 0x00900100,
1512         0xc424, 0xffffffff, 0x0020003f,
1513         0x38, 0xffffffff, 0x0140001c,
1514         0x3c, 0x000f0000, 0x000f0000,
1515         0x220, 0xffffffff, 0xc060000c,
1516         0x224, 0xc0000fff, 0x00000100,
1517         0xf90, 0xffffffff, 0x00000100,
1518         0xf98, 0x00000101, 0x00000000,
1519         0x20a8, 0xffffffff, 0x00000104,
1520         0x55e4, 0xff000fff, 0x00000100,
1521         0x30cc, 0xc0000fff, 0x00000104,
1522         0xc1e4, 0x00000001, 0x00000001,
1523         0xd00c, 0xff000ff0, 0x00000100,
1524         0xd80c, 0xff000ff0, 0x00000100
1525 };
1526
1527 static const u32 godavari_golden_registers[] =
1528 {
1529         0x55e4, 0xff607fff, 0xfc000100,
1530         0x6ed8, 0x00010101, 0x00010000,
1531         0x9830, 0xffffffff, 0x00000000,
1532         0x98302, 0xf00fffff, 0x00000400,
1533         0x6130, 0xffffffff, 0x00010000,
1534         0x5bb0, 0x000000f0, 0x00000070,
1535         0x5bc0, 0xf0311fff, 0x80300000,
1536         0x98f8, 0x73773777, 0x12010001,
1537         0x98fc, 0xffffffff, 0x00000010,
1538         0x8030, 0x00001f0f, 0x0000100a,
1539         0x2f48, 0x73773777, 0x12010001,
1540         0x2408, 0x000fffff, 0x000c007f,
1541         0x8a14, 0xf000003f, 0x00000007,
1542         0x8b24, 0xffffffff, 0x00ff0fff,
1543         0x30a04, 0x0000ff0f, 0x00000000,
1544         0x28a4c, 0x07ffffff, 0x06000000,
1545         0x4d8, 0x00000fff, 0x00000100,
1546         0xd014, 0x00010000, 0x00810001,
1547         0xd814, 0x00010000, 0x00810001,
1548         0x3e78, 0x00000001, 0x00000002,
1549         0xc768, 0x00000008, 0x00000008,
1550         0xc770, 0x00000f00, 0x00000800,
1551         0xc774, 0x00000f00, 0x00000800,
1552         0xc798, 0x00ffffff, 0x00ff7fbf,
1553         0xc79c, 0x00ffffff, 0x00ff7faf,
1554         0x8c00, 0x000000ff, 0x00000001,
1555         0x214f8, 0x01ff01ff, 0x00000002,
1556         0x21498, 0x007ff800, 0x00200000,
1557         0x2015c, 0xffffffff, 0x00000f40,
1558         0x88c4, 0x001f3ae3, 0x00000082,
1559         0x88d4, 0x0000001f, 0x00000010,
1560         0x30934, 0xffffffff, 0x00000000
1561 };
1562
1563
1564 static void cik_init_golden_registers(struct radeon_device *rdev)
1565 {
1566         switch (rdev->family) {
1567         case CHIP_BONAIRE:
1568                 radeon_program_register_sequence(rdev,
1569                                                  bonaire_mgcg_cgcg_init,
1570                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571                 radeon_program_register_sequence(rdev,
1572                                                  bonaire_golden_registers,
1573                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574                 radeon_program_register_sequence(rdev,
1575                                                  bonaire_golden_common_registers,
1576                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577                 radeon_program_register_sequence(rdev,
1578                                                  bonaire_golden_spm_registers,
1579                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580                 break;
1581         case CHIP_KABINI:
1582                 radeon_program_register_sequence(rdev,
1583                                                  kalindi_mgcg_cgcg_init,
1584                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585                 radeon_program_register_sequence(rdev,
1586                                                  kalindi_golden_registers,
1587                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588                 radeon_program_register_sequence(rdev,
1589                                                  kalindi_golden_common_registers,
1590                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591                 radeon_program_register_sequence(rdev,
1592                                                  kalindi_golden_spm_registers,
1593                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594                 break;
1595         case CHIP_MULLINS:
1596                 radeon_program_register_sequence(rdev,
1597                                                  kalindi_mgcg_cgcg_init,
1598                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599                 radeon_program_register_sequence(rdev,
1600                                                  godavari_golden_registers,
1601                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
1602                 radeon_program_register_sequence(rdev,
1603                                                  kalindi_golden_common_registers,
1604                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605                 radeon_program_register_sequence(rdev,
1606                                                  kalindi_golden_spm_registers,
1607                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608                 break;
1609         case CHIP_KAVERI:
1610                 radeon_program_register_sequence(rdev,
1611                                                  spectre_mgcg_cgcg_init,
1612                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613                 radeon_program_register_sequence(rdev,
1614                                                  spectre_golden_registers,
1615                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1616                 radeon_program_register_sequence(rdev,
1617                                                  spectre_golden_common_registers,
1618                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619                 radeon_program_register_sequence(rdev,
1620                                                  spectre_golden_spm_registers,
1621                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622                 break;
1623         case CHIP_HAWAII:
1624                 radeon_program_register_sequence(rdev,
1625                                                  hawaii_mgcg_cgcg_init,
1626                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627                 radeon_program_register_sequence(rdev,
1628                                                  hawaii_golden_registers,
1629                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630                 radeon_program_register_sequence(rdev,
1631                                                  hawaii_golden_common_registers,
1632                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633                 radeon_program_register_sequence(rdev,
1634                                                  hawaii_golden_spm_registers,
1635                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636                 break;
1637         default:
1638                 break;
1639         }
1640 }
1641
1642 /**
1643  * cik_get_xclk - get the xclk
1644  *
1645  * @rdev: radeon_device pointer
1646  *
1647  * Returns the reference clock used by the gfx engine
1648  * (CIK).
1649  */
1650 u32 cik_get_xclk(struct radeon_device *rdev)
1651 {
1652         u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654         if (rdev->flags & RADEON_IS_IGP) {
1655                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656                         return reference_clock / 2;
1657         } else {
1658                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659                         return reference_clock / 4;
1660         }
1661         return reference_clock;
1662 }
1663
1664 /**
1665  * cik_mm_rdoorbell - read a doorbell dword
1666  *
1667  * @rdev: radeon_device pointer
1668  * @index: doorbell index
1669  *
1670  * Returns the value in the doorbell aperture at the
1671  * requested doorbell index (CIK).
1672  */
1673 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1674 {
1675         if (index < rdev->doorbell.num_doorbells) {
1676                 return readl(rdev->doorbell.ptr + index);
1677         } else {
1678                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1679                 return 0;
1680         }
1681 }
1682
1683 /**
1684  * cik_mm_wdoorbell - write a doorbell dword
1685  *
1686  * @rdev: radeon_device pointer
1687  * @index: doorbell index
1688  * @v: value to write
1689  *
1690  * Writes @v to the doorbell aperture at the
1691  * requested doorbell index (CIK).
1692  */
1693 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1694 {
1695         if (index < rdev->doorbell.num_doorbells) {
1696                 writel(v, rdev->doorbell.ptr + index);
1697         } else {
1698                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1699         }
1700 }
1701
1702 #define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705 {
1706         {0x00000070, 0x04400000},
1707         {0x00000071, 0x80c01803},
1708         {0x00000072, 0x00004004},
1709         {0x00000073, 0x00000100},
1710         {0x00000074, 0x00ff0000},
1711         {0x00000075, 0x34000000},
1712         {0x00000076, 0x08000014},
1713         {0x00000077, 0x00cc08ec},
1714         {0x00000078, 0x00000400},
1715         {0x00000079, 0x00000000},
1716         {0x0000007a, 0x04090000},
1717         {0x0000007c, 0x00000000},
1718         {0x0000007e, 0x4408a8e8},
1719         {0x0000007f, 0x00000304},
1720         {0x00000080, 0x00000000},
1721         {0x00000082, 0x00000001},
1722         {0x00000083, 0x00000002},
1723         {0x00000084, 0xf3e4f400},
1724         {0x00000085, 0x052024e3},
1725         {0x00000087, 0x00000000},
1726         {0x00000088, 0x01000000},
1727         {0x0000008a, 0x1c0a0000},
1728         {0x0000008b, 0xff010000},
1729         {0x0000008d, 0xffffefff},
1730         {0x0000008e, 0xfff3efff},
1731         {0x0000008f, 0xfff3efbf},
1732         {0x00000092, 0xf7ffffff},
1733         {0x00000093, 0xffffff7f},
1734         {0x00000095, 0x00101101},
1735         {0x00000096, 0x00000fff},
1736         {0x00000097, 0x00116fff},
1737         {0x00000098, 0x60010000},
1738         {0x00000099, 0x10010000},
1739         {0x0000009a, 0x00006000},
1740         {0x0000009b, 0x00001000},
1741         {0x0000009f, 0x00b48000}
1742 };
1743
1744 #define HAWAII_IO_MC_REGS_SIZE 22
1745
1746 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747 {
1748         {0x0000007d, 0x40000000},
1749         {0x0000007e, 0x40180304},
1750         {0x0000007f, 0x0000ff00},
1751         {0x00000081, 0x00000000},
1752         {0x00000083, 0x00000800},
1753         {0x00000086, 0x00000000},
1754         {0x00000087, 0x00000100},
1755         {0x00000088, 0x00020100},
1756         {0x00000089, 0x00000000},
1757         {0x0000008b, 0x00040000},
1758         {0x0000008c, 0x00000100},
1759         {0x0000008e, 0xff010000},
1760         {0x00000090, 0xffffefff},
1761         {0x00000091, 0xfff3efff},
1762         {0x00000092, 0xfff3efbf},
1763         {0x00000093, 0xf7ffffff},
1764         {0x00000094, 0xffffff7f},
1765         {0x00000095, 0x00000fff},
1766         {0x00000096, 0x00116fff},
1767         {0x00000097, 0x60010000},
1768         {0x00000098, 0x10010000},
1769         {0x0000009f, 0x00c79000}
1770 };
1771
1772
1773 /**
1774  * cik_srbm_select - select specific register instances
1775  *
1776  * @rdev: radeon_device pointer
1777  * @me: selected ME (micro engine)
1778  * @pipe: pipe
1779  * @queue: queue
1780  * @vmid: VMID
1781  *
1782  * Switches the currently active registers instances.  Some
1783  * registers are instanced per VMID, others are instanced per
1784  * me/pipe/queue combination.
1785  */
1786 static void cik_srbm_select(struct radeon_device *rdev,
1787                             u32 me, u32 pipe, u32 queue, u32 vmid)
1788 {
1789         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790                              MEID(me & 0x3) |
1791                              VMID(vmid & 0xf) |
1792                              QUEUEID(queue & 0x7));
1793         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794 }
1795
1796 /* ucode loading */
1797 /**
1798  * ci_mc_load_microcode - load MC ucode into the hw
1799  *
1800  * @rdev: radeon_device pointer
1801  *
1802  * Load the GDDR MC ucode into the hw (CIK).
1803  * Returns 0 on success, error on failure.
1804  */
1805 int ci_mc_load_microcode(struct radeon_device *rdev)
1806 {
1807         const __be32 *fw_data = NULL;
1808         const __le32 *new_fw_data = NULL;
1809         u32 running, blackout = 0;
1810         u32 *io_mc_regs = NULL;
1811         const __le32 *new_io_mc_regs = NULL;
1812         int i, regs_size, ucode_size;
1813
1814         if (!rdev->mc_fw)
1815                 return -EINVAL;
1816
1817         if (rdev->new_fw) {
1818                 const struct mc_firmware_header_v1_0 *hdr =
1819                         (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1820
1821                 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823                 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824                 new_io_mc_regs = (const __le32 *)
1825                         (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827                 new_fw_data = (const __le32 *)
1828                         (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829         } else {
1830                 ucode_size = rdev->mc_fw->size / 4;
1831
1832                 switch (rdev->family) {
1833                 case CHIP_BONAIRE:
1834                         io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835                         regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836                         break;
1837                 case CHIP_HAWAII:
1838                         io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839                         regs_size = HAWAII_IO_MC_REGS_SIZE;
1840                         break;
1841                 default:
1842                         return -EINVAL;
1843                 }
1844                 fw_data = (const __be32 *)rdev->mc_fw->data;
1845         }
1846
1847         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849         if (running == 0) {
1850                 if (running) {
1851                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853                 }
1854
1855                 /* reset the engine and set to writable */
1856                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859                 /* load mc io regs */
1860                 for (i = 0; i < regs_size; i++) {
1861                         if (rdev->new_fw) {
1862                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863                                 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864                         } else {
1865                                 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866                                 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867                         }
1868                 }
1869                 /* load the MC ucode */
1870                 for (i = 0; i < ucode_size; i++) {
1871                         if (rdev->new_fw)
1872                                 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873                         else
1874                                 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875                 }
1876
1877                 /* put the engine back into the active state */
1878                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881
1882                 /* wait for training to complete */
1883                 for (i = 0; i < rdev->usec_timeout; i++) {
1884                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885                                 break;
1886                         udelay(1);
1887                 }
1888                 for (i = 0; i < rdev->usec_timeout; i++) {
1889                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890                                 break;
1891                         udelay(1);
1892                 }
1893
1894                 if (running)
1895                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896         }
1897
1898         return 0;
1899 }
1900
1901 /**
1902  * cik_init_microcode - load ucode images from disk
1903  *
1904  * @rdev: radeon_device pointer
1905  *
1906  * Use the firmware interface to load the ucode images into
1907  * the driver (not loaded into hw).
1908  * Returns 0 on success, error on failure.
1909  */
1910 static int cik_init_microcode(struct radeon_device *rdev)
1911 {
1912         const char *chip_name;
1913         const char *new_chip_name;
1914         size_t pfp_req_size, me_req_size, ce_req_size,
1915                 mec_req_size, rlc_req_size, mc_req_size = 0,
1916                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1917         char fw_name[30];
1918         int new_fw = 0;
1919         int err;
1920         int num_fw;
1921
1922         DRM_DEBUG("\n");
1923
1924         switch (rdev->family) {
1925         case CHIP_BONAIRE:
1926                 chip_name = "BONAIRE";
1927                 new_chip_name = "bonaire";
1928                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1933                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1935                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1936                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1937                 num_fw = 8;
1938                 break;
1939         case CHIP_HAWAII:
1940                 chip_name = "HAWAII";
1941                 new_chip_name = "hawaii";
1942                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1948                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1949                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1951                 num_fw = 8;
1952                 break;
1953         case CHIP_KAVERI:
1954                 chip_name = "KAVERI";
1955                 new_chip_name = "kaveri";
1956                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1961                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1962                 num_fw = 7;
1963                 break;
1964         case CHIP_KABINI:
1965                 chip_name = "KABINI";
1966                 new_chip_name = "kabini";
1967                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1972                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1973                 num_fw = 6;
1974                 break;
1975         case CHIP_MULLINS:
1976                 chip_name = "MULLINS";
1977                 new_chip_name = "mullins";
1978                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982                 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1984                 num_fw = 6;
1985                 break;
1986         default: BUG();
1987         }
1988
1989         DRM_INFO("Loading %s Microcode\n", new_chip_name);
1990
1991         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1992         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1993         if (err) {
1994                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995                 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996                 if (err)
1997                         goto out;
1998                 if (rdev->pfp_fw->size != pfp_req_size) {
1999                         printk(KERN_ERR
2000                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001                                rdev->pfp_fw->size, fw_name);
2002                         err = -EINVAL;
2003                         goto out;
2004                 }
2005         } else {
2006                 err = radeon_ucode_validate(rdev->pfp_fw);
2007                 if (err) {
2008                         printk(KERN_ERR
2009                                "cik_fw: validation failed for firmware \"%s\"\n",
2010                                fw_name);
2011                         goto out;
2012                 } else {
2013                         new_fw++;
2014                 }
2015         }
2016
2017         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
2018         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2019         if (err) {
2020                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021                 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022                 if (err)
2023                         goto out;
2024                 if (rdev->me_fw->size != me_req_size) {
2025                         printk(KERN_ERR
2026                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027                                rdev->me_fw->size, fw_name);
2028                         err = -EINVAL;
2029                 }
2030         } else {
2031                 err = radeon_ucode_validate(rdev->me_fw);
2032                 if (err) {
2033                         printk(KERN_ERR
2034                                "cik_fw: validation failed for firmware \"%s\"\n",
2035                                fw_name);
2036                         goto out;
2037                 } else {
2038                         new_fw++;
2039                 }
2040         }
2041
2042         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
2043         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2044         if (err) {
2045                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046                 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047                 if (err)
2048                         goto out;
2049                 if (rdev->ce_fw->size != ce_req_size) {
2050                         printk(KERN_ERR
2051                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052                                rdev->ce_fw->size, fw_name);
2053                         err = -EINVAL;
2054                 }
2055         } else {
2056                 err = radeon_ucode_validate(rdev->ce_fw);
2057                 if (err) {
2058                         printk(KERN_ERR
2059                                "cik_fw: validation failed for firmware \"%s\"\n",
2060                                fw_name);
2061                         goto out;
2062                 } else {
2063                         new_fw++;
2064                 }
2065         }
2066
2067         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
2068         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2069         if (err) {
2070                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071                 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072                 if (err)
2073                         goto out;
2074                 if (rdev->mec_fw->size != mec_req_size) {
2075                         printk(KERN_ERR
2076                                "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077                                rdev->mec_fw->size, fw_name);
2078                         err = -EINVAL;
2079                 }
2080         } else {
2081                 err = radeon_ucode_validate(rdev->mec_fw);
2082                 if (err) {
2083                         printk(KERN_ERR
2084                                "cik_fw: validation failed for firmware \"%s\"\n",
2085                                fw_name);
2086                         goto out;
2087                 } else {
2088                         new_fw++;
2089                 }
2090         }
2091
2092         if (rdev->family == CHIP_KAVERI) {
2093                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094                 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095                 if (err) {
2096                         goto out;
2097                 } else {
2098                         err = radeon_ucode_validate(rdev->mec2_fw);
2099                         if (err) {
2100                                 goto out;
2101                         } else {
2102                                 new_fw++;
2103                         }
2104                 }
2105         }
2106
2107         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
2108         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2109         if (err) {
2110                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111                 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112                 if (err)
2113                         goto out;
2114                 if (rdev->rlc_fw->size != rlc_req_size) {
2115                         printk(KERN_ERR
2116                                "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117                                rdev->rlc_fw->size, fw_name);
2118                         err = -EINVAL;
2119                 }
2120         } else {
2121                 err = radeon_ucode_validate(rdev->rlc_fw);
2122                 if (err) {
2123                         printk(KERN_ERR
2124                                "cik_fw: validation failed for firmware \"%s\"\n",
2125                                fw_name);
2126                         goto out;
2127                 } else {
2128                         new_fw++;
2129                 }
2130         }
2131
2132         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
2133         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2134         if (err) {
2135                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136                 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137                 if (err)
2138                         goto out;
2139                 if (rdev->sdma_fw->size != sdma_req_size) {
2140                         printk(KERN_ERR
2141                                "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142                                rdev->sdma_fw->size, fw_name);
2143                         err = -EINVAL;
2144                 }
2145         } else {
2146                 err = radeon_ucode_validate(rdev->sdma_fw);
2147                 if (err) {
2148                         printk(KERN_ERR
2149                                "cik_fw: validation failed for firmware \"%s\"\n",
2150                                fw_name);
2151                         goto out;
2152                 } else {
2153                         new_fw++;
2154                 }
2155         }
2156
2157         /* No SMC, MC ucode on APUs */
2158         if (!(rdev->flags & RADEON_IS_IGP)) {
2159                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
2160                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2161                 if (err) {
2162                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
2163                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2164                         if (err) {
2165                                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166                                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167                                 if (err)
2168                                         goto out;
2169                         }
2170                         if ((rdev->mc_fw->size != mc_req_size) &&
2171                             (rdev->mc_fw->size != mc2_req_size)){
2172                                 printk(KERN_ERR
2173                                        "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174                                        rdev->mc_fw->size, fw_name);
2175                                 err = -EINVAL;
2176                         }
2177                         DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178                 } else {
2179                         err = radeon_ucode_validate(rdev->mc_fw);
2180                         if (err) {
2181                                 printk(KERN_ERR
2182                                        "cik_fw: validation failed for firmware \"%s\"\n",
2183                                        fw_name);
2184                                 goto out;
2185                         } else {
2186                                 new_fw++;
2187                         }
2188                 }
2189
2190                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
2191                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192                 if (err) {
2193                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194                         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195                         if (err) {
2196                                 printk(KERN_ERR
2197                                        "smc: error loading firmware \"%s\"\n",
2198                                        fw_name);
2199                                 release_firmware(rdev->smc_fw);
2200                                 rdev->smc_fw = NULL;
2201                                 err = 0;
2202                         } else if (rdev->smc_fw->size != smc_req_size) {
2203                                 printk(KERN_ERR
2204                                        "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205                                        rdev->smc_fw->size, fw_name);
2206                                 err = -EINVAL;
2207                         }
2208                 } else {
2209                         err = radeon_ucode_validate(rdev->smc_fw);
2210                         if (err) {
2211                                 printk(KERN_ERR
2212                                        "cik_fw: validation failed for firmware \"%s\"\n",
2213                                        fw_name);
2214                                 goto out;
2215                         } else {
2216                                 new_fw++;
2217                         }
2218                 }
2219         }
2220
2221         if (new_fw == 0) {
2222                 rdev->new_fw = false;
2223         } else if (new_fw < num_fw) {
2224                 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225                 err = -EINVAL;
2226         } else {
2227                 rdev->new_fw = true;
2228         }
2229
2230 out:
2231         if (err) {
2232                 if (err != -EINVAL)
2233                         printk(KERN_ERR
2234                                "cik_cp: Failed to load firmware \"%s\"\n",
2235                                fw_name);
2236                 release_firmware(rdev->pfp_fw);
2237                 rdev->pfp_fw = NULL;
2238                 release_firmware(rdev->me_fw);
2239                 rdev->me_fw = NULL;
2240                 release_firmware(rdev->ce_fw);
2241                 rdev->ce_fw = NULL;
2242                 release_firmware(rdev->mec_fw);
2243                 rdev->mec_fw = NULL;
2244                 release_firmware(rdev->mec2_fw);
2245                 rdev->mec2_fw = NULL;
2246                 release_firmware(rdev->rlc_fw);
2247                 rdev->rlc_fw = NULL;
2248                 release_firmware(rdev->sdma_fw);
2249                 rdev->sdma_fw = NULL;
2250                 release_firmware(rdev->mc_fw);
2251                 rdev->mc_fw = NULL;
2252                 release_firmware(rdev->smc_fw);
2253                 rdev->smc_fw = NULL;
2254         }
2255         return err;
2256 }
2257
2258 /*
2259  * Core functions
2260  */
2261 /**
2262  * cik_tiling_mode_table_init - init the hw tiling table
2263  *
2264  * @rdev: radeon_device pointer
2265  *
2266  * Starting with SI, the tiling setup is done globally in a
2267  * set of 32 tiling modes.  Rather than selecting each set of
2268  * parameters per surface as on older asics, we just select
2269  * which index in the tiling table we want to use, and the
2270  * surface uses those parameters (CIK).
2271  */
2272 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273 {
2274         const u32 num_tile_mode_states = 32;
2275         const u32 num_secondary_tile_mode_states = 16;
2276         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277         u32 num_pipe_configs;
2278         u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279                 rdev->config.cik.max_shader_engines;
2280
2281         switch (rdev->config.cik.mem_row_size_in_kb) {
2282         case 1:
2283                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284                 break;
2285         case 2:
2286         default:
2287                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288                 break;
2289         case 4:
2290                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291                 break;
2292         }
2293
2294         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295         if (num_pipe_configs > 8)
2296                 num_pipe_configs = 16;
2297
2298         if (num_pipe_configs == 16) {
2299                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300                         switch (reg_offset) {
2301                         case 0:
2302                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306                                 break;
2307                         case 1:
2308                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312                                 break;
2313                         case 2:
2314                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318                                 break;
2319                         case 3:
2320                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324                                 break;
2325                         case 4:
2326                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329                                                  TILE_SPLIT(split_equal_to_row_size));
2330                                 break;
2331                         case 5:
2332                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2333                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335                                 break;
2336                         case 6:
2337                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341                                 break;
2342                         case 7:
2343                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346                                                  TILE_SPLIT(split_equal_to_row_size));
2347                                 break;
2348                         case 8:
2349                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351                                 break;
2352                         case 9:
2353                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2354                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2355                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356                                 break;
2357                         case 10:
2358                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362                                 break;
2363                         case 11:
2364                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368                                 break;
2369                         case 12:
2370                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374                                 break;
2375                         case 13:
2376                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2377                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2378                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379                                 break;
2380                         case 14:
2381                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385                                 break;
2386                         case 16:
2387                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391                                 break;
2392                         case 17:
2393                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397                                 break;
2398                         case 27:
2399                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2400                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402                                 break;
2403                         case 28:
2404                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408                                 break;
2409                         case 29:
2410                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414                                 break;
2415                         case 30:
2416                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420                                 break;
2421                         default:
2422                                 gb_tile_moden = 0;
2423                                 break;
2424                         }
2425                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427                 }
2428                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429                         switch (reg_offset) {
2430                         case 0:
2431                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2435                                 break;
2436                         case 1:
2437                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2441                                 break;
2442                         case 2:
2443                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2447                                 break;
2448                         case 3:
2449                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2453                                 break;
2454                         case 4:
2455                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2459                                 break;
2460                         case 5:
2461                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2465                                 break;
2466                         case 6:
2467                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2471                                 break;
2472                         case 8:
2473                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2477                                 break;
2478                         case 9:
2479                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2483                                 break;
2484                         case 10:
2485                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2489                                 break;
2490                         case 11:
2491                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2495                                 break;
2496                         case 12:
2497                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2501                                 break;
2502                         case 13:
2503                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2507                                 break;
2508                         case 14:
2509                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2513                                 break;
2514                         default:
2515                                 gb_tile_moden = 0;
2516                                 break;
2517                         }
2518                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2519                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520                 }
2521         } else if (num_pipe_configs == 8) {
2522                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523                         switch (reg_offset) {
2524                         case 0:
2525                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529                                 break;
2530                         case 1:
2531                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535                                 break;
2536                         case 2:
2537                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541                                 break;
2542                         case 3:
2543                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547                                 break;
2548                         case 4:
2549                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552                                                  TILE_SPLIT(split_equal_to_row_size));
2553                                 break;
2554                         case 5:
2555                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2556                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558                                 break;
2559                         case 6:
2560                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564                                 break;
2565                         case 7:
2566                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569                                                  TILE_SPLIT(split_equal_to_row_size));
2570                                 break;
2571                         case 8:
2572                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574                                 break;
2575                         case 9:
2576                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2577                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579                                 break;
2580                         case 10:
2581                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585                                 break;
2586                         case 11:
2587                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591                                 break;
2592                         case 12:
2593                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597                                 break;
2598                         case 13:
2599                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602                                 break;
2603                         case 14:
2604                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608                                 break;
2609                         case 16:
2610                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614                                 break;
2615                         case 17:
2616                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620                                 break;
2621                         case 27:
2622                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2623                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2624                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625                                 break;
2626                         case 28:
2627                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631                                 break;
2632                         case 29:
2633                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637                                 break;
2638                         case 30:
2639                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643                                 break;
2644                         default:
2645                                 gb_tile_moden = 0;
2646                                 break;
2647                         }
2648                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2649                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650                 }
2651                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652                         switch (reg_offset) {
2653                         case 0:
2654                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2658                                 break;
2659                         case 1:
2660                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2664                                 break;
2665                         case 2:
2666                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2670                                 break;
2671                         case 3:
2672                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2676                                 break;
2677                         case 4:
2678                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2682                                 break;
2683                         case 5:
2684                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2688                                 break;
2689                         case 6:
2690                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2694                                 break;
2695                         case 8:
2696                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2700                                 break;
2701                         case 9:
2702                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2706                                 break;
2707                         case 10:
2708                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2712                                 break;
2713                         case 11:
2714                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2718                                 break;
2719                         case 12:
2720                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2724                                 break;
2725                         case 13:
2726                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2730                                 break;
2731                         case 14:
2732                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2736                                 break;
2737                         default:
2738                                 gb_tile_moden = 0;
2739                                 break;
2740                         }
2741                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2742                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743                 }
2744         } else if (num_pipe_configs == 4) {
2745                 if (num_rbs == 4) {
2746                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747                                 switch (reg_offset) {
2748                                 case 0:
2749                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753                                         break;
2754                                 case 1:
2755                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759                                         break;
2760                                 case 2:
2761                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765                                         break;
2766                                 case 3:
2767                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771                                         break;
2772                                 case 4:
2773                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776                                                          TILE_SPLIT(split_equal_to_row_size));
2777                                         break;
2778                                 case 5:
2779                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2780                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2781                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782                                         break;
2783                                 case 6:
2784                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788                                         break;
2789                                 case 7:
2790                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793                                                          TILE_SPLIT(split_equal_to_row_size));
2794                                         break;
2795                                 case 8:
2796                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798                                         break;
2799                                 case 9:
2800                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2801                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2802                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803                                         break;
2804                                 case 10:
2805                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809                                         break;
2810                                 case 11:
2811                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815                                         break;
2816                                 case 12:
2817                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821                                         break;
2822                                 case 13:
2823                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826                                         break;
2827                                 case 14:
2828                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832                                         break;
2833                                 case 16:
2834                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838                                         break;
2839                                 case 17:
2840                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844                                         break;
2845                                 case 27:
2846                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2847                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849                                         break;
2850                                 case 28:
2851                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855                                         break;
2856                                 case 29:
2857                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861                                         break;
2862                                 case 30:
2863                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867                                         break;
2868                                 default:
2869                                         gb_tile_moden = 0;
2870                                         break;
2871                                 }
2872                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2873                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874                         }
2875                 } else if (num_rbs < 4) {
2876                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877                                 switch (reg_offset) {
2878                                 case 0:
2879                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883                                         break;
2884                                 case 1:
2885                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889                                         break;
2890                                 case 2:
2891                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895                                         break;
2896                                 case 3:
2897                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901                                         break;
2902                                 case 4:
2903                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906                                                          TILE_SPLIT(split_equal_to_row_size));
2907                                         break;
2908                                 case 5:
2909                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2910                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2911                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912                                         break;
2913                                 case 6:
2914                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918                                         break;
2919                                 case 7:
2920                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923                                                          TILE_SPLIT(split_equal_to_row_size));
2924                                         break;
2925                                 case 8:
2926                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928                                         break;
2929                                 case 9:
2930                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2931                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2932                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933                                         break;
2934                                 case 10:
2935                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939                                         break;
2940                                 case 11:
2941                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945                                         break;
2946                                 case 12:
2947                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951                                         break;
2952                                 case 13:
2953                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2954                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2955                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956                                         break;
2957                                 case 14:
2958                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962                                         break;
2963                                 case 16:
2964                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968                                         break;
2969                                 case 17:
2970                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974                                         break;
2975                                 case 27:
2976                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2977                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2978                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979                                         break;
2980                                 case 28:
2981                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985                                         break;
2986                                 case 29:
2987                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991                                         break;
2992                                 case 30:
2993                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997                                         break;
2998                                 default:
2999                                         gb_tile_moden = 0;
3000                                         break;
3001                                 }
3002                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3003                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004                         }
3005                 }
3006                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007                         switch (reg_offset) {
3008                         case 0:
3009                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3013                                 break;
3014                         case 1:
3015                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3019                                 break;
3020                         case 2:
3021                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3025                                 break;
3026                         case 3:
3027                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3031                                 break;
3032                         case 4:
3033                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3037                                 break;
3038                         case 5:
3039                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3043                                 break;
3044                         case 6:
3045                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048                                                  NUM_BANKS(ADDR_SURF_4_BANK));
3049                                 break;
3050                         case 8:
3051                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3055                                 break;
3056                         case 9:
3057                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3061                                 break;
3062                         case 10:
3063                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3067                                 break;
3068                         case 11:
3069                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3073                                 break;
3074                         case 12:
3075                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3079                                 break;
3080                         case 13:
3081                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3085                                 break;
3086                         case 14:
3087                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090                                                  NUM_BANKS(ADDR_SURF_4_BANK));
3091                                 break;
3092                         default:
3093                                 gb_tile_moden = 0;
3094                                 break;
3095                         }
3096                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3097                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098                 }
3099         } else if (num_pipe_configs == 2) {
3100                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101                         switch (reg_offset) {
3102                         case 0:
3103                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3106                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107                                 break;
3108                         case 1:
3109                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3112                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113                                 break;
3114                         case 2:
3115                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3118                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119                                 break;
3120                         case 3:
3121                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3124                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125                                 break;
3126                         case 4:
3127                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3130                                                  TILE_SPLIT(split_equal_to_row_size));
3131                                 break;
3132                         case 5:
3133                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3134                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3135                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136                                 break;
3137                         case 6:
3138                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3141                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142                                 break;
3143                         case 7:
3144                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3147                                                  TILE_SPLIT(split_equal_to_row_size));
3148                                 break;
3149                         case 8:
3150                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151                                                 PIPE_CONFIG(ADDR_SURF_P2);
3152                                 break;
3153                         case 9:
3154                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3155                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156                                                  PIPE_CONFIG(ADDR_SURF_P2));
3157                                 break;
3158                         case 10:
3159                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3162                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163                                 break;
3164                         case 11:
3165                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3168                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169                                 break;
3170                         case 12:
3171                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3174                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175                                 break;
3176                         case 13:
3177                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3178                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3179                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180                                 break;
3181                         case 14:
3182                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3185                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186                                 break;
3187                         case 16:
3188                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3191                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192                                 break;
3193                         case 17:
3194                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3197                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198                                 break;
3199                         case 27:
3200                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3201                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202                                                  PIPE_CONFIG(ADDR_SURF_P2));
3203                                 break;
3204                         case 28:
3205                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3208                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209                                 break;
3210                         case 29:
3211                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3214                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215                                 break;
3216                         case 30:
3217                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219                                                  PIPE_CONFIG(ADDR_SURF_P2) |
3220                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221                                 break;
3222                         default:
3223                                 gb_tile_moden = 0;
3224                                 break;
3225                         }
3226                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3227                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228                 }
3229                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230                         switch (reg_offset) {
3231                         case 0:
3232                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3236                                 break;
3237                         case 1:
3238                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3242                                 break;
3243                         case 2:
3244                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3248                                 break;
3249                         case 3:
3250                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3254                                 break;
3255                         case 4:
3256                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3260                                 break;
3261                         case 5:
3262                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3266                                 break;
3267                         case 6:
3268                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3272                                 break;
3273                         case 8:
3274                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3278                                 break;
3279                         case 9:
3280                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3284                                 break;
3285                         case 10:
3286                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3290                                 break;
3291                         case 11:
3292                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3296                                 break;
3297                         case 12:
3298                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3302                                 break;
3303                         case 13:
3304                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3308                                 break;
3309                         case 14:
3310                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3314                                 break;
3315                         default:
3316                                 gb_tile_moden = 0;
3317                                 break;
3318                         }
3319                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3320                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321                 }
3322         } else
3323                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324 }
3325
3326 /**
3327  * cik_select_se_sh - select which SE, SH to address
3328  *
3329  * @rdev: radeon_device pointer
3330  * @se_num: shader engine to address
3331  * @sh_num: sh block to address
3332  *
3333  * Select which SE, SH combinations to address. Certain
3334  * registers are instanced per SE or SH.  0xffffffff means
3335  * broadcast to all SEs or SHs (CIK).
3336  */
3337 static void cik_select_se_sh(struct radeon_device *rdev,
3338                              u32 se_num, u32 sh_num)
3339 {
3340         u32 data = INSTANCE_BROADCAST_WRITES;
3341
3342         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3343                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3344         else if (se_num == 0xffffffff)
3345                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346         else if (sh_num == 0xffffffff)
3347                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348         else
3349                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350         WREG32(GRBM_GFX_INDEX, data);
3351 }
3352
3353 /**
3354  * cik_create_bitmask - create a bitmask
3355  *
3356  * @bit_width: length of the mask
3357  *
3358  * create a variable length bit mask (CIK).
3359  * Returns the bitmask.
3360  */
3361 static u32 cik_create_bitmask(u32 bit_width)
3362 {
3363         u32 i, mask = 0;
3364
3365         for (i = 0; i < bit_width; i++) {
3366                 mask <<= 1;
3367                 mask |= 1;
3368         }
3369         return mask;
3370 }
3371
3372 /**
3373  * cik_get_rb_disabled - computes the mask of disabled RBs
3374  *
3375  * @rdev: radeon_device pointer
3376  * @max_rb_num: max RBs (render backends) for the asic
3377  * @se_num: number of SEs (shader engines) for the asic
3378  * @sh_per_se: number of SH blocks per SE for the asic
3379  *
3380  * Calculates the bitmask of disabled RBs (CIK).
3381  * Returns the disabled RB bitmask.
3382  */
3383 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3384                               u32 max_rb_num_per_se,
3385                               u32 sh_per_se)
3386 {
3387         u32 data, mask;
3388
3389         data = RREG32(CC_RB_BACKEND_DISABLE);
3390         if (data & 1)
3391                 data &= BACKEND_DISABLE_MASK;
3392         else
3393                 data = 0;
3394         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395
3396         data >>= BACKEND_DISABLE_SHIFT;
3397
3398         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3399
3400         return data & mask;
3401 }
3402
3403 /**
3404  * cik_setup_rb - setup the RBs on the asic
3405  *
3406  * @rdev: radeon_device pointer
3407  * @se_num: number of SEs (shader engines) for the asic
3408  * @sh_per_se: number of SH blocks per SE for the asic
3409  * @max_rb_num: max RBs (render backends) for the asic
3410  *
3411  * Configures per-SE/SH RB registers (CIK).
3412  */
3413 static void cik_setup_rb(struct radeon_device *rdev,
3414                          u32 se_num, u32 sh_per_se,
3415                          u32 max_rb_num_per_se)
3416 {
3417         int i, j;
3418         u32 data, mask;
3419         u32 disabled_rbs = 0;
3420         u32 enabled_rbs = 0;
3421
3422         for (i = 0; i < se_num; i++) {
3423                 for (j = 0; j < sh_per_se; j++) {
3424                         cik_select_se_sh(rdev, i, j);
3425                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3426                         if (rdev->family == CHIP_HAWAII)
3427                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428                         else
3429                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3430                 }
3431         }
3432         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433
3434         mask = 1;
3435         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3436                 if (!(disabled_rbs & mask))
3437                         enabled_rbs |= mask;
3438                 mask <<= 1;
3439         }
3440
3441         rdev->config.cik.backend_enable_mask = enabled_rbs;
3442
3443         for (i = 0; i < se_num; i++) {
3444                 cik_select_se_sh(rdev, i, 0xffffffff);
3445                 data = 0;
3446                 for (j = 0; j < sh_per_se; j++) {
3447                         switch (enabled_rbs & 3) {
3448                         case 0:
3449                                 if (j == 0)
3450                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451                                 else
3452                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453                                 break;
3454                         case 1:
3455                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456                                 break;
3457                         case 2:
3458                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459                                 break;
3460                         case 3:
3461                         default:
3462                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463                                 break;
3464                         }
3465                         enabled_rbs >>= 2;
3466                 }
3467                 WREG32(PA_SC_RASTER_CONFIG, data);
3468         }
3469         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470 }
3471
3472 /**
3473  * cik_gpu_init - setup the 3D engine
3474  *
3475  * @rdev: radeon_device pointer
3476  *
3477  * Configures the 3D engine and tiling configuration
3478  * registers so that the 3D engine is usable.
3479  */
3480 static void cik_gpu_init(struct radeon_device *rdev)
3481 {
3482         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483         u32 mc_shared_chmap, mc_arb_ramcfg;
3484         u32 hdp_host_path_cntl;
3485         u32 tmp;
3486         int i, j, k;
3487
3488         switch (rdev->family) {
3489         case CHIP_BONAIRE:
3490                 rdev->config.cik.max_shader_engines = 2;
3491                 rdev->config.cik.max_tile_pipes = 4;
3492                 rdev->config.cik.max_cu_per_sh = 7;
3493                 rdev->config.cik.max_sh_per_se = 1;
3494                 rdev->config.cik.max_backends_per_se = 2;
3495                 rdev->config.cik.max_texture_channel_caches = 4;
3496                 rdev->config.cik.max_gprs = 256;
3497                 rdev->config.cik.max_gs_threads = 32;
3498                 rdev->config.cik.max_hw_contexts = 8;
3499
3500                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505                 break;
3506         case CHIP_HAWAII:
3507                 rdev->config.cik.max_shader_engines = 4;
3508                 rdev->config.cik.max_tile_pipes = 16;
3509                 rdev->config.cik.max_cu_per_sh = 11;
3510                 rdev->config.cik.max_sh_per_se = 1;
3511                 rdev->config.cik.max_backends_per_se = 4;
3512                 rdev->config.cik.max_texture_channel_caches = 16;
3513                 rdev->config.cik.max_gprs = 256;
3514                 rdev->config.cik.max_gs_threads = 32;
3515                 rdev->config.cik.max_hw_contexts = 8;
3516
3517                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522                 break;
3523         case CHIP_KAVERI:
3524                 rdev->config.cik.max_shader_engines = 1;
3525                 rdev->config.cik.max_tile_pipes = 4;
3526                 if ((rdev->pdev->device == 0x1304) ||
3527                     (rdev->pdev->device == 0x1305) ||
3528                     (rdev->pdev->device == 0x130C) ||
3529                     (rdev->pdev->device == 0x130F) ||
3530                     (rdev->pdev->device == 0x1310) ||
3531                     (rdev->pdev->device == 0x1311) ||
3532                     (rdev->pdev->device == 0x131C)) {
3533                         rdev->config.cik.max_cu_per_sh = 8;
3534                         rdev->config.cik.max_backends_per_se = 2;
3535                 } else if ((rdev->pdev->device == 0x1309) ||
3536                            (rdev->pdev->device == 0x130A) ||
3537                            (rdev->pdev->device == 0x130D) ||
3538                            (rdev->pdev->device == 0x1313) ||
3539                            (rdev->pdev->device == 0x131D)) {
3540                         rdev->config.cik.max_cu_per_sh = 6;
3541                         rdev->config.cik.max_backends_per_se = 2;
3542                 } else if ((rdev->pdev->device == 0x1306) ||
3543                            (rdev->pdev->device == 0x1307) ||
3544                            (rdev->pdev->device == 0x130B) ||
3545                            (rdev->pdev->device == 0x130E) ||
3546                            (rdev->pdev->device == 0x1315) ||
3547                            (rdev->pdev->device == 0x131B)) {
3548                         rdev->config.cik.max_cu_per_sh = 4;
3549                         rdev->config.cik.max_backends_per_se = 1;
3550                 } else {
3551                         rdev->config.cik.max_cu_per_sh = 3;
3552                         rdev->config.cik.max_backends_per_se = 1;
3553                 }
3554                 rdev->config.cik.max_sh_per_se = 1;
3555                 rdev->config.cik.max_texture_channel_caches = 4;
3556                 rdev->config.cik.max_gprs = 256;
3557                 rdev->config.cik.max_gs_threads = 16;
3558                 rdev->config.cik.max_hw_contexts = 8;
3559
3560                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3561                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3562                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3563                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3564                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3565                 break;
3566         case CHIP_KABINI:
3567         case CHIP_MULLINS:
3568         default:
3569                 rdev->config.cik.max_shader_engines = 1;
3570                 rdev->config.cik.max_tile_pipes = 2;
3571                 rdev->config.cik.max_cu_per_sh = 2;
3572                 rdev->config.cik.max_sh_per_se = 1;
3573                 rdev->config.cik.max_backends_per_se = 1;
3574                 rdev->config.cik.max_texture_channel_caches = 2;
3575                 rdev->config.cik.max_gprs = 256;
3576                 rdev->config.cik.max_gs_threads = 16;
3577                 rdev->config.cik.max_hw_contexts = 8;
3578
3579                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3584                 break;
3585         }
3586
3587         /* Initialize HDP */
3588         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3589                 WREG32((0x2c14 + j), 0x00000000);
3590                 WREG32((0x2c18 + j), 0x00000000);
3591                 WREG32((0x2c1c + j), 0x00000000);
3592                 WREG32((0x2c20 + j), 0x00000000);
3593                 WREG32((0x2c24 + j), 0x00000000);
3594         }
3595
3596         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3597
3598         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3599
3600         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3601         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3602
3603         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3604         rdev->config.cik.mem_max_burst_length_bytes = 256;
3605         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3606         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3607         if (rdev->config.cik.mem_row_size_in_kb > 4)
3608                 rdev->config.cik.mem_row_size_in_kb = 4;
3609         /* XXX use MC settings? */
3610         rdev->config.cik.shader_engine_tile_size = 32;
3611         rdev->config.cik.num_gpus = 1;
3612         rdev->config.cik.multi_gpu_tile_size = 64;
3613
3614         /* fix up row size */
3615         gb_addr_config &= ~ROW_SIZE_MASK;
3616         switch (rdev->config.cik.mem_row_size_in_kb) {
3617         case 1:
3618         default:
3619                 gb_addr_config |= ROW_SIZE(0);
3620                 break;
3621         case 2:
3622                 gb_addr_config |= ROW_SIZE(1);
3623                 break;
3624         case 4:
3625                 gb_addr_config |= ROW_SIZE(2);
3626                 break;
3627         }
3628
3629         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3630          * not have bank info, so create a custom tiling dword.
3631          * bits 3:0   num_pipes
3632          * bits 7:4   num_banks
3633          * bits 11:8  group_size
3634          * bits 15:12 row_size
3635          */
3636         rdev->config.cik.tile_config = 0;
3637         switch (rdev->config.cik.num_tile_pipes) {
3638         case 1:
3639                 rdev->config.cik.tile_config |= (0 << 0);
3640                 break;
3641         case 2:
3642                 rdev->config.cik.tile_config |= (1 << 0);
3643                 break;
3644         case 4:
3645                 rdev->config.cik.tile_config |= (2 << 0);
3646                 break;
3647         case 8:
3648         default:
3649                 /* XXX what about 12? */
3650                 rdev->config.cik.tile_config |= (3 << 0);
3651                 break;
3652         }
3653         rdev->config.cik.tile_config |=
3654                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3655         rdev->config.cik.tile_config |=
3656                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3657         rdev->config.cik.tile_config |=
3658                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3659
3660         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3661         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3662         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3663         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3664         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3665         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3666         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3667         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3668
3669         cik_tiling_mode_table_init(rdev);
3670
3671         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3672                      rdev->config.cik.max_sh_per_se,
3673                      rdev->config.cik.max_backends_per_se);
3674
3675         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3676                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3677                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3678                                 rdev->config.cik.active_cus +=
3679                                         hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3680                         }
3681                 }
3682         }
3683
3684         /* set HW defaults for 3D engine */
3685         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3686
3687         WREG32(SX_DEBUG_1, 0x20);
3688
3689         WREG32(TA_CNTL_AUX, 0x00010000);
3690
3691         tmp = RREG32(SPI_CONFIG_CNTL);
3692         tmp |= 0x03000000;
3693         WREG32(SPI_CONFIG_CNTL, tmp);
3694
3695         WREG32(SQ_CONFIG, 1);
3696
3697         WREG32(DB_DEBUG, 0);
3698
3699         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3700         tmp |= 0x00000400;
3701         WREG32(DB_DEBUG2, tmp);
3702
3703         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3704         tmp |= 0x00020200;
3705         WREG32(DB_DEBUG3, tmp);
3706
3707         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3708         tmp |= 0x00018208;
3709         WREG32(CB_HW_CONTROL, tmp);
3710
3711         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3712
3713         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3714                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3715                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3716                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3717
3718         WREG32(VGT_NUM_INSTANCES, 1);
3719
3720         WREG32(CP_PERFMON_CNTL, 0);
3721
3722         WREG32(SQ_CONFIG, 0);
3723
3724         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3725                                           FORCE_EOV_MAX_REZ_CNT(255)));
3726
3727         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3728                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3729
3730         WREG32(VGT_GS_VERTEX_REUSE, 16);
3731         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3732
3733         tmp = RREG32(HDP_MISC_CNTL);
3734         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3735         WREG32(HDP_MISC_CNTL, tmp);
3736
3737         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3738         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3739
3740         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3741         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3742
3743         udelay(50);
3744 }
3745
3746 /*
3747  * GPU scratch registers helpers function.
3748  */
3749 /**
3750  * cik_scratch_init - setup driver info for CP scratch regs
3751  *
3752  * @rdev: radeon_device pointer
3753  *
3754  * Set up the number and offset of the CP scratch registers.
3755  * NOTE: use of CP scratch registers is a legacy inferface and
3756  * is not used by default on newer asics (r6xx+).  On newer asics,
3757  * memory buffers are used for fences rather than scratch regs.
3758  */
3759 static void cik_scratch_init(struct radeon_device *rdev)
3760 {
3761         int i;
3762
3763         rdev->scratch.num_reg = 7;
3764         rdev->scratch.reg_base = SCRATCH_REG0;
3765         for (i = 0; i < rdev->scratch.num_reg; i++) {
3766                 rdev->scratch.free[i] = true;
3767                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3768         }
3769 }
3770
3771 /**
3772  * cik_ring_test - basic gfx ring test
3773  *
3774  * @rdev: radeon_device pointer
3775  * @ring: radeon_ring structure holding ring information
3776  *
3777  * Allocate a scratch register and write to it using the gfx ring (CIK).
3778  * Provides a basic gfx ring test to verify that the ring is working.
3779  * Used by cik_cp_gfx_resume();
3780  * Returns 0 on success, error on failure.
3781  */
3782 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3783 {
3784         uint32_t scratch;
3785         uint32_t tmp = 0;
3786         unsigned i;
3787         int r;
3788
3789         r = radeon_scratch_get(rdev, &scratch);
3790         if (r) {
3791                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3792                 return r;
3793         }
3794         WREG32(scratch, 0xCAFEDEAD);
3795         r = radeon_ring_lock(rdev, ring, 3);
3796         if (r) {
3797                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3798                 radeon_scratch_free(rdev, scratch);
3799                 return r;
3800         }
3801         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3802         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3803         radeon_ring_write(ring, 0xDEADBEEF);
3804         radeon_ring_unlock_commit(rdev, ring);
3805
3806         for (i = 0; i < rdev->usec_timeout; i++) {
3807                 tmp = RREG32(scratch);
3808                 if (tmp == 0xDEADBEEF)
3809                         break;
3810                 DRM_UDELAY(1);
3811         }
3812         if (i < rdev->usec_timeout) {
3813                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3814         } else {
3815                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3816                           ring->idx, scratch, tmp);
3817                 r = -EINVAL;
3818         }
3819         radeon_scratch_free(rdev, scratch);
3820         return r;
3821 }
3822
3823 /**
3824  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3825  *
3826  * @rdev: radeon_device pointer
3827  * @ridx: radeon ring index
3828  *
3829  * Emits an hdp flush on the cp.
3830  */
3831 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3832                                        int ridx)
3833 {
3834         struct radeon_ring *ring = &rdev->ring[ridx];
3835         u32 ref_and_mask;
3836
3837         switch (ring->idx) {
3838         case CAYMAN_RING_TYPE_CP1_INDEX:
3839         case CAYMAN_RING_TYPE_CP2_INDEX:
3840         default:
3841                 switch (ring->me) {
3842                 case 0:
3843                         ref_and_mask = CP2 << ring->pipe;
3844                         break;
3845                 case 1:
3846                         ref_and_mask = CP6 << ring->pipe;
3847                         break;
3848                 default:
3849                         return;
3850                 }
3851                 break;
3852         case RADEON_RING_TYPE_GFX_INDEX:
3853                 ref_and_mask = CP0;
3854                 break;
3855         }
3856
3857         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3858         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3859                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3860                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3861         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3862         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3863         radeon_ring_write(ring, ref_and_mask);
3864         radeon_ring_write(ring, ref_and_mask);
3865         radeon_ring_write(ring, 0x20); /* poll interval */
3866 }
3867
3868 /**
3869  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3870  *
3871  * @rdev: radeon_device pointer
3872  * @fence: radeon fence object
3873  *
3874  * Emits a fence sequnce number on the gfx ring and flushes
3875  * GPU caches.
3876  */
3877 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3878                              struct radeon_fence *fence)
3879 {
3880         struct radeon_ring *ring = &rdev->ring[fence->ring];
3881         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3882
3883         /* EVENT_WRITE_EOP - flush caches, send int */
3884         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3885         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3886                                  EOP_TC_ACTION_EN |
3887                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3888                                  EVENT_INDEX(5)));
3889         radeon_ring_write(ring, addr & 0xfffffffc);
3890         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3891         radeon_ring_write(ring, fence->seq);
3892         radeon_ring_write(ring, 0);
3893 }
3894
3895 /**
3896  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3897  *
3898  * @rdev: radeon_device pointer
3899  * @fence: radeon fence object
3900  *
3901  * Emits a fence sequnce number on the compute ring and flushes
3902  * GPU caches.
3903  */
3904 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3905                                  struct radeon_fence *fence)
3906 {
3907         struct radeon_ring *ring = &rdev->ring[fence->ring];
3908         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3909
3910         /* RELEASE_MEM - flush caches, send int */
3911         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3912         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3913                                  EOP_TC_ACTION_EN |
3914                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3915                                  EVENT_INDEX(5)));
3916         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3917         radeon_ring_write(ring, addr & 0xfffffffc);
3918         radeon_ring_write(ring, upper_32_bits(addr));
3919         radeon_ring_write(ring, fence->seq);
3920         radeon_ring_write(ring, 0);
3921 }
3922
3923 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3924                              struct radeon_ring *ring,
3925                              struct radeon_semaphore *semaphore,
3926                              bool emit_wait)
3927 {
3928         uint64_t addr = semaphore->gpu_addr;
3929         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3930
3931         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3932         radeon_ring_write(ring, lower_32_bits(addr));
3933         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3934
3935         return true;
3936 }
3937
3938 /**
3939  * cik_copy_cpdma - copy pages using the CP DMA engine
3940  *
3941  * @rdev: radeon_device pointer
3942  * @src_offset: src GPU address
3943  * @dst_offset: dst GPU address
3944  * @num_gpu_pages: number of GPU pages to xfer
3945  * @fence: radeon fence object
3946  *
3947  * Copy GPU paging using the CP DMA engine (CIK+).
3948  * Used by the radeon ttm implementation to move pages if
3949  * registered as the asic copy callback.
3950  */
3951 int cik_copy_cpdma(struct radeon_device *rdev,
3952                    uint64_t src_offset, uint64_t dst_offset,
3953                    unsigned num_gpu_pages,
3954                    struct radeon_fence **fence)
3955 {
3956         struct radeon_semaphore *sem = NULL;
3957         int ring_index = rdev->asic->copy.blit_ring_index;
3958         struct radeon_ring *ring = &rdev->ring[ring_index];
3959         u32 size_in_bytes, cur_size_in_bytes, control;
3960         int i, num_loops;
3961         int r = 0;
3962
3963         r = radeon_semaphore_create(rdev, &sem);
3964         if (r) {
3965                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3966                 return r;
3967         }
3968
3969         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3970         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3971         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3972         if (r) {
3973                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3974                 radeon_semaphore_free(rdev, &sem, NULL);
3975                 return r;
3976         }
3977
3978         radeon_semaphore_sync_to(sem, *fence);
3979         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3980
3981         for (i = 0; i < num_loops; i++) {
3982                 cur_size_in_bytes = size_in_bytes;
3983                 if (cur_size_in_bytes > 0x1fffff)
3984                         cur_size_in_bytes = 0x1fffff;
3985                 size_in_bytes -= cur_size_in_bytes;
3986                 control = 0;
3987                 if (size_in_bytes == 0)
3988                         control |= PACKET3_DMA_DATA_CP_SYNC;
3989                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3990                 radeon_ring_write(ring, control);
3991                 radeon_ring_write(ring, lower_32_bits(src_offset));
3992                 radeon_ring_write(ring, upper_32_bits(src_offset));
3993                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3994                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3995                 radeon_ring_write(ring, cur_size_in_bytes);
3996                 src_offset += cur_size_in_bytes;
3997                 dst_offset += cur_size_in_bytes;
3998         }
3999
4000         r = radeon_fence_emit(rdev, fence, ring->idx);
4001         if (r) {
4002                 radeon_ring_unlock_undo(rdev, ring);
4003                 radeon_semaphore_free(rdev, &sem, NULL);
4004                 return r;
4005         }
4006
4007         radeon_ring_unlock_commit(rdev, ring);
4008         radeon_semaphore_free(rdev, &sem, *fence);
4009
4010         return r;
4011 }
4012
4013 /*
4014  * IB stuff
4015  */
4016 /**
4017  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4018  *
4019  * @rdev: radeon_device pointer
4020  * @ib: radeon indirect buffer object
4021  *
4022  * Emits an DE (drawing engine) or CE (constant engine) IB
4023  * on the gfx ring.  IBs are usually generated by userspace
4024  * acceleration drivers and submitted to the kernel for
4025  * sheduling on the ring.  This function schedules the IB
4026  * on the gfx ring for execution by the GPU.
4027  */
4028 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4029 {
4030         struct radeon_ring *ring = &rdev->ring[ib->ring];
4031         u32 header, control = INDIRECT_BUFFER_VALID;
4032
4033         if (ib->is_const_ib) {
4034                 /* set switch buffer packet before const IB */
4035                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4036                 radeon_ring_write(ring, 0);
4037
4038                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4039         } else {
4040                 u32 next_rptr;
4041                 if (ring->rptr_save_reg) {
4042                         next_rptr = ring->wptr + 3 + 4;
4043                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4044                         radeon_ring_write(ring, ((ring->rptr_save_reg -
4045                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
4046                         radeon_ring_write(ring, next_rptr);
4047                 } else if (rdev->wb.enabled) {
4048                         next_rptr = ring->wptr + 5 + 4;
4049                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4050                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4051                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
4052                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
4053                         radeon_ring_write(ring, next_rptr);
4054                 }
4055
4056                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4057         }
4058
4059         control |= ib->length_dw |
4060                 (ib->vm ? (ib->vm->id << 24) : 0);
4061
4062         radeon_ring_write(ring, header);
4063         radeon_ring_write(ring,
4064 #ifdef __BIG_ENDIAN
4065                           (2 << 0) |
4066 #endif
4067                           (ib->gpu_addr & 0xFFFFFFFC));
4068         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4069         radeon_ring_write(ring, control);
4070 }
4071
4072 /**
4073  * cik_ib_test - basic gfx ring IB test
4074  *
4075  * @rdev: radeon_device pointer
4076  * @ring: radeon_ring structure holding ring information
4077  *
4078  * Allocate an IB and execute it on the gfx ring (CIK).
4079  * Provides a basic gfx ring test to verify that IBs are working.
4080  * Returns 0 on success, error on failure.
4081  */
4082 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4083 {
4084         struct radeon_ib ib;
4085         uint32_t scratch;
4086         uint32_t tmp = 0;
4087         unsigned i;
4088         int r;
4089
4090         r = radeon_scratch_get(rdev, &scratch);
4091         if (r) {
4092                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4093                 return r;
4094         }
4095         WREG32(scratch, 0xCAFEDEAD);
4096         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4097         if (r) {
4098                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
4099                 radeon_scratch_free(rdev, scratch);
4100                 return r;
4101         }
4102         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4103         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4104         ib.ptr[2] = 0xDEADBEEF;
4105         ib.length_dw = 3;
4106         r = radeon_ib_schedule(rdev, &ib, NULL);
4107         if (r) {
4108                 radeon_scratch_free(rdev, scratch);
4109                 radeon_ib_free(rdev, &ib);
4110                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4111                 return r;
4112         }
4113         r = radeon_fence_wait(ib.fence, false);
4114         if (r) {
4115                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
4116                 radeon_scratch_free(rdev, scratch);
4117                 radeon_ib_free(rdev, &ib);
4118                 return r;
4119         }
4120         for (i = 0; i < rdev->usec_timeout; i++) {
4121                 tmp = RREG32(scratch);
4122                 if (tmp == 0xDEADBEEF)
4123                         break;
4124                 DRM_UDELAY(1);
4125         }
4126         if (i < rdev->usec_timeout) {
4127                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4128         } else {
4129                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4130                           scratch, tmp);
4131                 r = -EINVAL;
4132         }
4133         radeon_scratch_free(rdev, scratch);
4134         radeon_ib_free(rdev, &ib);
4135         return r;
4136 }
4137
4138 /*
4139  * CP.
4140  * On CIK, gfx and compute now have independant command processors.
4141  *
4142  * GFX
4143  * Gfx consists of a single ring and can process both gfx jobs and
4144  * compute jobs.  The gfx CP consists of three microengines (ME):
4145  * PFP - Pre-Fetch Parser
4146  * ME - Micro Engine
4147  * CE - Constant Engine
4148  * The PFP and ME make up what is considered the Drawing Engine (DE).
4149  * The CE is an asynchronous engine used for updating buffer desciptors
4150  * used by the DE so that they can be loaded into cache in parallel
4151  * while the DE is processing state update packets.
4152  *
4153  * Compute
4154  * The compute CP consists of two microengines (ME):
4155  * MEC1 - Compute MicroEngine 1
4156  * MEC2 - Compute MicroEngine 2
4157  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4158  * The queues are exposed to userspace and are programmed directly
4159  * by the compute runtime.
4160  */
4161 /**
4162  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4163  *
4164  * @rdev: radeon_device pointer
4165  * @enable: enable or disable the MEs
4166  *
4167  * Halts or unhalts the gfx MEs.
4168  */
4169 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4170 {
4171         if (enable)
4172                 WREG32(CP_ME_CNTL, 0);
4173         else {
4174                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4175                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
4176                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4177                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4178         }
4179         udelay(50);
4180 }
4181
4182 /**
4183  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4184  *
4185  * @rdev: radeon_device pointer
4186  *
4187  * Loads the gfx PFP, ME, and CE ucode.
4188  * Returns 0 for success, -EINVAL if the ucode is not available.
4189  */
4190 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4191 {
4192         int i;
4193
4194         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4195                 return -EINVAL;
4196
4197         cik_cp_gfx_enable(rdev, false);
4198
4199         if (rdev->new_fw) {
4200                 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4201                         (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4202                 const struct gfx_firmware_header_v1_0 *ce_hdr =
4203                         (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4204                 const struct gfx_firmware_header_v1_0 *me_hdr =
4205                         (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4206                 const __le32 *fw_data;
4207                 u32 fw_size;
4208
4209                 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4210                 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4211                 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4212
4213                 /* PFP */
4214                 fw_data = (const __le32 *)
4215                         (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4216                 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4217                 WREG32(CP_PFP_UCODE_ADDR, 0);
4218                 for (i = 0; i < fw_size; i++)
4219                         WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4220                 WREG32(CP_PFP_UCODE_ADDR, 0);
4221
4222                 /* CE */
4223                 fw_data = (const __le32 *)
4224                         (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4225                 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4226                 WREG32(CP_CE_UCODE_ADDR, 0);
4227                 for (i = 0; i < fw_size; i++)
4228                         WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4229                 WREG32(CP_CE_UCODE_ADDR, 0);
4230
4231                 /* ME */
4232                 fw_data = (const __be32 *)
4233                         (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4234                 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4235                 WREG32(CP_ME_RAM_WADDR, 0);
4236                 for (i = 0; i < fw_size; i++)
4237                         WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4238                 WREG32(CP_ME_RAM_WADDR, 0);
4239         } else {
4240                 const __be32 *fw_data;
4241
4242                 /* PFP */
4243                 fw_data = (const __be32 *)rdev->pfp_fw->data;
4244                 WREG32(CP_PFP_UCODE_ADDR, 0);
4245                 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4246                         WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4247                 WREG32(CP_PFP_UCODE_ADDR, 0);
4248
4249                 /* CE */
4250                 fw_data = (const __be32 *)rdev->ce_fw->data;
4251                 WREG32(CP_CE_UCODE_ADDR, 0);
4252                 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4253                         WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4254                 WREG32(CP_CE_UCODE_ADDR, 0);
4255
4256                 /* ME */
4257                 fw_data = (const __be32 *)rdev->me_fw->data;
4258                 WREG32(CP_ME_RAM_WADDR, 0);
4259                 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4260                         WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4261                 WREG32(CP_ME_RAM_WADDR, 0);
4262         }
4263
4264         WREG32(CP_PFP_UCODE_ADDR, 0);
4265         WREG32(CP_CE_UCODE_ADDR, 0);
4266         WREG32(CP_ME_RAM_WADDR, 0);
4267         WREG32(CP_ME_RAM_RADDR, 0);
4268         return 0;
4269 }
4270
4271 /**
4272  * cik_cp_gfx_start - start the gfx ring
4273  *
4274  * @rdev: radeon_device pointer
4275  *
4276  * Enables the ring and loads the clear state context and other
4277  * packets required to init the ring.
4278  * Returns 0 for success, error for failure.
4279  */
4280 static int cik_cp_gfx_start(struct radeon_device *rdev)
4281 {
4282         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4283         int r, i;
4284
4285         /* init the CP */
4286         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4287         WREG32(CP_ENDIAN_SWAP, 0);
4288         WREG32(CP_DEVICE_ID, 1);
4289
4290         cik_cp_gfx_enable(rdev, true);
4291
4292         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4293         if (r) {
4294                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4295                 return r;
4296         }
4297
4298         /* init the CE partitions.  CE only used for gfx on CIK */
4299         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4300         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4301         radeon_ring_write(ring, 0xc000);
4302         radeon_ring_write(ring, 0xc000);
4303
4304         /* setup clear context state */
4305         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4306         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4307
4308         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4309         radeon_ring_write(ring, 0x80000000);
4310         radeon_ring_write(ring, 0x80000000);
4311
4312         for (i = 0; i < cik_default_size; i++)
4313                 radeon_ring_write(ring, cik_default_state[i]);
4314
4315         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4316         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4317
4318         /* set clear context state */
4319         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4320         radeon_ring_write(ring, 0);
4321
4322         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4323         radeon_ring_write(ring, 0x00000316);
4324         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4325         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4326
4327         radeon_ring_unlock_commit(rdev, ring);
4328
4329         return 0;
4330 }
4331
4332 /**
4333  * cik_cp_gfx_fini - stop the gfx ring
4334  *
4335  * @rdev: radeon_device pointer
4336  *
4337  * Stop the gfx ring and tear down the driver ring
4338  * info.
4339  */
4340 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4341 {
4342         cik_cp_gfx_enable(rdev, false);
4343         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4344 }
4345
4346 /**
4347  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4348  *
4349  * @rdev: radeon_device pointer
4350  *
4351  * Program the location and size of the gfx ring buffer
4352  * and test it to make sure it's working.
4353  * Returns 0 for success, error for failure.
4354  */
4355 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4356 {
4357         struct radeon_ring *ring;
4358         u32 tmp;
4359         u32 rb_bufsz;
4360         u64 rb_addr;
4361         int r;
4362
4363         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4364         if (rdev->family != CHIP_HAWAII)
4365                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4366
4367         /* Set the write pointer delay */
4368         WREG32(CP_RB_WPTR_DELAY, 0);
4369
4370         /* set the RB to use vmid 0 */
4371         WREG32(CP_RB_VMID, 0);
4372
4373         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4374
4375         /* ring 0 - compute and gfx */
4376         /* Set ring buffer size */
4377         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4378         rb_bufsz = order_base_2(ring->ring_size / 8);
4379         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4380 #ifdef __BIG_ENDIAN
4381         tmp |= BUF_SWAP_32BIT;
4382 #endif
4383         WREG32(CP_RB0_CNTL, tmp);
4384
4385         /* Initialize the ring buffer's read and write pointers */
4386         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4387         ring->wptr = 0;
4388         WREG32(CP_RB0_WPTR, ring->wptr);
4389
4390         /* set the wb address wether it's enabled or not */
4391         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4392         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4393
4394         /* scratch register shadowing is no longer supported */
4395         WREG32(SCRATCH_UMSK, 0);
4396
4397         if (!rdev->wb.enabled)
4398                 tmp |= RB_NO_UPDATE;
4399
4400         mdelay(1);
4401         WREG32(CP_RB0_CNTL, tmp);
4402
4403         rb_addr = ring->gpu_addr >> 8;
4404         WREG32(CP_RB0_BASE, rb_addr);
4405         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4406
4407         /* start the ring */
4408         cik_cp_gfx_start(rdev);
4409         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4410         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4411         if (r) {
4412                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4413                 return r;
4414         }
4415
4416         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4417                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4418
4419         return 0;
4420 }
4421
4422 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4423                      struct radeon_ring *ring)
4424 {
4425         u32 rptr;
4426
4427         if (rdev->wb.enabled)
4428                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4429         else
4430                 rptr = RREG32(CP_RB0_RPTR);
4431
4432         return rptr;
4433 }
4434
4435 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4436                      struct radeon_ring *ring)
4437 {
4438         u32 wptr;
4439
4440         wptr = RREG32(CP_RB0_WPTR);
4441
4442         return wptr;
4443 }
4444
4445 void cik_gfx_set_wptr(struct radeon_device *rdev,
4446                       struct radeon_ring *ring)
4447 {
4448         WREG32(CP_RB0_WPTR, ring->wptr);
4449         (void)RREG32(CP_RB0_WPTR);
4450 }
4451
4452 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4453                          struct radeon_ring *ring)
4454 {
4455         u32 rptr;
4456
4457         if (rdev->wb.enabled) {
4458                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4459         } else {
4460                 mutex_lock(&rdev->srbm_mutex);
4461                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4462                 rptr = RREG32(CP_HQD_PQ_RPTR);
4463                 cik_srbm_select(rdev, 0, 0, 0, 0);
4464                 mutex_unlock(&rdev->srbm_mutex);
4465         }
4466
4467         return rptr;
4468 }
4469
4470 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4471                          struct radeon_ring *ring)
4472 {
4473         u32 wptr;
4474
4475         if (rdev->wb.enabled) {
4476                 /* XXX check if swapping is necessary on BE */
4477                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4478         } else {
4479                 mutex_lock(&rdev->srbm_mutex);
4480                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4481                 wptr = RREG32(CP_HQD_PQ_WPTR);
4482                 cik_srbm_select(rdev, 0, 0, 0, 0);
4483                 mutex_unlock(&rdev->srbm_mutex);
4484         }
4485
4486         return wptr;
4487 }
4488
4489 void cik_compute_set_wptr(struct radeon_device *rdev,
4490                           struct radeon_ring *ring)
4491 {
4492         /* XXX check if swapping is necessary on BE */
4493         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4494         WDOORBELL32(ring->doorbell_index, ring->wptr);
4495 }
4496
4497 /**
4498  * cik_cp_compute_enable - enable/disable the compute CP MEs
4499  *
4500  * @rdev: radeon_device pointer
4501  * @enable: enable or disable the MEs
4502  *
4503  * Halts or unhalts the compute MEs.
4504  */
4505 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4506 {
4507         if (enable)
4508                 WREG32(CP_MEC_CNTL, 0);
4509         else {
4510                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4511                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4512                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4513         }
4514         udelay(50);
4515 }
4516
4517 /**
4518  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4519  *
4520  * @rdev: radeon_device pointer
4521  *
4522  * Loads the compute MEC1&2 ucode.
4523  * Returns 0 for success, -EINVAL if the ucode is not available.
4524  */
4525 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4526 {
4527         int i;
4528
4529         if (!rdev->mec_fw)
4530                 return -EINVAL;
4531
4532         cik_cp_compute_enable(rdev, false);
4533
4534         if (rdev->new_fw) {
4535                 const struct gfx_firmware_header_v1_0 *mec_hdr =
4536                         (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4537                 const __le32 *fw_data;
4538                 u32 fw_size;
4539
4540                 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4541
4542                 /* MEC1 */
4543                 fw_data = (const __le32 *)
4544                         (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4545                 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4546                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4547                 for (i = 0; i < fw_size; i++)
4548                         WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4549                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4550
4551                 /* MEC2 */
4552                 if (rdev->family == CHIP_KAVERI) {
4553                         const struct gfx_firmware_header_v1_0 *mec2_hdr =
4554                                 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4555
4556                         fw_data = (const __le32 *)
4557                                 (rdev->mec2_fw->data +
4558                                  le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4559                         fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4560                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4561                         for (i = 0; i < fw_size; i++)
4562                                 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4563                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4564                 }
4565         } else {
4566                 const __be32 *fw_data;
4567
4568                 /* MEC1 */
4569                 fw_data = (const __be32 *)rdev->mec_fw->data;
4570                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4571                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4572                         WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4573                 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4574
4575                 if (rdev->family == CHIP_KAVERI) {
4576                         /* MEC2 */
4577                         fw_data = (const __be32 *)rdev->mec_fw->data;
4578                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4579                         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4580                                 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4581                         WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4582                 }
4583         }
4584
4585         return 0;
4586 }
4587
4588 /**
4589  * cik_cp_compute_start - start the compute queues
4590  *
4591  * @rdev: radeon_device pointer
4592  *
4593  * Enable the compute queues.
4594  * Returns 0 for success, error for failure.
4595  */
4596 static int cik_cp_compute_start(struct radeon_device *rdev)
4597 {
4598         cik_cp_compute_enable(rdev, true);
4599
4600         return 0;
4601 }
4602
4603 /**
4604  * cik_cp_compute_fini - stop the compute queues
4605  *
4606  * @rdev: radeon_device pointer
4607  *
4608  * Stop the compute queues and tear down the driver queue
4609  * info.
4610  */
4611 static void cik_cp_compute_fini(struct radeon_device *rdev)
4612 {
4613         int i, idx, r;
4614
4615         cik_cp_compute_enable(rdev, false);
4616
4617         for (i = 0; i < 2; i++) {
4618                 if (i == 0)
4619                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4620                 else
4621                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4622
4623                 if (rdev->ring[idx].mqd_obj) {
4624                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4625                         if (unlikely(r != 0))
4626                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4627
4628                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4629                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4630
4631                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4632                         rdev->ring[idx].mqd_obj = NULL;
4633                 }
4634         }
4635 }
4636
4637 static void cik_mec_fini(struct radeon_device *rdev)
4638 {
4639         int r;
4640
4641         if (rdev->mec.hpd_eop_obj) {
4642                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4643                 if (unlikely(r != 0))
4644                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4645                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4646                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4647
4648                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4649                 rdev->mec.hpd_eop_obj = NULL;
4650         }
4651 }
4652
4653 #define MEC_HPD_SIZE 2048
4654
4655 static int cik_mec_init(struct radeon_device *rdev)
4656 {
4657         int r;
4658         u32 *hpd;
4659
4660         /*
4661          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4662          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4663          */
4664         if (rdev->family == CHIP_KAVERI)
4665                 rdev->mec.num_mec = 2;
4666         else
4667                 rdev->mec.num_mec = 1;
4668         rdev->mec.num_pipe = 4;
4669         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4670
4671         if (rdev->mec.hpd_eop_obj == NULL) {
4672                 r = radeon_bo_create(rdev,
4673                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4674                                      PAGE_SIZE, true,
4675                                      RADEON_GEM_DOMAIN_GTT, 0, NULL,
4676                                      &rdev->mec.hpd_eop_obj);
4677                 if (r) {
4678                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4679                         return r;
4680                 }
4681         }
4682
4683         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4684         if (unlikely(r != 0)) {
4685                 cik_mec_fini(rdev);
4686                 return r;
4687         }
4688         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4689                           &rdev->mec.hpd_eop_gpu_addr);
4690         if (r) {
4691                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4692                 cik_mec_fini(rdev);
4693                 return r;
4694         }
4695         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4696         if (r) {
4697                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4698                 cik_mec_fini(rdev);
4699                 return r;
4700         }
4701
4702         /* clear memory.  Not sure if this is required or not */
4703         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4704
4705         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4706         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4707
4708         return 0;
4709 }
4710
4711 struct hqd_registers
4712 {
4713         u32 cp_mqd_base_addr;
4714         u32 cp_mqd_base_addr_hi;
4715         u32 cp_hqd_active;
4716         u32 cp_hqd_vmid;
4717         u32 cp_hqd_persistent_state;
4718         u32 cp_hqd_pipe_priority;
4719         u32 cp_hqd_queue_priority;
4720         u32 cp_hqd_quantum;
4721         u32 cp_hqd_pq_base;
4722         u32 cp_hqd_pq_base_hi;
4723         u32 cp_hqd_pq_rptr;
4724         u32 cp_hqd_pq_rptr_report_addr;
4725         u32 cp_hqd_pq_rptr_report_addr_hi;
4726         u32 cp_hqd_pq_wptr_poll_addr;
4727         u32 cp_hqd_pq_wptr_poll_addr_hi;
4728         u32 cp_hqd_pq_doorbell_control;
4729         u32 cp_hqd_pq_wptr;
4730         u32 cp_hqd_pq_control;
4731         u32 cp_hqd_ib_base_addr;
4732         u32 cp_hqd_ib_base_addr_hi;
4733         u32 cp_hqd_ib_rptr;
4734         u32 cp_hqd_ib_control;
4735         u32 cp_hqd_iq_timer;
4736         u32 cp_hqd_iq_rptr;
4737         u32 cp_hqd_dequeue_request;
4738         u32 cp_hqd_dma_offload;
4739         u32 cp_hqd_sema_cmd;
4740         u32 cp_hqd_msg_type;
4741         u32 cp_hqd_atomic0_preop_lo;
4742         u32 cp_hqd_atomic0_preop_hi;
4743         u32 cp_hqd_atomic1_preop_lo;
4744         u32 cp_hqd_atomic1_preop_hi;
4745         u32 cp_hqd_hq_scheduler0;
4746         u32 cp_hqd_hq_scheduler1;
4747         u32 cp_mqd_control;
4748 };
4749
4750 struct bonaire_mqd
4751 {
4752         u32 header;
4753         u32 dispatch_initiator;
4754         u32 dimensions[3];
4755         u32 start_idx[3];
4756         u32 num_threads[3];
4757         u32 pipeline_stat_enable;
4758         u32 perf_counter_enable;
4759         u32 pgm[2];
4760         u32 tba[2];
4761         u32 tma[2];
4762         u32 pgm_rsrc[2];
4763         u32 vmid;
4764         u32 resource_limits;
4765         u32 static_thread_mgmt01[2];
4766         u32 tmp_ring_size;
4767         u32 static_thread_mgmt23[2];
4768         u32 restart[3];
4769         u32 thread_trace_enable;
4770         u32 reserved1;
4771         u32 user_data[16];
4772         u32 vgtcs_invoke_count[2];
4773         struct hqd_registers queue_state;
4774         u32 dequeue_cntr;
4775         u32 interrupt_queue[64];
4776 };
4777
4778 /**
4779  * cik_cp_compute_resume - setup the compute queue registers
4780  *
4781  * @rdev: radeon_device pointer
4782  *
4783  * Program the compute queues and test them to make sure they
4784  * are working.
4785  * Returns 0 for success, error for failure.
4786  */
4787 static int cik_cp_compute_resume(struct radeon_device *rdev)
4788 {
4789         int r, i, idx;
4790         u32 tmp;
4791         bool use_doorbell = true;
4792         u64 hqd_gpu_addr;
4793         u64 mqd_gpu_addr;
4794         u64 eop_gpu_addr;
4795         u64 wb_gpu_addr;
4796         u32 *buf;
4797         struct bonaire_mqd *mqd;
4798
4799         r = cik_cp_compute_start(rdev);
4800         if (r)
4801                 return r;
4802
4803         /* fix up chicken bits */
4804         tmp = RREG32(CP_CPF_DEBUG);
4805         tmp |= (1 << 23);
4806         WREG32(CP_CPF_DEBUG, tmp);
4807
4808         /* init the pipes */
4809         mutex_lock(&rdev->srbm_mutex);
4810         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4811                 int me = (i < 4) ? 1 : 2;
4812                 int pipe = (i < 4) ? i : (i - 4);
4813
4814                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4815
4816                 cik_srbm_select(rdev, me, pipe, 0, 0);
4817
4818                 /* write the EOP addr */
4819                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4820                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4821
4822                 /* set the VMID assigned */
4823                 WREG32(CP_HPD_EOP_VMID, 0);
4824
4825                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4826                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4827                 tmp &= ~EOP_SIZE_MASK;
4828                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4829                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4830         }
4831         cik_srbm_select(rdev, 0, 0, 0, 0);
4832         mutex_unlock(&rdev->srbm_mutex);
4833
4834         /* init the queues.  Just two for now. */
4835         for (i = 0; i < 2; i++) {
4836                 if (i == 0)
4837                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4838                 else
4839                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4840
4841                 if (rdev->ring[idx].mqd_obj == NULL) {
4842                         r = radeon_bo_create(rdev,
4843                                              sizeof(struct bonaire_mqd),
4844                                              PAGE_SIZE, true,
4845                                              RADEON_GEM_DOMAIN_GTT, 0, NULL,
4846                                              &rdev->ring[idx].mqd_obj);
4847                         if (r) {
4848                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4849                                 return r;
4850                         }
4851                 }
4852
4853                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4854                 if (unlikely(r != 0)) {
4855                         cik_cp_compute_fini(rdev);
4856                         return r;
4857                 }
4858                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4859                                   &mqd_gpu_addr);
4860                 if (r) {
4861                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4862                         cik_cp_compute_fini(rdev);
4863                         return r;
4864                 }
4865                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4866                 if (r) {
4867                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4868                         cik_cp_compute_fini(rdev);
4869                         return r;
4870                 }
4871
4872                 /* init the mqd struct */
4873                 memset(buf, 0, sizeof(struct bonaire_mqd));
4874
4875                 mqd = (struct bonaire_mqd *)buf;
4876                 mqd->header = 0xC0310800;
4877                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4878                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4879                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4880                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4881
4882                 mutex_lock(&rdev->srbm_mutex);
4883                 cik_srbm_select(rdev, rdev->ring[idx].me,
4884                                 rdev->ring[idx].pipe,
4885                                 rdev->ring[idx].queue, 0);
4886
4887                 /* disable wptr polling */
4888                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4889                 tmp &= ~WPTR_POLL_EN;
4890                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4891
4892                 /* enable doorbell? */
4893                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4894                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4895                 if (use_doorbell)
4896                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4897                 else
4898                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4899                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4900                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4901
4902                 /* disable the queue if it's active */
4903                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4904                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4905                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4906                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4907                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4908                         for (i = 0; i < rdev->usec_timeout; i++) {
4909                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4910                                         break;
4911                                 udelay(1);
4912                         }
4913                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4914                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4915                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4916                 }
4917
4918                 /* set the pointer to the MQD */
4919                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4920                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4921                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4922                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4923                 /* set MQD vmid to 0 */
4924                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4925                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4926                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4927
4928                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4929                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4930                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4931                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4932                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4933                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4934
4935                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4936                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4937                 mqd->queue_state.cp_hqd_pq_control &=
4938                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4939
4940                 mqd->queue_state.cp_hqd_pq_control |=
4941                         order_base_2(rdev->ring[idx].ring_size / 8);
4942                 mqd->queue_state.cp_hqd_pq_control |=
4943                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4944 #ifdef __BIG_ENDIAN
4945                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4946 #endif
4947                 mqd->queue_state.cp_hqd_pq_control &=
4948                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4949                 mqd->queue_state.cp_hqd_pq_control |=
4950                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4951                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4952
4953                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4954                 if (i == 0)
4955                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4956                 else
4957                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4958                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4959                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4960                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4961                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4962                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4963
4964                 /* set the wb address wether it's enabled or not */
4965                 if (i == 0)
4966                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4967                 else
4968                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4969                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4970                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4971                         upper_32_bits(wb_gpu_addr) & 0xffff;
4972                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4973                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4974                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4975                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4976
4977                 /* enable the doorbell if requested */
4978                 if (use_doorbell) {
4979                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4980                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4981                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4982                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4983                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4984                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4985                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4986                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4987
4988                 } else {
4989                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4990                 }
4991                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4992                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4993
4994                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4995                 rdev->ring[idx].wptr = 0;
4996                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4997                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4998                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4999
5000                 /* set the vmid for the queue */
5001                 mqd->queue_state.cp_hqd_vmid = 0;
5002                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5003
5004                 /* activate the queue */
5005                 mqd->queue_state.cp_hqd_active = 1;
5006                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5007
5008                 cik_srbm_select(rdev, 0, 0, 0, 0);
5009                 mutex_unlock(&rdev->srbm_mutex);
5010
5011                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5012                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5013
5014                 rdev->ring[idx].ready = true;
5015                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5016                 if (r)
5017                         rdev->ring[idx].ready = false;
5018         }
5019
5020         return 0;
5021 }
5022
5023 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5024 {
5025         cik_cp_gfx_enable(rdev, enable);
5026         cik_cp_compute_enable(rdev, enable);
5027 }
5028
5029 static int cik_cp_load_microcode(struct radeon_device *rdev)
5030 {
5031         int r;
5032
5033         r = cik_cp_gfx_load_microcode(rdev);
5034         if (r)
5035                 return r;
5036         r = cik_cp_compute_load_microcode(rdev);
5037         if (r)
5038                 return r;
5039
5040         return 0;
5041 }
5042
5043 static void cik_cp_fini(struct radeon_device *rdev)
5044 {
5045         cik_cp_gfx_fini(rdev);
5046         cik_cp_compute_fini(rdev);
5047 }
5048
5049 static int cik_cp_resume(struct radeon_device *rdev)
5050 {
5051         int r;
5052
5053         cik_enable_gui_idle_interrupt(rdev, false);
5054
5055         r = cik_cp_load_microcode(rdev);
5056         if (r)
5057                 return r;
5058
5059         r = cik_cp_gfx_resume(rdev);
5060         if (r)
5061                 return r;
5062         r = cik_cp_compute_resume(rdev);
5063         if (r)
5064                 return r;
5065
5066         cik_enable_gui_idle_interrupt(rdev, true);
5067
5068         return 0;
5069 }
5070
5071 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
5072 {
5073         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
5074                 RREG32(GRBM_STATUS));
5075         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
5076                 RREG32(GRBM_STATUS2));
5077         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
5078                 RREG32(GRBM_STATUS_SE0));
5079         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
5080                 RREG32(GRBM_STATUS_SE1));
5081         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
5082                 RREG32(GRBM_STATUS_SE2));
5083         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
5084                 RREG32(GRBM_STATUS_SE3));
5085         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
5086                 RREG32(SRBM_STATUS));
5087         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
5088                 RREG32(SRBM_STATUS2));
5089         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
5090                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5091         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
5092                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
5093         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5094         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
5095                  RREG32(CP_STALLED_STAT1));
5096         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
5097                  RREG32(CP_STALLED_STAT2));
5098         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
5099                  RREG32(CP_STALLED_STAT3));
5100         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
5101                  RREG32(CP_CPF_BUSY_STAT));
5102         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
5103                  RREG32(CP_CPF_STALLED_STAT1));
5104         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5105         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5106         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
5107                  RREG32(CP_CPC_STALLED_STAT1));
5108         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
5109 }
5110
5111 /**
5112  * cik_gpu_check_soft_reset - check which blocks are busy
5113  *
5114  * @rdev: radeon_device pointer
5115  *
5116  * Check which blocks are busy and return the relevant reset
5117  * mask to be used by cik_gpu_soft_reset().
5118  * Returns a mask of the blocks to be reset.
5119  */
5120 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
5121 {
5122         u32 reset_mask = 0;
5123         u32 tmp;
5124
5125         /* GRBM_STATUS */
5126         tmp = RREG32(GRBM_STATUS);
5127         if (tmp & (PA_BUSY | SC_BUSY |
5128                    BCI_BUSY | SX_BUSY |
5129                    TA_BUSY | VGT_BUSY |
5130                    DB_BUSY | CB_BUSY |
5131                    GDS_BUSY | SPI_BUSY |
5132                    IA_BUSY | IA_BUSY_NO_DMA))
5133                 reset_mask |= RADEON_RESET_GFX;
5134
5135         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5136                 reset_mask |= RADEON_RESET_CP;
5137
5138         /* GRBM_STATUS2 */
5139         tmp = RREG32(GRBM_STATUS2);
5140         if (tmp & RLC_BUSY)
5141                 reset_mask |= RADEON_RESET_RLC;
5142
5143         /* SDMA0_STATUS_REG */
5144         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5145         if (!(tmp & SDMA_IDLE))
5146                 reset_mask |= RADEON_RESET_DMA;
5147
5148         /* SDMA1_STATUS_REG */
5149         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5150         if (!(tmp & SDMA_IDLE))
5151                 reset_mask |= RADEON_RESET_DMA1;
5152
5153         /* SRBM_STATUS2 */
5154         tmp = RREG32(SRBM_STATUS2);
5155         if (tmp & SDMA_BUSY)
5156                 reset_mask |= RADEON_RESET_DMA;
5157
5158         if (tmp & SDMA1_BUSY)
5159                 reset_mask |= RADEON_RESET_DMA1;
5160
5161         /* SRBM_STATUS */
5162         tmp = RREG32(SRBM_STATUS);
5163
5164         if (tmp & IH_BUSY)
5165                 reset_mask |= RADEON_RESET_IH;
5166
5167         if (tmp & SEM_BUSY)
5168                 reset_mask |= RADEON_RESET_SEM;
5169
5170         if (tmp & GRBM_RQ_PENDING)
5171                 reset_mask |= RADEON_RESET_GRBM;
5172
5173         if (tmp & VMC_BUSY)
5174                 reset_mask |= RADEON_RESET_VMC;
5175
5176         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5177                    MCC_BUSY | MCD_BUSY))
5178                 reset_mask |= RADEON_RESET_MC;
5179
5180         if (evergreen_is_display_hung(rdev))
5181                 reset_mask |= RADEON_RESET_DISPLAY;
5182
5183         /* Skip MC reset as it's mostly likely not hung, just busy */
5184         if (reset_mask & RADEON_RESET_MC) {
5185                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5186                 reset_mask &= ~RADEON_RESET_MC;
5187         }
5188
5189         return reset_mask;
5190 }
5191
5192 /**
5193  * cik_gpu_soft_reset - soft reset GPU
5194  *
5195  * @rdev: radeon_device pointer
5196  * @reset_mask: mask of which blocks to reset
5197  *
5198  * Soft reset the blocks specified in @reset_mask.
5199  */
5200 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
5201 {
5202         struct evergreen_mc_save save;
5203         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5204         u32 tmp;
5205
5206         if (reset_mask == 0)
5207                 return;
5208
5209         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
5210
5211         cik_print_gpu_status_regs(rdev);
5212         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5213                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5214         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5215                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5216
5217         /* disable CG/PG */
5218         cik_fini_pg(rdev);
5219         cik_fini_cg(rdev);
5220
5221         /* stop the rlc */
5222         cik_rlc_stop(rdev);
5223
5224         /* Disable GFX parsing/prefetching */
5225         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5226
5227         /* Disable MEC parsing/prefetching */
5228         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5229
5230         if (reset_mask & RADEON_RESET_DMA) {
5231                 /* sdma0 */
5232                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5233                 tmp |= SDMA_HALT;
5234                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5235         }
5236         if (reset_mask & RADEON_RESET_DMA1) {
5237                 /* sdma1 */
5238                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5239                 tmp |= SDMA_HALT;
5240                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5241         }
5242
5243         evergreen_mc_stop(rdev, &save);
5244         if (evergreen_mc_wait_for_idle(rdev)) {
5245                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5246         }
5247
5248         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5249                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
5250
5251         if (reset_mask & RADEON_RESET_CP) {
5252                 grbm_soft_reset |= SOFT_RESET_CP;
5253
5254                 srbm_soft_reset |= SOFT_RESET_GRBM;
5255         }
5256
5257         if (reset_mask & RADEON_RESET_DMA)
5258                 srbm_soft_reset |= SOFT_RESET_SDMA;
5259
5260         if (reset_mask & RADEON_RESET_DMA1)
5261                 srbm_soft_reset |= SOFT_RESET_SDMA1;
5262
5263         if (reset_mask & RADEON_RESET_DISPLAY)
5264                 srbm_soft_reset |= SOFT_RESET_DC;
5265
5266         if (reset_mask & RADEON_RESET_RLC)
5267                 grbm_soft_reset |= SOFT_RESET_RLC;
5268
5269         if (reset_mask & RADEON_RESET_SEM)
5270                 srbm_soft_reset |= SOFT_RESET_SEM;
5271
5272         if (reset_mask & RADEON_RESET_IH)
5273                 srbm_soft_reset |= SOFT_RESET_IH;
5274
5275         if (reset_mask & RADEON_RESET_GRBM)
5276                 srbm_soft_reset |= SOFT_RESET_GRBM;
5277
5278         if (reset_mask & RADEON_RESET_VMC)
5279                 srbm_soft_reset |= SOFT_RESET_VMC;
5280
5281         if (!(rdev->flags & RADEON_IS_IGP)) {
5282                 if (reset_mask & RADEON_RESET_MC)
5283                         srbm_soft_reset |= SOFT_RESET_MC;
5284         }
5285
5286         if (grbm_soft_reset) {
5287                 tmp = RREG32(GRBM_SOFT_RESET);
5288                 tmp |= grbm_soft_reset;
5289                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5290                 WREG32(GRBM_SOFT_RESET, tmp);
5291                 tmp = RREG32(GRBM_SOFT_RESET);
5292
5293                 udelay(50);
5294
5295                 tmp &= ~grbm_soft_reset;
5296                 WREG32(GRBM_SOFT_RESET, tmp);
5297                 tmp = RREG32(GRBM_SOFT_RESET);
5298         }
5299
5300         if (srbm_soft_reset) {
5301                 tmp = RREG32(SRBM_SOFT_RESET);
5302                 tmp |= srbm_soft_reset;
5303                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5304                 WREG32(SRBM_SOFT_RESET, tmp);
5305                 tmp = RREG32(SRBM_SOFT_RESET);
5306
5307                 udelay(50);
5308
5309                 tmp &= ~srbm_soft_reset;
5310                 WREG32(SRBM_SOFT_RESET, tmp);
5311                 tmp = RREG32(SRBM_SOFT_RESET);
5312         }
5313
5314         /* Wait a little for things to settle down */
5315         udelay(50);
5316
5317         evergreen_mc_resume(rdev, &save);
5318         udelay(50);
5319
5320         cik_print_gpu_status_regs(rdev);
5321 }
5322
5323 struct kv_reset_save_regs {
5324         u32 gmcon_reng_execute;
5325         u32 gmcon_misc;
5326         u32 gmcon_misc3;
5327 };
5328
5329 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5330                                    struct kv_reset_save_regs *save)
5331 {
5332         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5333         save->gmcon_misc = RREG32(GMCON_MISC);
5334         save->gmcon_misc3 = RREG32(GMCON_MISC3);
5335
5336         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5337         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5338                                                 STCTRL_STUTTER_EN));
5339 }
5340
5341 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5342                                       struct kv_reset_save_regs *save)
5343 {
5344         int i;
5345
5346         WREG32(GMCON_PGFSM_WRITE, 0);
5347         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5348
5349         for (i = 0; i < 5; i++)
5350                 WREG32(GMCON_PGFSM_WRITE, 0);
5351
5352         WREG32(GMCON_PGFSM_WRITE, 0);
5353         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5354
5355         for (i = 0; i < 5; i++)
5356                 WREG32(GMCON_PGFSM_WRITE, 0);
5357
5358         WREG32(GMCON_PGFSM_WRITE, 0x210000);
5359         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5360
5361         for (i = 0; i < 5; i++)
5362                 WREG32(GMCON_PGFSM_WRITE, 0);
5363
5364         WREG32(GMCON_PGFSM_WRITE, 0x21003);
5365         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5366
5367         for (i = 0; i < 5; i++)
5368                 WREG32(GMCON_PGFSM_WRITE, 0);
5369
5370         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5371         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5372
5373         for (i = 0; i < 5; i++)
5374                 WREG32(GMCON_PGFSM_WRITE, 0);
5375
5376         WREG32(GMCON_PGFSM_WRITE, 0);
5377         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5378
5379         for (i = 0; i < 5; i++)
5380                 WREG32(GMCON_PGFSM_WRITE, 0);
5381
5382         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5383         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5384
5385         for (i = 0; i < 5; i++)
5386                 WREG32(GMCON_PGFSM_WRITE, 0);
5387
5388         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5389         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5390
5391         for (i = 0; i < 5; i++)
5392                 WREG32(GMCON_PGFSM_WRITE, 0);
5393
5394         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5395         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5396
5397         for (i = 0; i < 5; i++)
5398                 WREG32(GMCON_PGFSM_WRITE, 0);
5399
5400         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5401         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5402
5403         for (i = 0; i < 5; i++)
5404                 WREG32(GMCON_PGFSM_WRITE, 0);
5405
5406         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5407         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5408
5409         WREG32(GMCON_MISC3, save->gmcon_misc3);
5410         WREG32(GMCON_MISC, save->gmcon_misc);
5411         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5412 }
5413
5414 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5415 {
5416         struct evergreen_mc_save save;
5417         struct kv_reset_save_regs kv_save = { 0 };
5418         u32 tmp, i;
5419
5420         dev_info(rdev->dev, "GPU pci config reset\n");
5421
5422         /* disable dpm? */
5423
5424         /* disable cg/pg */
5425         cik_fini_pg(rdev);
5426         cik_fini_cg(rdev);
5427
5428         /* Disable GFX parsing/prefetching */
5429         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5430
5431         /* Disable MEC parsing/prefetching */
5432         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5433
5434         /* sdma0 */
5435         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5436         tmp |= SDMA_HALT;
5437         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5438         /* sdma1 */
5439         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5440         tmp |= SDMA_HALT;
5441         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5442         /* XXX other engines? */
5443
5444         /* halt the rlc, disable cp internal ints */
5445         cik_rlc_stop(rdev);
5446
5447         udelay(50);
5448
5449         /* disable mem access */
5450         evergreen_mc_stop(rdev, &save);
5451         if (evergreen_mc_wait_for_idle(rdev)) {
5452                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5453         }
5454
5455         if (rdev->flags & RADEON_IS_IGP)
5456                 kv_save_regs_for_reset(rdev, &kv_save);
5457
5458         /* disable BM */
5459         pci_clear_master(rdev->pdev);
5460         /* reset */
5461         radeon_pci_config_reset(rdev);
5462
5463         udelay(100);
5464
5465         /* wait for asic to come out of reset */
5466         for (i = 0; i < rdev->usec_timeout; i++) {
5467                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5468                         break;
5469                 udelay(1);
5470         }
5471
5472         /* does asic init need to be run first??? */
5473         if (rdev->flags & RADEON_IS_IGP)
5474                 kv_restore_regs_for_reset(rdev, &kv_save);
5475 }
5476
5477 /**
5478  * cik_asic_reset - soft reset GPU
5479  *
5480  * @rdev: radeon_device pointer
5481  *
5482  * Look up which blocks are hung and attempt
5483  * to reset them.
5484  * Returns 0 for success.
5485  */
5486 int cik_asic_reset(struct radeon_device *rdev)
5487 {
5488         u32 reset_mask;
5489
5490         reset_mask = cik_gpu_check_soft_reset(rdev);
5491
5492         if (reset_mask)
5493                 r600_set_bios_scratch_engine_hung(rdev, true);
5494
5495         /* try soft reset */
5496         cik_gpu_soft_reset(rdev, reset_mask);
5497
5498         reset_mask = cik_gpu_check_soft_reset(rdev);
5499
5500         /* try pci config reset */
5501         if (reset_mask && radeon_hard_reset)
5502                 cik_gpu_pci_config_reset(rdev);
5503
5504         reset_mask = cik_gpu_check_soft_reset(rdev);
5505
5506         if (!reset_mask)
5507                 r600_set_bios_scratch_engine_hung(rdev, false);
5508
5509         return 0;
5510 }
5511
5512 /**
5513  * cik_gfx_is_lockup - check if the 3D engine is locked up
5514  *
5515  * @rdev: radeon_device pointer
5516  * @ring: radeon_ring structure holding ring information
5517  *
5518  * Check if the 3D engine is locked up (CIK).
5519  * Returns true if the engine is locked, false if not.
5520  */
5521 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5522 {
5523         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5524
5525         if (!(reset_mask & (RADEON_RESET_GFX |
5526                             RADEON_RESET_COMPUTE |
5527                             RADEON_RESET_CP))) {
5528                 radeon_ring_lockup_update(rdev, ring);
5529                 return false;
5530         }
5531         return radeon_ring_test_lockup(rdev, ring);
5532 }
5533
5534 /* MC */
5535 /**
5536  * cik_mc_program - program the GPU memory controller
5537  *
5538  * @rdev: radeon_device pointer
5539  *
5540  * Set the location of vram, gart, and AGP in the GPU's
5541  * physical address space (CIK).
5542  */
5543 static void cik_mc_program(struct radeon_device *rdev)
5544 {
5545         struct evergreen_mc_save save;
5546         u32 tmp;
5547         int i, j;
5548
5549         /* Initialize HDP */
5550         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5551                 WREG32((0x2c14 + j), 0x00000000);
5552                 WREG32((0x2c18 + j), 0x00000000);
5553                 WREG32((0x2c1c + j), 0x00000000);
5554                 WREG32((0x2c20 + j), 0x00000000);
5555                 WREG32((0x2c24 + j), 0x00000000);
5556         }
5557         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5558
5559         evergreen_mc_stop(rdev, &save);
5560         if (radeon_mc_wait_for_idle(rdev)) {
5561                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5562         }
5563         /* Lockout access through VGA aperture*/
5564         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5565         /* Update configuration */
5566         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5567                rdev->mc.vram_start >> 12);
5568         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5569                rdev->mc.vram_end >> 12);
5570         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5571                rdev->vram_scratch.gpu_addr >> 12);
5572         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5573         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5574         WREG32(MC_VM_FB_LOCATION, tmp);
5575         /* XXX double check these! */
5576         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5577         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5578         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5579         WREG32(MC_VM_AGP_BASE, 0);
5580         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5581         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5582         if (radeon_mc_wait_for_idle(rdev)) {
5583                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5584         }
5585         evergreen_mc_resume(rdev, &save);
5586         /* we need to own VRAM, so turn off the VGA renderer here
5587          * to stop it overwriting our objects */
5588         rv515_vga_render_disable(rdev);
5589 }
5590
5591 /**
5592  * cik_mc_init - initialize the memory controller driver params
5593  *
5594  * @rdev: radeon_device pointer
5595  *
5596  * Look up the amount of vram, vram width, and decide how to place
5597  * vram and gart within the GPU's physical address space (CIK).
5598  * Returns 0 for success.
5599  */
5600 static int cik_mc_init(struct radeon_device *rdev)
5601 {
5602         u32 tmp;
5603         int chansize, numchan;
5604
5605         /* Get VRAM informations */
5606         rdev->mc.vram_is_ddr = true;
5607         tmp = RREG32(MC_ARB_RAMCFG);
5608         if (tmp & CHANSIZE_MASK) {
5609                 chansize = 64;
5610         } else {
5611                 chansize = 32;
5612         }
5613         tmp = RREG32(MC_SHARED_CHMAP);
5614         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5615         case 0:
5616         default:
5617                 numchan = 1;
5618                 break;
5619         case 1:
5620                 numchan = 2;
5621                 break;
5622         case 2:
5623                 numchan = 4;
5624                 break;
5625         case 3:
5626                 numchan = 8;
5627                 break;
5628         case 4:
5629                 numchan = 3;
5630                 break;
5631         case 5:
5632                 numchan = 6;
5633                 break;
5634         case 6:
5635                 numchan = 10;
5636                 break;
5637         case 7:
5638                 numchan = 12;
5639                 break;
5640         case 8:
5641                 numchan = 16;
5642                 break;
5643         }
5644         rdev->mc.vram_width = numchan * chansize;
5645         /* Could aper size report 0 ? */
5646         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5647         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5648         /* size in MB on si */
5649         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5650         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5651         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5652         si_vram_gtt_location(rdev, &rdev->mc);
5653         radeon_update_bandwidth_info(rdev);
5654
5655         return 0;
5656 }
5657
5658 /*
5659  * GART
5660  * VMID 0 is the physical GPU addresses as used by the kernel.
5661  * VMIDs 1-15 are used for userspace clients and are handled
5662  * by the radeon vm/hsa code.
5663  */
5664 /**
5665  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5666  *
5667  * @rdev: radeon_device pointer
5668  *
5669  * Flush the TLB for the VMID 0 page table (CIK).
5670  */
5671 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5672 {
5673         /* flush hdp cache */
5674         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5675
5676         /* bits 0-15 are the VM contexts0-15 */
5677         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5678 }
5679
5680 /**
5681  * cik_pcie_gart_enable - gart enable
5682  *
5683  * @rdev: radeon_device pointer
5684  *
5685  * This sets up the TLBs, programs the page tables for VMID0,
5686  * sets up the hw for VMIDs 1-15 which are allocated on
5687  * demand, and sets up the global locations for the LDS, GDS,
5688  * and GPUVM for FSA64 clients (CIK).
5689  * Returns 0 for success, errors for failure.
5690  */
5691 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5692 {
5693         int r, i;
5694
5695         if (rdev->gart.robj == NULL) {
5696                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5697                 return -EINVAL;
5698         }
5699         r = radeon_gart_table_vram_pin(rdev);
5700         if (r)
5701                 return r;
5702         /* Setup TLB control */
5703         WREG32(MC_VM_MX_L1_TLB_CNTL,
5704                (0xA << 7) |
5705                ENABLE_L1_TLB |
5706                ENABLE_L1_FRAGMENT_PROCESSING |
5707                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5708                ENABLE_ADVANCED_DRIVER_MODEL |
5709                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5710         /* Setup L2 cache */
5711         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5712                ENABLE_L2_FRAGMENT_PROCESSING |
5713                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5714                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5715                EFFECTIVE_L2_QUEUE_SIZE(7) |
5716                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5717         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5718         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5719                BANK_SELECT(4) |
5720                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5721         /* setup context0 */
5722         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5723         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5724         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5725         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5726                         (u32)(rdev->dummy_page.addr >> 12));
5727         WREG32(VM_CONTEXT0_CNTL2, 0);
5728         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5729                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5730
5731         WREG32(0x15D4, 0);
5732         WREG32(0x15D8, 0);
5733         WREG32(0x15DC, 0);
5734
5735         /* empty context1-15 */
5736         /* FIXME start with 4G, once using 2 level pt switch to full
5737          * vm size space
5738          */
5739         /* set vm size, must be a multiple of 4 */
5740         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5741         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5742         for (i = 1; i < 16; i++) {
5743                 if (i < 8)
5744                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5745                                rdev->gart.table_addr >> 12);
5746                 else
5747                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5748                                rdev->gart.table_addr >> 12);
5749         }
5750
5751         /* enable context1-15 */
5752         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5753                (u32)(rdev->dummy_page.addr >> 12));
5754         WREG32(VM_CONTEXT1_CNTL2, 4);
5755         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5756                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5757                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5758                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5759                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5760                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5761                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5762                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5763                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5764                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5765                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5766                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5767                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5768                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5769
5770         if (rdev->family == CHIP_KAVERI) {
5771                 u32 tmp = RREG32(CHUB_CONTROL);
5772                 tmp &= ~BYPASS_VM;
5773                 WREG32(CHUB_CONTROL, tmp);
5774         }
5775
5776         /* XXX SH_MEM regs */
5777         /* where to put LDS, scratch, GPUVM in FSA64 space */
5778         mutex_lock(&rdev->srbm_mutex);
5779         for (i = 0; i < 16; i++) {
5780                 cik_srbm_select(rdev, 0, 0, 0, i);
5781                 /* CP and shaders */
5782                 WREG32(SH_MEM_CONFIG, 0);
5783                 WREG32(SH_MEM_APE1_BASE, 1);
5784                 WREG32(SH_MEM_APE1_LIMIT, 0);
5785                 WREG32(SH_MEM_BASES, 0);
5786                 /* SDMA GFX */
5787                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5788                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5789                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5790                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5791                 /* XXX SDMA RLC - todo */
5792         }
5793         cik_srbm_select(rdev, 0, 0, 0, 0);
5794         mutex_unlock(&rdev->srbm_mutex);
5795
5796         cik_pcie_gart_tlb_flush(rdev);
5797         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5798                  (unsigned)(rdev->mc.gtt_size >> 20),
5799                  (unsigned long long)rdev->gart.table_addr);
5800         rdev->gart.ready = true;
5801         return 0;
5802 }
5803
5804 /**
5805  * cik_pcie_gart_disable - gart disable
5806  *
5807  * @rdev: radeon_device pointer
5808  *
5809  * This disables all VM page table (CIK).
5810  */
5811 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5812 {
5813         /* Disable all tables */
5814         WREG32(VM_CONTEXT0_CNTL, 0);
5815         WREG32(VM_CONTEXT1_CNTL, 0);
5816         /* Setup TLB control */
5817         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5818                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5819         /* Setup L2 cache */
5820         WREG32(VM_L2_CNTL,
5821                ENABLE_L2_FRAGMENT_PROCESSING |
5822                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5823                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5824                EFFECTIVE_L2_QUEUE_SIZE(7) |
5825                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5826         WREG32(VM_L2_CNTL2, 0);
5827         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5828                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5829         radeon_gart_table_vram_unpin(rdev);
5830 }
5831
5832 /**
5833  * cik_pcie_gart_fini - vm fini callback
5834  *
5835  * @rdev: radeon_device pointer
5836  *
5837  * Tears down the driver GART/VM setup (CIK).
5838  */
5839 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5840 {
5841         cik_pcie_gart_disable(rdev);
5842         radeon_gart_table_vram_free(rdev);
5843         radeon_gart_fini(rdev);
5844 }
5845
5846 /* vm parser */
5847 /**
5848  * cik_ib_parse - vm ib_parse callback
5849  *
5850  * @rdev: radeon_device pointer
5851  * @ib: indirect buffer pointer
5852  *
5853  * CIK uses hw IB checking so this is a nop (CIK).
5854  */
5855 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5856 {
5857         return 0;
5858 }
5859
5860 /*
5861  * vm
5862  * VMID 0 is the physical GPU addresses as used by the kernel.
5863  * VMIDs 1-15 are used for userspace clients and are handled
5864  * by the radeon vm/hsa code.
5865  */
5866 /**
5867  * cik_vm_init - cik vm init callback
5868  *
5869  * @rdev: radeon_device pointer
5870  *
5871  * Inits cik specific vm parameters (number of VMs, base of vram for
5872  * VMIDs 1-15) (CIK).
5873  * Returns 0 for success.
5874  */
5875 int cik_vm_init(struct radeon_device *rdev)
5876 {
5877         /* number of VMs */
5878         rdev->vm_manager.nvm = 16;
5879         /* base offset of vram pages */
5880         if (rdev->flags & RADEON_IS_IGP) {
5881                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5882                 tmp <<= 22;
5883                 rdev->vm_manager.vram_base_offset = tmp;
5884         } else
5885                 rdev->vm_manager.vram_base_offset = 0;
5886
5887         return 0;
5888 }
5889
5890 /**
5891  * cik_vm_fini - cik vm fini callback
5892  *
5893  * @rdev: radeon_device pointer
5894  *
5895  * Tear down any asic specific VM setup (CIK).
5896  */
5897 void cik_vm_fini(struct radeon_device *rdev)
5898 {
5899 }
5900
5901 /**
5902  * cik_vm_decode_fault - print human readable fault info
5903  *
5904  * @rdev: radeon_device pointer
5905  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5906  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5907  *
5908  * Print human readable fault information (CIK).
5909  */
5910 static void cik_vm_decode_fault(struct radeon_device *rdev,
5911                                 u32 status, u32 addr, u32 mc_client)
5912 {
5913         u32 mc_id;
5914         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5915         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5916         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5917                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5918
5919         if (rdev->family == CHIP_HAWAII)
5920                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5921         else
5922                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5923
5924         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5925                protections, vmid, addr,
5926                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5927                block, mc_client, mc_id);
5928 }
5929
5930 /**
5931  * cik_vm_flush - cik vm flush using the CP
5932  *
5933  * @rdev: radeon_device pointer
5934  *
5935  * Update the page table base and flush the VM TLB
5936  * using the CP (CIK).
5937  */
5938 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5939 {
5940         struct radeon_ring *ring = &rdev->ring[ridx];
5941         int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5942
5943         if (vm == NULL)
5944                 return;
5945
5946         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5947         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5948                                  WRITE_DATA_DST_SEL(0)));
5949         if (vm->id < 8) {
5950                 radeon_ring_write(ring,
5951                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5952         } else {
5953                 radeon_ring_write(ring,
5954                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5955         }
5956         radeon_ring_write(ring, 0);
5957         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5958
5959         /* update SH_MEM_* regs */
5960         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5961         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5962                                  WRITE_DATA_DST_SEL(0)));
5963         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5964         radeon_ring_write(ring, 0);
5965         radeon_ring_write(ring, VMID(vm->id));
5966
5967         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5968         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5969                                  WRITE_DATA_DST_SEL(0)));
5970         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5971         radeon_ring_write(ring, 0);
5972
5973         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5974         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5975         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5976         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5977
5978         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5979         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5980                                  WRITE_DATA_DST_SEL(0)));
5981         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5982         radeon_ring_write(ring, 0);
5983         radeon_ring_write(ring, VMID(0));
5984
5985         /* HDP flush */
5986         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5987
5988         /* bits 0-15 are the VM contexts0-15 */
5989         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5990         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5991                                  WRITE_DATA_DST_SEL(0)));
5992         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5993         radeon_ring_write(ring, 0);
5994         radeon_ring_write(ring, 1 << vm->id);
5995
5996         /* compute doesn't have PFP */
5997         if (usepfp) {
5998                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5999                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6000                 radeon_ring_write(ring, 0x0);
6001         }
6002 }
6003
6004 /*
6005  * RLC
6006  * The RLC is a multi-purpose microengine that handles a
6007  * variety of functions, the most important of which is
6008  * the interrupt controller.
6009  */
6010 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6011                                           bool enable)
6012 {
6013         u32 tmp = RREG32(CP_INT_CNTL_RING0);
6014
6015         if (enable)
6016                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6017         else
6018                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6019         WREG32(CP_INT_CNTL_RING0, tmp);
6020 }
6021
6022 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
6023 {
6024         u32 tmp;
6025
6026         tmp = RREG32(RLC_LB_CNTL);
6027         if (enable)
6028                 tmp |= LOAD_BALANCE_ENABLE;
6029         else
6030                 tmp &= ~LOAD_BALANCE_ENABLE;
6031         WREG32(RLC_LB_CNTL, tmp);
6032 }
6033
6034 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6035 {
6036         u32 i, j, k;
6037         u32 mask;
6038
6039         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6040                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6041                         cik_select_se_sh(rdev, i, j);
6042                         for (k = 0; k < rdev->usec_timeout; k++) {
6043                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6044                                         break;
6045                                 udelay(1);
6046                         }
6047                 }
6048         }
6049         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6050
6051         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6052         for (k = 0; k < rdev->usec_timeout; k++) {
6053                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6054                         break;
6055                 udelay(1);
6056         }
6057 }
6058
6059 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6060 {
6061         u32 tmp;
6062
6063         tmp = RREG32(RLC_CNTL);
6064         if (tmp != rlc)
6065                 WREG32(RLC_CNTL, rlc);
6066 }
6067
6068 static u32 cik_halt_rlc(struct radeon_device *rdev)
6069 {
6070         u32 data, orig;
6071
6072         orig = data = RREG32(RLC_CNTL);
6073
6074         if (data & RLC_ENABLE) {
6075                 u32 i;
6076
6077                 data &= ~RLC_ENABLE;
6078                 WREG32(RLC_CNTL, data);
6079
6080                 for (i = 0; i < rdev->usec_timeout; i++) {
6081                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6082                                 break;
6083                         udelay(1);
6084                 }
6085
6086                 cik_wait_for_rlc_serdes(rdev);
6087         }
6088
6089         return orig;
6090 }
6091
6092 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6093 {
6094         u32 tmp, i, mask;
6095
6096         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6097         WREG32(RLC_GPR_REG2, tmp);
6098
6099         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6100         for (i = 0; i < rdev->usec_timeout; i++) {
6101                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6102                         break;
6103                 udelay(1);
6104         }
6105
6106         for (i = 0; i < rdev->usec_timeout; i++) {
6107                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6108                         break;
6109                 udelay(1);
6110         }
6111 }
6112
6113 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6114 {
6115         u32 tmp;
6116
6117         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6118         WREG32(RLC_GPR_REG2, tmp);
6119 }
6120
6121 /**
6122  * cik_rlc_stop - stop the RLC ME
6123  *
6124  * @rdev: radeon_device pointer
6125  *
6126  * Halt the RLC ME (MicroEngine) (CIK).
6127  */
6128 static void cik_rlc_stop(struct radeon_device *rdev)
6129 {
6130         WREG32(RLC_CNTL, 0);
6131
6132         cik_enable_gui_idle_interrupt(rdev, false);
6133
6134         cik_wait_for_rlc_serdes(rdev);
6135 }
6136
6137 /**
6138  * cik_rlc_start - start the RLC ME
6139  *
6140  * @rdev: radeon_device pointer
6141  *
6142  * Unhalt the RLC ME (MicroEngine) (CIK).
6143  */
6144 static void cik_rlc_start(struct radeon_device *rdev)
6145 {
6146         WREG32(RLC_CNTL, RLC_ENABLE);
6147
6148         cik_enable_gui_idle_interrupt(rdev, true);
6149
6150         udelay(50);
6151 }
6152
6153 /**
6154  * cik_rlc_resume - setup the RLC hw
6155  *
6156  * @rdev: radeon_device pointer
6157  *
6158  * Initialize the RLC registers, load the ucode,
6159  * and start the RLC (CIK).
6160  * Returns 0 for success, -EINVAL if the ucode is not available.
6161  */
6162 static int cik_rlc_resume(struct radeon_device *rdev)
6163 {
6164         u32 i, size, tmp;
6165
6166         if (!rdev->rlc_fw)
6167                 return -EINVAL;
6168
6169         cik_rlc_stop(rdev);
6170
6171         /* disable CG */
6172         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6173         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
6174
6175         si_rlc_reset(rdev);
6176
6177         cik_init_pg(rdev);
6178
6179         cik_init_cg(rdev);
6180
6181         WREG32(RLC_LB_CNTR_INIT, 0);
6182         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
6183
6184         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6185         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6186         WREG32(RLC_LB_PARAMS, 0x00600408);
6187         WREG32(RLC_LB_CNTL, 0x80000004);
6188
6189         WREG32(RLC_MC_CNTL, 0);
6190         WREG32(RLC_UCODE_CNTL, 0);
6191
6192         if (rdev->new_fw) {
6193                 const struct rlc_firmware_header_v1_0 *hdr =
6194                         (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6195                 const __le32 *fw_data = (const __le32 *)
6196                         (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6197
6198                 radeon_ucode_print_rlc_hdr(&hdr->header);
6199
6200                 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6201                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6202                 for (i = 0; i < size; i++)
6203                         WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
6204                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6205         } else {
6206                 const __be32 *fw_data;
6207
6208                 switch (rdev->family) {
6209                 case CHIP_BONAIRE:
6210                 case CHIP_HAWAII:
6211                 default:
6212                         size = BONAIRE_RLC_UCODE_SIZE;
6213                         break;
6214                 case CHIP_KAVERI:
6215                         size = KV_RLC_UCODE_SIZE;
6216                         break;
6217                 case CHIP_KABINI:
6218                         size = KB_RLC_UCODE_SIZE;
6219                         break;
6220                 case CHIP_MULLINS:
6221                         size = ML_RLC_UCODE_SIZE;
6222                         break;
6223                 }
6224
6225                 fw_data = (const __be32 *)rdev->rlc_fw->data;
6226                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6227                 for (i = 0; i < size; i++)
6228                         WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6229                 WREG32(RLC_GPM_UCODE_ADDR, 0);
6230         }
6231
6232         /* XXX - find out what chips support lbpw */
6233         cik_enable_lbpw(rdev, false);
6234
6235         if (rdev->family == CHIP_BONAIRE)
6236                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
6237
6238         cik_rlc_start(rdev);
6239
6240         return 0;
6241 }
6242
6243 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6244 {
6245         u32 data, orig, tmp, tmp2;
6246
6247         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
6248
6249         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
6250                 cik_enable_gui_idle_interrupt(rdev, true);
6251
6252                 tmp = cik_halt_rlc(rdev);
6253
6254                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6255                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6256                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6257                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6258                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
6259
6260                 cik_update_rlc(rdev, tmp);
6261
6262                 data |= CGCG_EN | CGLS_EN;
6263         } else {
6264                 cik_enable_gui_idle_interrupt(rdev, false);
6265
6266                 RREG32(CB_CGTT_SCLK_CTRL);
6267                 RREG32(CB_CGTT_SCLK_CTRL);
6268                 RREG32(CB_CGTT_SCLK_CTRL);
6269                 RREG32(CB_CGTT_SCLK_CTRL);
6270
6271                 data &= ~(CGCG_EN | CGLS_EN);
6272         }
6273
6274         if (orig != data)
6275                 WREG32(RLC_CGCG_CGLS_CTRL, data);
6276
6277 }
6278
6279 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6280 {
6281         u32 data, orig, tmp = 0;
6282
6283         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6284                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6285                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6286                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
6287                                 data |= CP_MEM_LS_EN;
6288                                 if (orig != data)
6289                                         WREG32(CP_MEM_SLP_CNTL, data);
6290                         }
6291                 }
6292
6293                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6294                 data &= 0xfffffffd;
6295                 if (orig != data)
6296                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6297
6298                 tmp = cik_halt_rlc(rdev);
6299
6300                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6301                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6302                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6303                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6304                 WREG32(RLC_SERDES_WR_CTRL, data);
6305
6306                 cik_update_rlc(rdev, tmp);
6307
6308                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6309                         orig = data = RREG32(CGTS_SM_CTRL_REG);
6310                         data &= ~SM_MODE_MASK;
6311                         data |= SM_MODE(0x2);
6312                         data |= SM_MODE_ENABLE;
6313                         data &= ~CGTS_OVERRIDE;
6314                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6315                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6316                                 data &= ~CGTS_LS_OVERRIDE;
6317                         data &= ~ON_MONITOR_ADD_MASK;
6318                         data |= ON_MONITOR_ADD_EN;
6319                         data |= ON_MONITOR_ADD(0x96);
6320                         if (orig != data)
6321                                 WREG32(CGTS_SM_CTRL_REG, data);
6322                 }
6323         } else {
6324                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6325                 data |= 0x00000002;
6326                 if (orig != data)
6327                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6328
6329                 data = RREG32(RLC_MEM_SLP_CNTL);
6330                 if (data & RLC_MEM_LS_EN) {
6331                         data &= ~RLC_MEM_LS_EN;
6332                         WREG32(RLC_MEM_SLP_CNTL, data);
6333                 }
6334
6335                 data = RREG32(CP_MEM_SLP_CNTL);
6336                 if (data & CP_MEM_LS_EN) {
6337                         data &= ~CP_MEM_LS_EN;
6338                         WREG32(CP_MEM_SLP_CNTL, data);
6339                 }
6340
6341                 orig = data = RREG32(CGTS_SM_CTRL_REG);
6342                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6343                 if (orig != data)
6344                         WREG32(CGTS_SM_CTRL_REG, data);
6345
6346                 tmp = cik_halt_rlc(rdev);
6347
6348                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6349                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6350                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6351                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6352                 WREG32(RLC_SERDES_WR_CTRL, data);
6353
6354                 cik_update_rlc(rdev, tmp);
6355         }
6356 }
6357
6358 static const u32 mc_cg_registers[] =
6359 {
6360         MC_HUB_MISC_HUB_CG,
6361         MC_HUB_MISC_SIP_CG,
6362         MC_HUB_MISC_VM_CG,
6363         MC_XPB_CLK_GAT,
6364         ATC_MISC_CG,
6365         MC_CITF_MISC_WR_CG,
6366         MC_CITF_MISC_RD_CG,
6367         MC_CITF_MISC_VM_CG,
6368         VM_L2_CG,
6369 };
6370
6371 static void cik_enable_mc_ls(struct radeon_device *rdev,
6372                              bool enable)
6373 {
6374         int i;
6375         u32 orig, data;
6376
6377         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6378                 orig = data = RREG32(mc_cg_registers[i]);
6379                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6380                         data |= MC_LS_ENABLE;
6381                 else
6382                         data &= ~MC_LS_ENABLE;
6383                 if (data != orig)
6384                         WREG32(mc_cg_registers[i], data);
6385         }
6386 }
6387
6388 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6389                                bool enable)
6390 {
6391         int i;
6392         u32 orig, data;
6393
6394         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6395                 orig = data = RREG32(mc_cg_registers[i]);
6396                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6397                         data |= MC_CG_ENABLE;
6398                 else
6399                         data &= ~MC_CG_ENABLE;
6400                 if (data != orig)
6401                         WREG32(mc_cg_registers[i], data);
6402         }
6403 }
6404
6405 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6406                                  bool enable)
6407 {
6408         u32 orig, data;
6409
6410         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6411                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6412                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6413         } else {
6414                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6415                 data |= 0xff000000;
6416                 if (data != orig)
6417                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6418
6419                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6420                 data |= 0xff000000;
6421                 if (data != orig)
6422                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6423         }
6424 }
6425
6426 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6427                                  bool enable)
6428 {
6429         u32 orig, data;
6430
6431         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6432                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6433                 data |= 0x100;
6434                 if (orig != data)
6435                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6436
6437                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6438                 data |= 0x100;
6439                 if (orig != data)
6440                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6441         } else {
6442                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6443                 data &= ~0x100;
6444                 if (orig != data)
6445                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6446
6447                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6448                 data &= ~0x100;
6449                 if (orig != data)
6450                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6451         }
6452 }
6453
6454 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6455                                 bool enable)
6456 {
6457         u32 orig, data;
6458
6459         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6460                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6461                 data = 0xfff;
6462                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6463
6464                 orig = data = RREG32(UVD_CGC_CTRL);
6465                 data |= DCM;
6466                 if (orig != data)
6467                         WREG32(UVD_CGC_CTRL, data);
6468         } else {
6469                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6470                 data &= ~0xfff;
6471                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6472
6473                 orig = data = RREG32(UVD_CGC_CTRL);
6474                 data &= ~DCM;
6475                 if (orig != data)
6476                         WREG32(UVD_CGC_CTRL, data);
6477         }
6478 }
6479
6480 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6481                                bool enable)
6482 {
6483         u32 orig, data;
6484
6485         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6486
6487         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6488                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6489                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6490         else
6491                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6492                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6493
6494         if (orig != data)
6495                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6496 }
6497
6498 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6499                                 bool enable)
6500 {
6501         u32 orig, data;
6502
6503         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6504
6505         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6506                 data &= ~CLOCK_GATING_DIS;
6507         else
6508                 data |= CLOCK_GATING_DIS;
6509
6510         if (orig != data)
6511                 WREG32(HDP_HOST_PATH_CNTL, data);
6512 }
6513
6514 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6515                               bool enable)
6516 {
6517         u32 orig, data;
6518
6519         orig = data = RREG32(HDP_MEM_POWER_LS);
6520
6521         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6522                 data |= HDP_LS_ENABLE;
6523         else
6524                 data &= ~HDP_LS_ENABLE;
6525
6526         if (orig != data)
6527                 WREG32(HDP_MEM_POWER_LS, data);
6528 }
6529
6530 void cik_update_cg(struct radeon_device *rdev,
6531                    u32 block, bool enable)
6532 {
6533
6534         if (block & RADEON_CG_BLOCK_GFX) {
6535                 cik_enable_gui_idle_interrupt(rdev, false);
6536                 /* order matters! */
6537                 if (enable) {
6538                         cik_enable_mgcg(rdev, true);
6539                         cik_enable_cgcg(rdev, true);
6540                 } else {
6541                         cik_enable_cgcg(rdev, false);
6542                         cik_enable_mgcg(rdev, false);
6543                 }
6544                 cik_enable_gui_idle_interrupt(rdev, true);
6545         }
6546
6547         if (block & RADEON_CG_BLOCK_MC) {
6548                 if (!(rdev->flags & RADEON_IS_IGP)) {
6549                         cik_enable_mc_mgcg(rdev, enable);
6550                         cik_enable_mc_ls(rdev, enable);
6551                 }
6552         }
6553
6554         if (block & RADEON_CG_BLOCK_SDMA) {
6555                 cik_enable_sdma_mgcg(rdev, enable);
6556                 cik_enable_sdma_mgls(rdev, enable);
6557         }
6558
6559         if (block & RADEON_CG_BLOCK_BIF) {
6560                 cik_enable_bif_mgls(rdev, enable);
6561         }
6562
6563         if (block & RADEON_CG_BLOCK_UVD) {
6564                 if (rdev->has_uvd)
6565                         cik_enable_uvd_mgcg(rdev, enable);
6566         }
6567
6568         if (block & RADEON_CG_BLOCK_HDP) {
6569                 cik_enable_hdp_mgcg(rdev, enable);
6570                 cik_enable_hdp_ls(rdev, enable);
6571         }
6572
6573         if (block & RADEON_CG_BLOCK_VCE) {
6574                 vce_v2_0_enable_mgcg(rdev, enable);
6575         }
6576 }
6577
6578 static void cik_init_cg(struct radeon_device *rdev)
6579 {
6580
6581         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6582
6583         if (rdev->has_uvd)
6584                 si_init_uvd_internal_cg(rdev);
6585
6586         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6587                              RADEON_CG_BLOCK_SDMA |
6588                              RADEON_CG_BLOCK_BIF |
6589                              RADEON_CG_BLOCK_UVD |
6590                              RADEON_CG_BLOCK_HDP), true);
6591 }
6592
6593 static void cik_fini_cg(struct radeon_device *rdev)
6594 {
6595         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6596                              RADEON_CG_BLOCK_SDMA |
6597                              RADEON_CG_BLOCK_BIF |
6598                              RADEON_CG_BLOCK_UVD |
6599                              RADEON_CG_BLOCK_HDP), false);
6600
6601         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6602 }
6603
6604 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6605                                           bool enable)
6606 {
6607         u32 data, orig;
6608
6609         orig = data = RREG32(RLC_PG_CNTL);
6610         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6611                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6612         else
6613                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6614         if (orig != data)
6615                 WREG32(RLC_PG_CNTL, data);
6616 }
6617
6618 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6619                                           bool enable)
6620 {
6621         u32 data, orig;
6622
6623         orig = data = RREG32(RLC_PG_CNTL);
6624         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6625                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6626         else
6627                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6628         if (orig != data)
6629                 WREG32(RLC_PG_CNTL, data);
6630 }
6631
6632 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6633 {
6634         u32 data, orig;
6635
6636         orig = data = RREG32(RLC_PG_CNTL);
6637         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6638                 data &= ~DISABLE_CP_PG;
6639         else
6640                 data |= DISABLE_CP_PG;
6641         if (orig != data)
6642                 WREG32(RLC_PG_CNTL, data);
6643 }
6644
6645 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6646 {
6647         u32 data, orig;
6648
6649         orig = data = RREG32(RLC_PG_CNTL);
6650         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6651                 data &= ~DISABLE_GDS_PG;
6652         else
6653                 data |= DISABLE_GDS_PG;
6654         if (orig != data)
6655                 WREG32(RLC_PG_CNTL, data);
6656 }
6657
6658 #define CP_ME_TABLE_SIZE    96
6659 #define CP_ME_TABLE_OFFSET  2048
6660 #define CP_MEC_TABLE_OFFSET 4096
6661
6662 void cik_init_cp_pg_table(struct radeon_device *rdev)
6663 {
6664         volatile u32 *dst_ptr;
6665         int me, i, max_me = 4;
6666         u32 bo_offset = 0;
6667         u32 table_offset, table_size;
6668
6669         if (rdev->family == CHIP_KAVERI)
6670                 max_me = 5;
6671
6672         if (rdev->rlc.cp_table_ptr == NULL)
6673                 return;
6674
6675         /* write the cp table buffer */
6676         dst_ptr = rdev->rlc.cp_table_ptr;
6677         for (me = 0; me < max_me; me++) {
6678                 if (rdev->new_fw) {
6679                         const __le32 *fw_data;
6680                         const struct gfx_firmware_header_v1_0 *hdr;
6681
6682                         if (me == 0) {
6683                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6684                                 fw_data = (const __le32 *)
6685                                         (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6686                                 table_offset = le32_to_cpu(hdr->jt_offset);
6687                                 table_size = le32_to_cpu(hdr->jt_size);
6688                         } else if (me == 1) {
6689                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6690                                 fw_data = (const __le32 *)
6691                                         (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6692                                 table_offset = le32_to_cpu(hdr->jt_offset);
6693                                 table_size = le32_to_cpu(hdr->jt_size);
6694                         } else if (me == 2) {
6695                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6696                                 fw_data = (const __le32 *)
6697                                         (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6698                                 table_offset = le32_to_cpu(hdr->jt_offset);
6699                                 table_size = le32_to_cpu(hdr->jt_size);
6700                         } else if (me == 3) {
6701                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6702                                 fw_data = (const __le32 *)
6703                                         (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6704                                 table_offset = le32_to_cpu(hdr->jt_offset);
6705                                 table_size = le32_to_cpu(hdr->jt_size);
6706                         } else {
6707                                 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6708                                 fw_data = (const __le32 *)
6709                                         (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6710                                 table_offset = le32_to_cpu(hdr->jt_offset);
6711                                 table_size = le32_to_cpu(hdr->jt_size);
6712                         }
6713
6714                         for (i = 0; i < table_size; i ++) {
6715                                 dst_ptr[bo_offset + i] =
6716                                         cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6717                         }
6718                         bo_offset += table_size;
6719                 } else {
6720                         const __be32 *fw_data;
6721                         table_size = CP_ME_TABLE_SIZE;
6722
6723                         if (me == 0) {
6724                                 fw_data = (const __be32 *)rdev->ce_fw->data;
6725                                 table_offset = CP_ME_TABLE_OFFSET;
6726                         } else if (me == 1) {
6727                                 fw_data = (const __be32 *)rdev->pfp_fw->data;
6728                                 table_offset = CP_ME_TABLE_OFFSET;
6729                         } else if (me == 2) {
6730                                 fw_data = (const __be32 *)rdev->me_fw->data;
6731                                 table_offset = CP_ME_TABLE_OFFSET;
6732                         } else {
6733                                 fw_data = (const __be32 *)rdev->mec_fw->data;
6734                                 table_offset = CP_MEC_TABLE_OFFSET;
6735                         }
6736
6737                         for (i = 0; i < table_size; i ++) {
6738                                 dst_ptr[bo_offset + i] =
6739                                         cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6740                         }
6741                         bo_offset += table_size;
6742                 }
6743         }
6744 }
6745
6746 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6747                                 bool enable)
6748 {
6749         u32 data, orig;
6750
6751         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6752                 orig = data = RREG32(RLC_PG_CNTL);
6753                 data |= GFX_PG_ENABLE;
6754                 if (orig != data)
6755                         WREG32(RLC_PG_CNTL, data);
6756
6757                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6758                 data |= AUTO_PG_EN;
6759                 if (orig != data)
6760                         WREG32(RLC_AUTO_PG_CTRL, data);
6761         } else {
6762                 orig = data = RREG32(RLC_PG_CNTL);
6763                 data &= ~GFX_PG_ENABLE;
6764                 if (orig != data)
6765                         WREG32(RLC_PG_CNTL, data);
6766
6767                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6768                 data &= ~AUTO_PG_EN;
6769                 if (orig != data)
6770                         WREG32(RLC_AUTO_PG_CTRL, data);
6771
6772                 data = RREG32(DB_RENDER_CONTROL);
6773         }
6774 }
6775
6776 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6777 {
6778         u32 mask = 0, tmp, tmp1;
6779         int i;
6780
6781         cik_select_se_sh(rdev, se, sh);
6782         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6783         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6784         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6785
6786         tmp &= 0xffff0000;
6787
6788         tmp |= tmp1;
6789         tmp >>= 16;
6790
6791         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6792                 mask <<= 1;
6793                 mask |= 1;
6794         }
6795
6796         return (~tmp) & mask;
6797 }
6798
6799 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6800 {
6801         u32 i, j, k, active_cu_number = 0;
6802         u32 mask, counter, cu_bitmap;
6803         u32 tmp = 0;
6804
6805         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6806                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6807                         mask = 1;
6808                         cu_bitmap = 0;
6809                         counter = 0;
6810                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6811                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6812                                         if (counter < 2)
6813                                                 cu_bitmap |= mask;
6814                                         counter ++;
6815                                 }
6816                                 mask <<= 1;
6817                         }
6818
6819                         active_cu_number += counter;
6820                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6821                 }
6822         }
6823
6824         WREG32(RLC_PG_AO_CU_MASK, tmp);
6825
6826         tmp = RREG32(RLC_MAX_PG_CU);
6827         tmp &= ~MAX_PU_CU_MASK;
6828         tmp |= MAX_PU_CU(active_cu_number);
6829         WREG32(RLC_MAX_PG_CU, tmp);
6830 }
6831
6832 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6833                                        bool enable)
6834 {
6835         u32 data, orig;
6836
6837         orig = data = RREG32(RLC_PG_CNTL);
6838         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6839                 data |= STATIC_PER_CU_PG_ENABLE;
6840         else
6841                 data &= ~STATIC_PER_CU_PG_ENABLE;
6842         if (orig != data)
6843                 WREG32(RLC_PG_CNTL, data);
6844 }
6845
6846 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6847                                         bool enable)
6848 {
6849         u32 data, orig;
6850
6851         orig = data = RREG32(RLC_PG_CNTL);
6852         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6853                 data |= DYN_PER_CU_PG_ENABLE;
6854         else
6855                 data &= ~DYN_PER_CU_PG_ENABLE;
6856         if (orig != data)
6857                 WREG32(RLC_PG_CNTL, data);
6858 }
6859
6860 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6861 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6862
6863 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6864 {
6865         u32 data, orig;
6866         u32 i;
6867
6868         if (rdev->rlc.cs_data) {
6869                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6870                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6871                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6872                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6873         } else {
6874                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6875                 for (i = 0; i < 3; i++)
6876                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6877         }
6878         if (rdev->rlc.reg_list) {
6879                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6880                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6881                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6882         }
6883
6884         orig = data = RREG32(RLC_PG_CNTL);
6885         data |= GFX_PG_SRC;
6886         if (orig != data)
6887                 WREG32(RLC_PG_CNTL, data);
6888
6889         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6890         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6891
6892         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6893         data &= ~IDLE_POLL_COUNT_MASK;
6894         data |= IDLE_POLL_COUNT(0x60);
6895         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6896
6897         data = 0x10101010;
6898         WREG32(RLC_PG_DELAY, data);
6899
6900         data = RREG32(RLC_PG_DELAY_2);
6901         data &= ~0xff;
6902         data |= 0x3;
6903         WREG32(RLC_PG_DELAY_2, data);
6904
6905         data = RREG32(RLC_AUTO_PG_CTRL);
6906         data &= ~GRBM_REG_SGIT_MASK;
6907         data |= GRBM_REG_SGIT(0x700);
6908         WREG32(RLC_AUTO_PG_CTRL, data);
6909
6910 }
6911
6912 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6913 {
6914         cik_enable_gfx_cgpg(rdev, enable);
6915         cik_enable_gfx_static_mgpg(rdev, enable);
6916         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6917 }
6918
6919 u32 cik_get_csb_size(struct radeon_device *rdev)
6920 {
6921         u32 count = 0;
6922         const struct cs_section_def *sect = NULL;
6923         const struct cs_extent_def *ext = NULL;
6924
6925         if (rdev->rlc.cs_data == NULL)
6926                 return 0;
6927
6928         /* begin clear state */
6929         count += 2;
6930         /* context control state */
6931         count += 3;
6932
6933         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6934                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6935                         if (sect->id == SECT_CONTEXT)
6936                                 count += 2 + ext->reg_count;
6937                         else
6938                                 return 0;
6939                 }
6940         }
6941         /* pa_sc_raster_config/pa_sc_raster_config1 */
6942         count += 4;
6943         /* end clear state */
6944         count += 2;
6945         /* clear state */
6946         count += 2;
6947
6948         return count;
6949 }
6950
6951 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6952 {
6953         u32 count = 0, i;
6954         const struct cs_section_def *sect = NULL;
6955         const struct cs_extent_def *ext = NULL;
6956
6957         if (rdev->rlc.cs_data == NULL)
6958                 return;
6959         if (buffer == NULL)
6960                 return;
6961
6962         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6963         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6964
6965         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6966         buffer[count++] = cpu_to_le32(0x80000000);
6967         buffer[count++] = cpu_to_le32(0x80000000);
6968
6969         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6970                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6971                         if (sect->id == SECT_CONTEXT) {
6972                                 buffer[count++] =
6973                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6974                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6975                                 for (i = 0; i < ext->reg_count; i++)
6976                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6977                         } else {
6978                                 return;
6979                         }
6980                 }
6981         }
6982
6983         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6984         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6985         switch (rdev->family) {
6986         case CHIP_BONAIRE:
6987                 buffer[count++] = cpu_to_le32(0x16000012);
6988                 buffer[count++] = cpu_to_le32(0x00000000);
6989                 break;
6990         case CHIP_KAVERI:
6991                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6992                 buffer[count++] = cpu_to_le32(0x00000000);
6993                 break;
6994         case CHIP_KABINI:
6995         case CHIP_MULLINS:
6996                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6997                 buffer[count++] = cpu_to_le32(0x00000000);
6998                 break;
6999         case CHIP_HAWAII:
7000                 buffer[count++] = cpu_to_le32(0x3a00161a);
7001                 buffer[count++] = cpu_to_le32(0x0000002e);
7002                 break;
7003         default:
7004                 buffer[count++] = cpu_to_le32(0x00000000);
7005                 buffer[count++] = cpu_to_le32(0x00000000);
7006                 break;
7007         }
7008
7009         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7010         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
7011
7012         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7013         buffer[count++] = cpu_to_le32(0);
7014 }
7015
7016 static void cik_init_pg(struct radeon_device *rdev)
7017 {
7018         if (rdev->pg_flags) {
7019                 cik_enable_sck_slowdown_on_pu(rdev, true);
7020                 cik_enable_sck_slowdown_on_pd(rdev, true);
7021                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7022                         cik_init_gfx_cgpg(rdev);
7023                         cik_enable_cp_pg(rdev, true);
7024                         cik_enable_gds_pg(rdev, true);
7025                 }
7026                 cik_init_ao_cu_mask(rdev);
7027                 cik_update_gfx_pg(rdev, true);
7028         }
7029 }
7030
7031 static void cik_fini_pg(struct radeon_device *rdev)
7032 {
7033         if (rdev->pg_flags) {
7034                 cik_update_gfx_pg(rdev, false);
7035                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
7036                         cik_enable_cp_pg(rdev, false);
7037                         cik_enable_gds_pg(rdev, false);
7038                 }
7039         }
7040 }
7041
7042 /*
7043  * Interrupts
7044  * Starting with r6xx, interrupts are handled via a ring buffer.
7045  * Ring buffers are areas of GPU accessible memory that the GPU
7046  * writes interrupt vectors into and the host reads vectors out of.
7047  * There is a rptr (read pointer) that determines where the
7048  * host is currently reading, and a wptr (write pointer)
7049  * which determines where the GPU has written.  When the
7050  * pointers are equal, the ring is idle.  When the GPU
7051  * writes vectors to the ring buffer, it increments the
7052  * wptr.  When there is an interrupt, the host then starts
7053  * fetching commands and processing them until the pointers are
7054  * equal again at which point it updates the rptr.
7055  */
7056
7057 /**
7058  * cik_enable_interrupts - Enable the interrupt ring buffer
7059  *
7060  * @rdev: radeon_device pointer
7061  *
7062  * Enable the interrupt ring buffer (CIK).
7063  */
7064 static void cik_enable_interrupts(struct radeon_device *rdev)
7065 {
7066         u32 ih_cntl = RREG32(IH_CNTL);
7067         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7068
7069         ih_cntl |= ENABLE_INTR;
7070         ih_rb_cntl |= IH_RB_ENABLE;
7071         WREG32(IH_CNTL, ih_cntl);
7072         WREG32(IH_RB_CNTL, ih_rb_cntl);
7073         rdev->ih.enabled = true;
7074 }
7075
7076 /**
7077  * cik_disable_interrupts - Disable the interrupt ring buffer
7078  *
7079  * @rdev: radeon_device pointer
7080  *
7081  * Disable the interrupt ring buffer (CIK).
7082  */
7083 static void cik_disable_interrupts(struct radeon_device *rdev)
7084 {
7085         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7086         u32 ih_cntl = RREG32(IH_CNTL);
7087
7088         ih_rb_cntl &= ~IH_RB_ENABLE;
7089         ih_cntl &= ~ENABLE_INTR;
7090         WREG32(IH_RB_CNTL, ih_rb_cntl);
7091         WREG32(IH_CNTL, ih_cntl);
7092         /* set rptr, wptr to 0 */
7093         WREG32(IH_RB_RPTR, 0);
7094         WREG32(IH_RB_WPTR, 0);
7095         rdev->ih.enabled = false;
7096         rdev->ih.rptr = 0;
7097 }
7098
7099 /**
7100  * cik_disable_interrupt_state - Disable all interrupt sources
7101  *
7102  * @rdev: radeon_device pointer
7103  *
7104  * Clear all interrupt enable bits used by the driver (CIK).
7105  */
7106 static void cik_disable_interrupt_state(struct radeon_device *rdev)
7107 {
7108         u32 tmp;
7109
7110         /* gfx ring */
7111         tmp = RREG32(CP_INT_CNTL_RING0) &
7112                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7113         WREG32(CP_INT_CNTL_RING0, tmp);
7114         /* sdma */
7115         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7116         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7117         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7118         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
7119         /* compute queues */
7120         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7121         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7122         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7123         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7124         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7125         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7126         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7127         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7128         /* grbm */
7129         WREG32(GRBM_INT_CNTL, 0);
7130         /* vline/vblank, etc. */
7131         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7132         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7133         if (rdev->num_crtc >= 4) {
7134                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7135                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7136         }
7137         if (rdev->num_crtc >= 6) {
7138                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7139                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7140         }
7141         /* pflip */
7142         if (rdev->num_crtc >= 2) {
7143                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7144                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7145         }
7146         if (rdev->num_crtc >= 4) {
7147                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7148                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7149         }
7150         if (rdev->num_crtc >= 6) {
7151                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7152                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7153         }
7154
7155         /* dac hotplug */
7156         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7157
7158         /* digital hotplug */
7159         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7160         WREG32(DC_HPD1_INT_CONTROL, tmp);
7161         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7162         WREG32(DC_HPD2_INT_CONTROL, tmp);
7163         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7164         WREG32(DC_HPD3_INT_CONTROL, tmp);
7165         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7166         WREG32(DC_HPD4_INT_CONTROL, tmp);
7167         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7168         WREG32(DC_HPD5_INT_CONTROL, tmp);
7169         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7170         WREG32(DC_HPD6_INT_CONTROL, tmp);
7171
7172 }
7173
7174 /**
7175  * cik_irq_init - init and enable the interrupt ring
7176  *
7177  * @rdev: radeon_device pointer
7178  *
7179  * Allocate a ring buffer for the interrupt controller,
7180  * enable the RLC, disable interrupts, enable the IH
7181  * ring buffer and enable it (CIK).
7182  * Called at device load and reume.
7183  * Returns 0 for success, errors for failure.
7184  */
7185 static int cik_irq_init(struct radeon_device *rdev)
7186 {
7187         int ret = 0;
7188         int rb_bufsz;
7189         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7190
7191         /* allocate ring */
7192         ret = r600_ih_ring_alloc(rdev);
7193         if (ret)
7194                 return ret;
7195
7196         /* disable irqs */
7197         cik_disable_interrupts(rdev);
7198
7199         /* init rlc */
7200         ret = cik_rlc_resume(rdev);
7201         if (ret) {
7202                 r600_ih_ring_fini(rdev);
7203                 return ret;
7204         }
7205
7206         /* setup interrupt control */
7207         /* XXX this should actually be a bus address, not an MC address. same on older asics */
7208         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7209         interrupt_cntl = RREG32(INTERRUPT_CNTL);
7210         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7211          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7212          */
7213         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7214         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7215         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7216         WREG32(INTERRUPT_CNTL, interrupt_cntl);
7217
7218         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
7219         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
7220
7221         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7222                       IH_WPTR_OVERFLOW_CLEAR |
7223                       (rb_bufsz << 1));
7224
7225         if (rdev->wb.enabled)
7226                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7227
7228         /* set the writeback address whether it's enabled or not */
7229         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7230         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7231
7232         WREG32(IH_RB_CNTL, ih_rb_cntl);
7233
7234         /* set rptr, wptr to 0 */
7235         WREG32(IH_RB_RPTR, 0);
7236         WREG32(IH_RB_WPTR, 0);
7237
7238         /* Default settings for IH_CNTL (disabled at first) */
7239         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7240         /* RPTR_REARM only works if msi's are enabled */
7241         if (rdev->msi_enabled)
7242                 ih_cntl |= RPTR_REARM;
7243         WREG32(IH_CNTL, ih_cntl);
7244
7245         /* force the active interrupt state to all disabled */
7246         cik_disable_interrupt_state(rdev);
7247
7248         pci_set_master(rdev->pdev);
7249
7250         /* enable irqs */
7251         cik_enable_interrupts(rdev);
7252
7253         return ret;
7254 }
7255
7256 /**
7257  * cik_irq_set - enable/disable interrupt sources
7258  *
7259  * @rdev: radeon_device pointer
7260  *
7261  * Enable interrupt sources on the GPU (vblanks, hpd,
7262  * etc.) (CIK).
7263  * Returns 0 for success, errors for failure.
7264  */
7265 int cik_irq_set(struct radeon_device *rdev)
7266 {
7267         u32 cp_int_cntl;
7268         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7269         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
7270         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7271         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7272         u32 grbm_int_cntl = 0;
7273         u32 dma_cntl, dma_cntl1;
7274         u32 thermal_int;
7275
7276         if (!rdev->irq.installed) {
7277                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7278                 return -EINVAL;
7279         }
7280         /* don't enable anything if the ih is disabled */
7281         if (!rdev->ih.enabled) {
7282                 cik_disable_interrupts(rdev);
7283                 /* force the active interrupt state to all disabled */
7284                 cik_disable_interrupt_state(rdev);
7285                 return 0;
7286         }
7287
7288         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7289                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7290         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7291
7292         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7293         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7294         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7295         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7296         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7297         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7298
7299         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7300         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7301
7302         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7303         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7304         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7305         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7306         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7307         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7308         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7309         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7310
7311         if (rdev->flags & RADEON_IS_IGP)
7312                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7313                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
7314         else
7315                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7316                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
7317
7318         /* enable CP interrupts on all rings */
7319         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7320                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7321                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7322         }
7323         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7324                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7325                 DRM_DEBUG("si_irq_set: sw int cp1\n");
7326                 if (ring->me == 1) {
7327                         switch (ring->pipe) {
7328                         case 0:
7329                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7330                                 break;
7331                         case 1:
7332                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7333                                 break;
7334                         case 2:
7335                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7336                                 break;
7337                         case 3:
7338                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7339                                 break;
7340                         default:
7341                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7342                                 break;
7343                         }
7344                 } else if (ring->me == 2) {
7345                         switch (ring->pipe) {
7346                         case 0:
7347                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7348                                 break;
7349                         case 1:
7350                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7351                                 break;
7352                         case 2:
7353                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7354                                 break;
7355                         case 3:
7356                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7357                                 break;
7358                         default:
7359                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7360                                 break;
7361                         }
7362                 } else {
7363                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7364                 }
7365         }
7366         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7367                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7368                 DRM_DEBUG("si_irq_set: sw int cp2\n");
7369                 if (ring->me == 1) {
7370                         switch (ring->pipe) {
7371                         case 0:
7372                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7373                                 break;
7374                         case 1:
7375                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7376                                 break;
7377                         case 2:
7378                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7379                                 break;
7380                         case 3:
7381                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7382                                 break;
7383                         default:
7384                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7385                                 break;
7386                         }
7387                 } else if (ring->me == 2) {
7388                         switch (ring->pipe) {
7389                         case 0:
7390                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7391                                 break;
7392                         case 1:
7393                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7394                                 break;
7395                         case 2:
7396                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7397                                 break;
7398                         case 3:
7399                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7400                                 break;
7401                         default:
7402                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7403                                 break;
7404                         }
7405                 } else {
7406                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7407                 }
7408         }
7409
7410         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7411                 DRM_DEBUG("cik_irq_set: sw int dma\n");
7412                 dma_cntl |= TRAP_ENABLE;
7413         }
7414
7415         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7416                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7417                 dma_cntl1 |= TRAP_ENABLE;
7418         }
7419
7420         if (rdev->irq.crtc_vblank_int[0] ||
7421             atomic_read(&rdev->irq.pflip[0])) {
7422                 DRM_DEBUG("cik_irq_set: vblank 0\n");
7423                 crtc1 |= VBLANK_INTERRUPT_MASK;
7424         }
7425         if (rdev->irq.crtc_vblank_int[1] ||
7426             atomic_read(&rdev->irq.pflip[1])) {
7427                 DRM_DEBUG("cik_irq_set: vblank 1\n");
7428                 crtc2 |= VBLANK_INTERRUPT_MASK;
7429         }
7430         if (rdev->irq.crtc_vblank_int[2] ||
7431             atomic_read(&rdev->irq.pflip[2])) {
7432                 DRM_DEBUG("cik_irq_set: vblank 2\n");
7433                 crtc3 |= VBLANK_INTERRUPT_MASK;
7434         }
7435         if (rdev->irq.crtc_vblank_int[3] ||
7436             atomic_read(&rdev->irq.pflip[3])) {
7437                 DRM_DEBUG("cik_irq_set: vblank 3\n");
7438                 crtc4 |= VBLANK_INTERRUPT_MASK;
7439         }
7440         if (rdev->irq.crtc_vblank_int[4] ||
7441             atomic_read(&rdev->irq.pflip[4])) {
7442                 DRM_DEBUG("cik_irq_set: vblank 4\n");
7443                 crtc5 |= VBLANK_INTERRUPT_MASK;
7444         }
7445         if (rdev->irq.crtc_vblank_int[5] ||
7446             atomic_read(&rdev->irq.pflip[5])) {
7447                 DRM_DEBUG("cik_irq_set: vblank 5\n");
7448                 crtc6 |= VBLANK_INTERRUPT_MASK;
7449         }
7450         if (rdev->irq.hpd[0]) {
7451                 DRM_DEBUG("cik_irq_set: hpd 1\n");
7452                 hpd1 |= DC_HPDx_INT_EN;
7453         }
7454         if (rdev->irq.hpd[1]) {
7455                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7456                 hpd2 |= DC_HPDx_INT_EN;
7457         }
7458         if (rdev->irq.hpd[2]) {
7459                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7460                 hpd3 |= DC_HPDx_INT_EN;
7461         }
7462         if (rdev->irq.hpd[3]) {
7463                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7464                 hpd4 |= DC_HPDx_INT_EN;
7465         }
7466         if (rdev->irq.hpd[4]) {
7467                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7468                 hpd5 |= DC_HPDx_INT_EN;
7469         }
7470         if (rdev->irq.hpd[5]) {
7471                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7472                 hpd6 |= DC_HPDx_INT_EN;
7473         }
7474
7475         if (rdev->irq.dpm_thermal) {
7476                 DRM_DEBUG("dpm thermal\n");
7477                 if (rdev->flags & RADEON_IS_IGP)
7478                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7479                 else
7480                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7481         }
7482
7483         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7484
7485         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7486         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7487
7488         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7489         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7490         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7491         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7492         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7493         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7494         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7495         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7496
7497         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7498
7499         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7500         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7501         if (rdev->num_crtc >= 4) {
7502                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7503                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7504         }
7505         if (rdev->num_crtc >= 6) {
7506                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7507                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7508         }
7509
7510         if (rdev->num_crtc >= 2) {
7511                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7512                        GRPH_PFLIP_INT_MASK);
7513                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7514                        GRPH_PFLIP_INT_MASK);
7515         }
7516         if (rdev->num_crtc >= 4) {
7517                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7518                        GRPH_PFLIP_INT_MASK);
7519                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7520                        GRPH_PFLIP_INT_MASK);
7521         }
7522         if (rdev->num_crtc >= 6) {
7523                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7524                        GRPH_PFLIP_INT_MASK);
7525                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7526                        GRPH_PFLIP_INT_MASK);
7527         }
7528
7529         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7530         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7531         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7532         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7533         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7534         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7535
7536         if (rdev->flags & RADEON_IS_IGP)
7537                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7538         else
7539                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7540
7541         return 0;
7542 }
7543
7544 /**
7545  * cik_irq_ack - ack interrupt sources
7546  *
7547  * @rdev: radeon_device pointer
7548  *
7549  * Ack interrupt sources on the GPU (vblanks, hpd,
7550  * etc.) (CIK).  Certain interrupts sources are sw
7551  * generated and do not require an explicit ack.
7552  */
7553 static inline void cik_irq_ack(struct radeon_device *rdev)
7554 {
7555         u32 tmp;
7556
7557         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7558         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7559         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7560         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7561         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7562         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7563         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7564
7565         rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7566                 EVERGREEN_CRTC0_REGISTER_OFFSET);
7567         rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7568                 EVERGREEN_CRTC1_REGISTER_OFFSET);
7569         if (rdev->num_crtc >= 4) {
7570                 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7571                         EVERGREEN_CRTC2_REGISTER_OFFSET);
7572                 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7573                         EVERGREEN_CRTC3_REGISTER_OFFSET);
7574         }
7575         if (rdev->num_crtc >= 6) {
7576                 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7577                         EVERGREEN_CRTC4_REGISTER_OFFSET);
7578                 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7579                         EVERGREEN_CRTC5_REGISTER_OFFSET);
7580         }
7581
7582         if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7583                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7584                        GRPH_PFLIP_INT_CLEAR);
7585         if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7586                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7587                        GRPH_PFLIP_INT_CLEAR);
7588         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7589                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7590         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7591                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7592         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7593                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7594         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7595                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7596
7597         if (rdev->num_crtc >= 4) {
7598                 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7599                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7600                                GRPH_PFLIP_INT_CLEAR);
7601                 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7602                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7603                                GRPH_PFLIP_INT_CLEAR);
7604                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7605                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7606                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7607                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7608                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7609                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7610                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7611                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7612         }
7613
7614         if (rdev->num_crtc >= 6) {
7615                 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7616                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7617                                GRPH_PFLIP_INT_CLEAR);
7618                 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7619                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7620                                GRPH_PFLIP_INT_CLEAR);
7621                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7622                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7623                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7624                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7625                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7626                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7627                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7628                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7629         }
7630
7631         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7632                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7633                 tmp |= DC_HPDx_INT_ACK;
7634                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7635         }
7636         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7637                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7638                 tmp |= DC_HPDx_INT_ACK;
7639                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7640         }
7641         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7642                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7643                 tmp |= DC_HPDx_INT_ACK;
7644                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7645         }
7646         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7647                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7648                 tmp |= DC_HPDx_INT_ACK;
7649                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7650         }
7651         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7652                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7653                 tmp |= DC_HPDx_INT_ACK;
7654                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7655         }
7656         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7657                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7658                 tmp |= DC_HPDx_INT_ACK;
7659                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7660         }
7661 }
7662
7663 /**
7664  * cik_irq_disable - disable interrupts
7665  *
7666  * @rdev: radeon_device pointer
7667  *
7668  * Disable interrupts on the hw (CIK).
7669  */
7670 static void cik_irq_disable(struct radeon_device *rdev)
7671 {
7672         cik_disable_interrupts(rdev);
7673         /* Wait and acknowledge irq */
7674         mdelay(1);
7675         cik_irq_ack(rdev);
7676         cik_disable_interrupt_state(rdev);
7677 }
7678
7679 /**
7680  * cik_irq_disable - disable interrupts for suspend
7681  *
7682  * @rdev: radeon_device pointer
7683  *
7684  * Disable interrupts and stop the RLC (CIK).
7685  * Used for suspend.
7686  */
7687 static void cik_irq_suspend(struct radeon_device *rdev)
7688 {
7689         cik_irq_disable(rdev);
7690         cik_rlc_stop(rdev);
7691 }
7692
7693 /**
7694  * cik_irq_fini - tear down interrupt support
7695  *
7696  * @rdev: radeon_device pointer
7697  *
7698  * Disable interrupts on the hw and free the IH ring
7699  * buffer (CIK).
7700  * Used for driver unload.
7701  */
7702 static void cik_irq_fini(struct radeon_device *rdev)
7703 {
7704         cik_irq_suspend(rdev);
7705         r600_ih_ring_fini(rdev);
7706 }
7707
7708 /**
7709  * cik_get_ih_wptr - get the IH ring buffer wptr
7710  *
7711  * @rdev: radeon_device pointer
7712  *
7713  * Get the IH ring buffer wptr from either the register
7714  * or the writeback memory buffer (CIK).  Also check for
7715  * ring buffer overflow and deal with it.
7716  * Used by cik_irq_process().
7717  * Returns the value of the wptr.
7718  */
7719 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7720 {
7721         u32 wptr, tmp;
7722
7723         if (rdev->wb.enabled)
7724                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7725         else
7726                 wptr = RREG32(IH_RB_WPTR);
7727
7728         if (wptr & RB_OVERFLOW) {
7729                 /* When a ring buffer overflow happen start parsing interrupt
7730                  * from the last not overwritten vector (wptr + 16). Hopefully
7731                  * this should allow us to catchup.
7732                  */
7733                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7734                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7735                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7736                 tmp = RREG32(IH_RB_CNTL);
7737                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7738                 WREG32(IH_RB_CNTL, tmp);
7739                 wptr &= ~RB_OVERFLOW;
7740         }
7741         return (wptr & rdev->ih.ptr_mask);
7742 }
7743
7744 /*        CIK IV Ring
7745  * Each IV ring entry is 128 bits:
7746  * [7:0]    - interrupt source id
7747  * [31:8]   - reserved
7748  * [59:32]  - interrupt source data
7749  * [63:60]  - reserved
7750  * [71:64]  - RINGID
7751  *            CP:
7752  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7753  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7754  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7755  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7756  *            PIPE_ID - ME0 0=3D
7757  *                    - ME1&2 compute dispatcher (4 pipes each)
7758  *            SDMA:
7759  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7760  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7761  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7762  * [79:72]  - VMID
7763  * [95:80]  - PASID
7764  * [127:96] - reserved
7765  */
7766 /**
7767  * cik_irq_process - interrupt handler
7768  *
7769  * @rdev: radeon_device pointer
7770  *
7771  * Interrupt hander (CIK).  Walk the IH ring,
7772  * ack interrupts and schedule work to handle
7773  * interrupt events.
7774  * Returns irq process return code.
7775  */
7776 int cik_irq_process(struct radeon_device *rdev)
7777 {
7778         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7779         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7780         u32 wptr;
7781         u32 rptr;
7782         u32 src_id, src_data, ring_id;
7783         u8 me_id, pipe_id, queue_id;
7784         u32 ring_index;
7785         bool queue_hotplug = false;
7786         bool queue_reset = false;
7787         u32 addr, status, mc_client;
7788         bool queue_thermal = false;
7789
7790         if (!rdev->ih.enabled || rdev->shutdown)
7791                 return IRQ_NONE;
7792
7793         wptr = cik_get_ih_wptr(rdev);
7794
7795 restart_ih:
7796         /* is somebody else already processing irqs? */
7797         if (atomic_xchg(&rdev->ih.lock, 1))
7798                 return IRQ_NONE;
7799
7800         rptr = rdev->ih.rptr;
7801         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7802
7803         /* Order reading of wptr vs. reading of IH ring data */
7804         rmb();
7805
7806         /* display interrupts */
7807         cik_irq_ack(rdev);
7808
7809         while (rptr != wptr) {
7810                 /* wptr/rptr are in bytes! */
7811                 ring_index = rptr / 4;
7812                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7813                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7814                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7815
7816                 switch (src_id) {
7817                 case 1: /* D1 vblank/vline */
7818                         switch (src_data) {
7819                         case 0: /* D1 vblank */
7820                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7821                                         if (rdev->irq.crtc_vblank_int[0]) {
7822                                                 drm_handle_vblank(rdev->ddev, 0);
7823                                                 rdev->pm.vblank_sync = true;
7824                                                 wake_up(&rdev->irq.vblank_queue);
7825                                         }
7826                                         if (atomic_read(&rdev->irq.pflip[0]))
7827                                                 radeon_crtc_handle_vblank(rdev, 0);
7828                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7829                                         DRM_DEBUG("IH: D1 vblank\n");
7830                                 }
7831                                 break;
7832                         case 1: /* D1 vline */
7833                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7834                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7835                                         DRM_DEBUG("IH: D1 vline\n");
7836                                 }
7837                                 break;
7838                         default:
7839                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7840                                 break;
7841                         }
7842                         break;
7843                 case 2: /* D2 vblank/vline */
7844                         switch (src_data) {
7845                         case 0: /* D2 vblank */
7846                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7847                                         if (rdev->irq.crtc_vblank_int[1]) {
7848                                                 drm_handle_vblank(rdev->ddev, 1);
7849                                                 rdev->pm.vblank_sync = true;
7850                                                 wake_up(&rdev->irq.vblank_queue);
7851                                         }
7852                                         if (atomic_read(&rdev->irq.pflip[1]))
7853                                                 radeon_crtc_handle_vblank(rdev, 1);
7854                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7855                                         DRM_DEBUG("IH: D2 vblank\n");
7856                                 }
7857                                 break;
7858                         case 1: /* D2 vline */
7859                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7860                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7861                                         DRM_DEBUG("IH: D2 vline\n");
7862                                 }
7863                                 break;
7864                         default:
7865                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7866                                 break;
7867                         }
7868                         break;
7869                 case 3: /* D3 vblank/vline */
7870                         switch (src_data) {
7871                         case 0: /* D3 vblank */
7872                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7873                                         if (rdev->irq.crtc_vblank_int[2]) {
7874                                                 drm_handle_vblank(rdev->ddev, 2);
7875                                                 rdev->pm.vblank_sync = true;
7876                                                 wake_up(&rdev->irq.vblank_queue);
7877                                         }
7878                                         if (atomic_read(&rdev->irq.pflip[2]))
7879                                                 radeon_crtc_handle_vblank(rdev, 2);
7880                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7881                                         DRM_DEBUG("IH: D3 vblank\n");
7882                                 }
7883                                 break;
7884                         case 1: /* D3 vline */
7885                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7886                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7887                                         DRM_DEBUG("IH: D3 vline\n");
7888                                 }
7889                                 break;
7890                         default:
7891                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7892                                 break;
7893                         }
7894                         break;
7895                 case 4: /* D4 vblank/vline */
7896                         switch (src_data) {
7897                         case 0: /* D4 vblank */
7898                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7899                                         if (rdev->irq.crtc_vblank_int[3]) {
7900                                                 drm_handle_vblank(rdev->ddev, 3);
7901                                                 rdev->pm.vblank_sync = true;
7902                                                 wake_up(&rdev->irq.vblank_queue);
7903                                         }
7904                                         if (atomic_read(&rdev->irq.pflip[3]))
7905                                                 radeon_crtc_handle_vblank(rdev, 3);
7906                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7907                                         DRM_DEBUG("IH: D4 vblank\n");
7908                                 }
7909                                 break;
7910                         case 1: /* D4 vline */
7911                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7912                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7913                                         DRM_DEBUG("IH: D4 vline\n");
7914                                 }
7915                                 break;
7916                         default:
7917                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7918                                 break;
7919                         }
7920                         break;
7921                 case 5: /* D5 vblank/vline */
7922                         switch (src_data) {
7923                         case 0: /* D5 vblank */
7924                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7925                                         if (rdev->irq.crtc_vblank_int[4]) {
7926                                                 drm_handle_vblank(rdev->ddev, 4);
7927                                                 rdev->pm.vblank_sync = true;
7928                                                 wake_up(&rdev->irq.vblank_queue);
7929                                         }
7930                                         if (atomic_read(&rdev->irq.pflip[4]))
7931                                                 radeon_crtc_handle_vblank(rdev, 4);
7932                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7933                                         DRM_DEBUG("IH: D5 vblank\n");
7934                                 }
7935                                 break;
7936                         case 1: /* D5 vline */
7937                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7938                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7939                                         DRM_DEBUG("IH: D5 vline\n");
7940                                 }
7941                                 break;
7942                         default:
7943                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7944                                 break;
7945                         }
7946                         break;
7947                 case 6: /* D6 vblank/vline */
7948                         switch (src_data) {
7949                         case 0: /* D6 vblank */
7950                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7951                                         if (rdev->irq.crtc_vblank_int[5]) {
7952                                                 drm_handle_vblank(rdev->ddev, 5);
7953                                                 rdev->pm.vblank_sync = true;
7954                                                 wake_up(&rdev->irq.vblank_queue);
7955                                         }
7956                                         if (atomic_read(&rdev->irq.pflip[5]))
7957                                                 radeon_crtc_handle_vblank(rdev, 5);
7958                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7959                                         DRM_DEBUG("IH: D6 vblank\n");
7960                                 }
7961                                 break;
7962                         case 1: /* D6 vline */
7963                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7964                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7965                                         DRM_DEBUG("IH: D6 vline\n");
7966                                 }
7967                                 break;
7968                         default:
7969                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7970                                 break;
7971                         }
7972                         break;
7973                 case 8: /* D1 page flip */
7974                 case 10: /* D2 page flip */
7975                 case 12: /* D3 page flip */
7976                 case 14: /* D4 page flip */
7977                 case 16: /* D5 page flip */
7978                 case 18: /* D6 page flip */
7979                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7980                         if (radeon_use_pflipirq > 0)
7981                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7982                         break;
7983                 case 42: /* HPD hotplug */
7984                         switch (src_data) {
7985                         case 0:
7986                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7987                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7988                                         queue_hotplug = true;
7989                                         DRM_DEBUG("IH: HPD1\n");
7990                                 }
7991                                 break;
7992                         case 1:
7993                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7994                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7995                                         queue_hotplug = true;
7996                                         DRM_DEBUG("IH: HPD2\n");
7997                                 }
7998                                 break;
7999                         case 2:
8000                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8001                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8002                                         queue_hotplug = true;
8003                                         DRM_DEBUG("IH: HPD3\n");
8004                                 }
8005                                 break;
8006                         case 3:
8007                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8008                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8009                                         queue_hotplug = true;
8010                                         DRM_DEBUG("IH: HPD4\n");
8011                                 }
8012                                 break;
8013                         case 4:
8014                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8015                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8016                                         queue_hotplug = true;
8017                                         DRM_DEBUG("IH: HPD5\n");
8018                                 }
8019                                 break;
8020                         case 5:
8021                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8022                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8023                                         queue_hotplug = true;
8024                                         DRM_DEBUG("IH: HPD6\n");
8025                                 }
8026                                 break;
8027                         default:
8028                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8029                                 break;
8030                         }
8031                         break;
8032                 case 124: /* UVD */
8033                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8034                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8035                         break;
8036                 case 146:
8037                 case 147:
8038                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8039                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8040                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
8041                         /* reset addr and status */
8042                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8043                         if (addr == 0x0 && status == 0x0)
8044                                 break;
8045                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8046                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
8047                                 addr);
8048                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
8049                                 status);
8050                         cik_vm_decode_fault(rdev, status, addr, mc_client);
8051                         break;
8052                 case 167: /* VCE */
8053                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8054                         switch (src_data) {
8055                         case 0:
8056                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8057                                 break;
8058                         case 1:
8059                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8060                                 break;
8061                         default:
8062                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8063                                 break;
8064                         }
8065                         break;
8066                 case 176: /* GFX RB CP_INT */
8067                 case 177: /* GFX IB CP_INT */
8068                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8069                         break;
8070                 case 181: /* CP EOP event */
8071                         DRM_DEBUG("IH: CP EOP\n");
8072                         /* XXX check the bitfield order! */
8073                         me_id = (ring_id & 0x60) >> 5;
8074                         pipe_id = (ring_id & 0x18) >> 3;
8075                         queue_id = (ring_id & 0x7) >> 0;
8076                         switch (me_id) {
8077                         case 0:
8078                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8079                                 break;
8080                         case 1:
8081                         case 2:
8082                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8083                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8084                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8085                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8086                                 break;
8087                         }
8088                         break;
8089                 case 184: /* CP Privileged reg access */
8090                         DRM_ERROR("Illegal register access in command stream\n");
8091                         /* XXX check the bitfield order! */
8092                         me_id = (ring_id & 0x60) >> 5;
8093                         pipe_id = (ring_id & 0x18) >> 3;
8094                         queue_id = (ring_id & 0x7) >> 0;
8095                         switch (me_id) {
8096                         case 0:
8097                                 /* This results in a full GPU reset, but all we need to do is soft
8098                                  * reset the CP for gfx
8099                                  */
8100                                 queue_reset = true;
8101                                 break;
8102                         case 1:
8103                                 /* XXX compute */
8104                                 queue_reset = true;
8105                                 break;
8106                         case 2:
8107                                 /* XXX compute */
8108                                 queue_reset = true;
8109                                 break;
8110                         }
8111                         break;
8112                 case 185: /* CP Privileged inst */
8113                         DRM_ERROR("Illegal instruction in command stream\n");
8114                         /* XXX check the bitfield order! */
8115                         me_id = (ring_id & 0x60) >> 5;
8116                         pipe_id = (ring_id & 0x18) >> 3;
8117                         queue_id = (ring_id & 0x7) >> 0;
8118                         switch (me_id) {
8119                         case 0:
8120                                 /* This results in a full GPU reset, but all we need to do is soft
8121                                  * reset the CP for gfx
8122                                  */
8123                                 queue_reset = true;
8124                                 break;
8125                         case 1:
8126                                 /* XXX compute */
8127                                 queue_reset = true;
8128                                 break;
8129                         case 2:
8130                                 /* XXX compute */
8131                                 queue_reset = true;
8132                                 break;
8133                         }
8134                         break;
8135                 case 224: /* SDMA trap event */
8136                         /* XXX check the bitfield order! */
8137                         me_id = (ring_id & 0x3) >> 0;
8138                         queue_id = (ring_id & 0xc) >> 2;
8139                         DRM_DEBUG("IH: SDMA trap\n");
8140                         switch (me_id) {
8141                         case 0:
8142                                 switch (queue_id) {
8143                                 case 0:
8144                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8145                                         break;
8146                                 case 1:
8147                                         /* XXX compute */
8148                                         break;
8149                                 case 2:
8150                                         /* XXX compute */
8151                                         break;
8152                                 }
8153                                 break;
8154                         case 1:
8155                                 switch (queue_id) {
8156                                 case 0:
8157                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8158                                         break;
8159                                 case 1:
8160                                         /* XXX compute */
8161                                         break;
8162                                 case 2:
8163                                         /* XXX compute */
8164                                         break;
8165                                 }
8166                                 break;
8167                         }
8168                         break;
8169                 case 230: /* thermal low to high */
8170                         DRM_DEBUG("IH: thermal low to high\n");
8171                         rdev->pm.dpm.thermal.high_to_low = false;
8172                         queue_thermal = true;
8173                         break;
8174                 case 231: /* thermal high to low */
8175                         DRM_DEBUG("IH: thermal high to low\n");
8176                         rdev->pm.dpm.thermal.high_to_low = true;
8177                         queue_thermal = true;
8178                         break;
8179                 case 233: /* GUI IDLE */
8180                         DRM_DEBUG("IH: GUI idle\n");
8181                         break;
8182                 case 241: /* SDMA Privileged inst */
8183                 case 247: /* SDMA Privileged inst */
8184                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
8185                         /* XXX check the bitfield order! */
8186                         me_id = (ring_id & 0x3) >> 0;
8187                         queue_id = (ring_id & 0xc) >> 2;
8188                         switch (me_id) {
8189                         case 0:
8190                                 switch (queue_id) {
8191                                 case 0:
8192                                         queue_reset = true;
8193                                         break;
8194                                 case 1:
8195                                         /* XXX compute */
8196                                         queue_reset = true;
8197                                         break;
8198                                 case 2:
8199                                         /* XXX compute */
8200                                         queue_reset = true;
8201                                         break;
8202                                 }
8203                                 break;
8204                         case 1:
8205                                 switch (queue_id) {
8206                                 case 0:
8207                                         queue_reset = true;
8208                                         break;
8209                                 case 1:
8210                                         /* XXX compute */
8211                                         queue_reset = true;
8212                                         break;
8213                                 case 2:
8214                                         /* XXX compute */
8215                                         queue_reset = true;
8216                                         break;
8217                                 }
8218                                 break;
8219                         }
8220                         break;
8221                 default:
8222                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8223                         break;
8224                 }
8225
8226                 /* wptr/rptr are in bytes! */
8227                 rptr += 16;
8228                 rptr &= rdev->ih.ptr_mask;
8229         }
8230         if (queue_hotplug)
8231                 schedule_work(&rdev->hotplug_work);
8232         if (queue_reset)
8233                 schedule_work(&rdev->reset_work);
8234         if (queue_thermal)
8235                 schedule_work(&rdev->pm.dpm.thermal.work);
8236         rdev->ih.rptr = rptr;
8237         WREG32(IH_RB_RPTR, rdev->ih.rptr);
8238         atomic_set(&rdev->ih.lock, 0);
8239
8240         /* make sure wptr hasn't changed while processing */
8241         wptr = cik_get_ih_wptr(rdev);
8242         if (wptr != rptr)
8243                 goto restart_ih;
8244
8245         return IRQ_HANDLED;
8246 }
8247
8248 /*
8249  * startup/shutdown callbacks
8250  */
8251 /**
8252  * cik_startup - program the asic to a functional state
8253  *
8254  * @rdev: radeon_device pointer
8255  *
8256  * Programs the asic to a functional state (CIK).
8257  * Called by cik_init() and cik_resume().
8258  * Returns 0 for success, error for failure.
8259  */
8260 static int cik_startup(struct radeon_device *rdev)
8261 {
8262         struct radeon_ring *ring;
8263         u32 nop;
8264         int r;
8265
8266         /* enable pcie gen2/3 link */
8267         cik_pcie_gen3_enable(rdev);
8268         /* enable aspm */
8269         cik_program_aspm(rdev);
8270
8271         /* scratch needs to be initialized before MC */
8272         r = r600_vram_scratch_init(rdev);
8273         if (r)
8274                 return r;
8275
8276         cik_mc_program(rdev);
8277
8278         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
8279                 r = ci_mc_load_microcode(rdev);
8280                 if (r) {
8281                         DRM_ERROR("Failed to load MC firmware!\n");
8282                         return r;
8283                 }
8284         }
8285
8286         r = cik_pcie_gart_enable(rdev);
8287         if (r)
8288                 return r;
8289         cik_gpu_init(rdev);
8290
8291         /* allocate rlc buffers */
8292         if (rdev->flags & RADEON_IS_IGP) {
8293                 if (rdev->family == CHIP_KAVERI) {
8294                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8295                         rdev->rlc.reg_list_size =
8296                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8297                 } else {
8298                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8299                         rdev->rlc.reg_list_size =
8300                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8301                 }
8302         }
8303         rdev->rlc.cs_data = ci_cs_data;
8304         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
8305         r = sumo_rlc_init(rdev);
8306         if (r) {
8307                 DRM_ERROR("Failed to init rlc BOs!\n");
8308                 return r;
8309         }
8310
8311         /* allocate wb buffer */
8312         r = radeon_wb_init(rdev);
8313         if (r)
8314                 return r;
8315
8316         /* allocate mec buffers */
8317         r = cik_mec_init(rdev);
8318         if (r) {
8319                 DRM_ERROR("Failed to init MEC BOs!\n");
8320                 return r;
8321         }
8322
8323         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8324         if (r) {
8325                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8326                 return r;
8327         }
8328
8329         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8330         if (r) {
8331                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8332                 return r;
8333         }
8334
8335         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8336         if (r) {
8337                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8338                 return r;
8339         }
8340
8341         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8342         if (r) {
8343                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8344                 return r;
8345         }
8346
8347         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8348         if (r) {
8349                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8350                 return r;
8351         }
8352
8353         r = radeon_uvd_resume(rdev);
8354         if (!r) {
8355                 r = uvd_v4_2_resume(rdev);
8356                 if (!r) {
8357                         r = radeon_fence_driver_start_ring(rdev,
8358                                                            R600_RING_TYPE_UVD_INDEX);
8359                         if (r)
8360                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8361                 }
8362         }
8363         if (r)
8364                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8365
8366         r = radeon_vce_resume(rdev);
8367         if (!r) {
8368                 r = vce_v2_0_resume(rdev);
8369                 if (!r)
8370                         r = radeon_fence_driver_start_ring(rdev,
8371                                                            TN_RING_TYPE_VCE1_INDEX);
8372                 if (!r)
8373                         r = radeon_fence_driver_start_ring(rdev,
8374                                                            TN_RING_TYPE_VCE2_INDEX);
8375         }
8376         if (r) {
8377                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8378                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8379                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8380         }
8381
8382         /* Enable IRQ */
8383         if (!rdev->irq.installed) {
8384                 r = radeon_irq_kms_init(rdev);
8385                 if (r)
8386                         return r;
8387         }
8388
8389         r = cik_irq_init(rdev);
8390         if (r) {
8391                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8392                 radeon_irq_kms_fini(rdev);
8393                 return r;
8394         }
8395         cik_irq_set(rdev);
8396
8397         if (rdev->family == CHIP_HAWAII) {
8398                 if (rdev->new_fw)
8399                         nop = PACKET3(PACKET3_NOP, 0x3FFF);
8400                 else
8401                         nop = RADEON_CP_PACKET2;
8402         } else {
8403                 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8404         }
8405
8406         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8407         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8408                              nop);
8409         if (r)
8410                 return r;
8411
8412         /* set up the compute queues */
8413         /* type-2 packets are deprecated on MEC, use type-3 instead */
8414         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8415         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8416                              nop);
8417         if (r)
8418                 return r;
8419         ring->me = 1; /* first MEC */
8420         ring->pipe = 0; /* first pipe */
8421         ring->queue = 0; /* first queue */
8422         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8423
8424         /* type-2 packets are deprecated on MEC, use type-3 instead */
8425         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8426         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8427                              nop);
8428         if (r)
8429                 return r;
8430         /* dGPU only have 1 MEC */
8431         ring->me = 1; /* first MEC */
8432         ring->pipe = 0; /* first pipe */
8433         ring->queue = 1; /* second queue */
8434         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8435
8436         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8437         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8438                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8439         if (r)
8440                 return r;
8441
8442         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8443         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8444                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8445         if (r)
8446                 return r;
8447
8448         r = cik_cp_resume(rdev);
8449         if (r)
8450                 return r;
8451
8452         r = cik_sdma_resume(rdev);
8453         if (r)
8454                 return r;
8455
8456         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8457         if (ring->ring_size) {
8458                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8459                                      RADEON_CP_PACKET2);
8460                 if (!r)
8461                         r = uvd_v1_0_init(rdev);
8462                 if (r)
8463                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8464         }
8465
8466         r = -ENOENT;
8467
8468         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8469         if (ring->ring_size)
8470                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8471                                      VCE_CMD_NO_OP);
8472
8473         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8474         if (ring->ring_size)
8475                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8476                                      VCE_CMD_NO_OP);
8477
8478         if (!r)
8479                 r = vce_v1_0_init(rdev);
8480         else if (r != -ENOENT)
8481                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8482
8483         r = radeon_ib_pool_init(rdev);
8484         if (r) {
8485                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8486                 return r;
8487         }
8488
8489         r = radeon_vm_manager_init(rdev);
8490         if (r) {
8491                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8492                 return r;
8493         }
8494
8495         r = dce6_audio_init(rdev);
8496         if (r)
8497                 return r;
8498
8499         return 0;
8500 }
8501
8502 /**
8503  * cik_resume - resume the asic to a functional state
8504  *
8505  * @rdev: radeon_device pointer
8506  *
8507  * Programs the asic to a functional state (CIK).
8508  * Called at resume.
8509  * Returns 0 for success, error for failure.
8510  */
8511 int cik_resume(struct radeon_device *rdev)
8512 {
8513         int r;
8514
8515         /* post card */
8516         atom_asic_init(rdev->mode_info.atom_context);
8517
8518         /* init golden registers */
8519         cik_init_golden_registers(rdev);
8520
8521         if (rdev->pm.pm_method == PM_METHOD_DPM)
8522                 radeon_pm_resume(rdev);
8523
8524         rdev->accel_working = true;
8525         r = cik_startup(rdev);
8526         if (r) {
8527                 DRM_ERROR("cik startup failed on resume\n");
8528                 rdev->accel_working = false;
8529                 return r;
8530         }
8531
8532         return r;
8533
8534 }
8535
8536 /**
8537  * cik_suspend - suspend the asic
8538  *
8539  * @rdev: radeon_device pointer
8540  *
8541  * Bring the chip into a state suitable for suspend (CIK).
8542  * Called at suspend.
8543  * Returns 0 for success.
8544  */
8545 int cik_suspend(struct radeon_device *rdev)
8546 {
8547         radeon_pm_suspend(rdev);
8548         dce6_audio_fini(rdev);
8549         radeon_vm_manager_fini(rdev);
8550         cik_cp_enable(rdev, false);
8551         cik_sdma_enable(rdev, false);
8552         uvd_v1_0_fini(rdev);
8553         radeon_uvd_suspend(rdev);
8554         radeon_vce_suspend(rdev);
8555         cik_fini_pg(rdev);
8556         cik_fini_cg(rdev);
8557         cik_irq_suspend(rdev);
8558         radeon_wb_disable(rdev);
8559         cik_pcie_gart_disable(rdev);
8560         return 0;
8561 }
8562
8563 /* Plan is to move initialization in that function and use
8564  * helper function so that radeon_device_init pretty much
8565  * do nothing more than calling asic specific function. This
8566  * should also allow to remove a bunch of callback function
8567  * like vram_info.
8568  */
8569 /**
8570  * cik_init - asic specific driver and hw init
8571  *
8572  * @rdev: radeon_device pointer
8573  *
8574  * Setup asic specific driver variables and program the hw
8575  * to a functional state (CIK).
8576  * Called at driver startup.
8577  * Returns 0 for success, errors for failure.
8578  */
8579 int cik_init(struct radeon_device *rdev)
8580 {
8581         struct radeon_ring *ring;
8582         int r;
8583
8584         /* Read BIOS */
8585         if (!radeon_get_bios(rdev)) {
8586                 if (ASIC_IS_AVIVO(rdev))
8587                         return -EINVAL;
8588         }
8589         /* Must be an ATOMBIOS */
8590         if (!rdev->is_atom_bios) {
8591                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8592                 return -EINVAL;
8593         }
8594         r = radeon_atombios_init(rdev);
8595         if (r)
8596                 return r;
8597
8598         /* Post card if necessary */
8599         if (!radeon_card_posted(rdev)) {
8600                 if (!rdev->bios) {
8601                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8602                         return -EINVAL;
8603                 }
8604                 DRM_INFO("GPU not posted. posting now...\n");
8605                 atom_asic_init(rdev->mode_info.atom_context);
8606         }
8607         /* init golden registers */
8608         cik_init_golden_registers(rdev);
8609         /* Initialize scratch registers */
8610         cik_scratch_init(rdev);
8611         /* Initialize surface registers */
8612         radeon_surface_init(rdev);
8613         /* Initialize clocks */
8614         radeon_get_clock_info(rdev->ddev);
8615
8616         /* Fence driver */
8617         r = radeon_fence_driver_init(rdev);
8618         if (r)
8619                 return r;
8620
8621         /* initialize memory controller */
8622         r = cik_mc_init(rdev);
8623         if (r)
8624                 return r;
8625         /* Memory manager */
8626         r = radeon_bo_init(rdev);
8627         if (r)
8628                 return r;
8629
8630         if (rdev->flags & RADEON_IS_IGP) {
8631                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8632                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8633                         r = cik_init_microcode(rdev);
8634                         if (r) {
8635                                 DRM_ERROR("Failed to load firmware!\n");
8636                                 return r;
8637                         }
8638                 }
8639         } else {
8640                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8641                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8642                     !rdev->mc_fw) {
8643                         r = cik_init_microcode(rdev);
8644                         if (r) {
8645                                 DRM_ERROR("Failed to load firmware!\n");
8646                                 return r;
8647                         }
8648                 }
8649         }
8650
8651         /* Initialize power management */
8652         radeon_pm_init(rdev);
8653
8654         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8655         ring->ring_obj = NULL;
8656         r600_ring_init(rdev, ring, 1024 * 1024);
8657
8658         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8659         ring->ring_obj = NULL;
8660         r600_ring_init(rdev, ring, 1024 * 1024);
8661         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8662         if (r)
8663                 return r;
8664
8665         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8666         ring->ring_obj = NULL;
8667         r600_ring_init(rdev, ring, 1024 * 1024);
8668         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8669         if (r)
8670                 return r;
8671
8672         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8673         ring->ring_obj = NULL;
8674         r600_ring_init(rdev, ring, 256 * 1024);
8675
8676         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8677         ring->ring_obj = NULL;
8678         r600_ring_init(rdev, ring, 256 * 1024);
8679
8680         r = radeon_uvd_init(rdev);
8681         if (!r) {
8682                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8683                 ring->ring_obj = NULL;
8684                 r600_ring_init(rdev, ring, 4096);
8685         }
8686
8687         r = radeon_vce_init(rdev);
8688         if (!r) {
8689                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8690                 ring->ring_obj = NULL;
8691                 r600_ring_init(rdev, ring, 4096);
8692
8693                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8694                 ring->ring_obj = NULL;
8695                 r600_ring_init(rdev, ring, 4096);
8696         }
8697
8698         rdev->ih.ring_obj = NULL;
8699         r600_ih_ring_init(rdev, 64 * 1024);
8700
8701         r = r600_pcie_gart_init(rdev);
8702         if (r)
8703                 return r;
8704
8705         rdev->accel_working = true;
8706         r = cik_startup(rdev);
8707         if (r) {
8708                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8709                 cik_cp_fini(rdev);
8710                 cik_sdma_fini(rdev);
8711                 cik_irq_fini(rdev);
8712                 sumo_rlc_fini(rdev);
8713                 cik_mec_fini(rdev);
8714                 radeon_wb_fini(rdev);
8715                 radeon_ib_pool_fini(rdev);
8716                 radeon_vm_manager_fini(rdev);
8717                 radeon_irq_kms_fini(rdev);
8718                 cik_pcie_gart_fini(rdev);
8719                 rdev->accel_working = false;
8720         }
8721
8722         /* Don't start up if the MC ucode is missing.
8723          * The default clocks and voltages before the MC ucode
8724          * is loaded are not suffient for advanced operations.
8725          */
8726         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8727                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8728                 return -EINVAL;
8729         }
8730
8731         return 0;
8732 }
8733
8734 /**
8735  * cik_fini - asic specific driver and hw fini
8736  *
8737  * @rdev: radeon_device pointer
8738  *
8739  * Tear down the asic specific driver variables and program the hw
8740  * to an idle state (CIK).
8741  * Called at driver unload.
8742  */
8743 void cik_fini(struct radeon_device *rdev)
8744 {
8745         radeon_pm_fini(rdev);
8746         cik_cp_fini(rdev);
8747         cik_sdma_fini(rdev);
8748         cik_fini_pg(rdev);
8749         cik_fini_cg(rdev);
8750         cik_irq_fini(rdev);
8751         sumo_rlc_fini(rdev);
8752         cik_mec_fini(rdev);
8753         radeon_wb_fini(rdev);
8754         radeon_vm_manager_fini(rdev);
8755         radeon_ib_pool_fini(rdev);
8756         radeon_irq_kms_fini(rdev);
8757         uvd_v1_0_fini(rdev);
8758         radeon_uvd_fini(rdev);
8759         radeon_vce_fini(rdev);
8760         cik_pcie_gart_fini(rdev);
8761         r600_vram_scratch_fini(rdev);
8762         radeon_gem_fini(rdev);
8763         radeon_fence_driver_fini(rdev);
8764         radeon_bo_fini(rdev);
8765         radeon_atombios_fini(rdev);
8766         kfree(rdev->bios);
8767         rdev->bios = NULL;
8768 }
8769
8770 void dce8_program_fmt(struct drm_encoder *encoder)
8771 {
8772         struct drm_device *dev = encoder->dev;
8773         struct radeon_device *rdev = dev->dev_private;
8774         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8775         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8776         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8777         int bpc = 0;
8778         u32 tmp = 0;
8779         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8780
8781         if (connector) {
8782                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8783                 bpc = radeon_get_monitor_bpc(connector);
8784                 dither = radeon_connector->dither;
8785         }
8786
8787         /* LVDS/eDP FMT is set up by atom */
8788         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8789                 return;
8790
8791         /* not needed for analog */
8792         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8793             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8794                 return;
8795
8796         if (bpc == 0)
8797                 return;
8798
8799         switch (bpc) {
8800         case 6:
8801                 if (dither == RADEON_FMT_DITHER_ENABLE)
8802                         /* XXX sort out optimal dither settings */
8803                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8804                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8805                 else
8806                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8807                 break;
8808         case 8:
8809                 if (dither == RADEON_FMT_DITHER_ENABLE)
8810                         /* XXX sort out optimal dither settings */
8811                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8812                                 FMT_RGB_RANDOM_ENABLE |
8813                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8814                 else
8815                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8816                 break;
8817         case 10:
8818                 if (dither == RADEON_FMT_DITHER_ENABLE)
8819                         /* XXX sort out optimal dither settings */
8820                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821                                 FMT_RGB_RANDOM_ENABLE |
8822                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8823                 else
8824                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8825                 break;
8826         default:
8827                 /* not needed */
8828                 break;
8829         }
8830
8831         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8832 }
8833
8834 /* display watermark setup */
8835 /**
8836  * dce8_line_buffer_adjust - Set up the line buffer
8837  *
8838  * @rdev: radeon_device pointer
8839  * @radeon_crtc: the selected display controller
8840  * @mode: the current display mode on the selected display
8841  * controller
8842  *
8843  * Setup up the line buffer allocation for
8844  * the selected display controller (CIK).
8845  * Returns the line buffer size in pixels.
8846  */
8847 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8848                                    struct radeon_crtc *radeon_crtc,
8849                                    struct drm_display_mode *mode)
8850 {
8851         u32 tmp, buffer_alloc, i;
8852         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8853         /*
8854          * Line Buffer Setup
8855          * There are 6 line buffers, one for each display controllers.
8856          * There are 3 partitions per LB. Select the number of partitions
8857          * to enable based on the display width.  For display widths larger
8858          * than 4096, you need use to use 2 display controllers and combine
8859          * them using the stereo blender.
8860          */
8861         if (radeon_crtc->base.enabled && mode) {
8862                 if (mode->crtc_hdisplay < 1920) {
8863                         tmp = 1;
8864                         buffer_alloc = 2;
8865                 } else if (mode->crtc_hdisplay < 2560) {
8866                         tmp = 2;
8867                         buffer_alloc = 2;
8868                 } else if (mode->crtc_hdisplay < 4096) {
8869                         tmp = 0;
8870                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8871                 } else {
8872                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8873                         tmp = 0;
8874                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8875                 }
8876         } else {
8877                 tmp = 1;
8878                 buffer_alloc = 0;
8879         }
8880
8881         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8882                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8883
8884         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8885                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8886         for (i = 0; i < rdev->usec_timeout; i++) {
8887                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8888                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8889                         break;
8890                 udelay(1);
8891         }
8892
8893         if (radeon_crtc->base.enabled && mode) {
8894                 switch (tmp) {
8895                 case 0:
8896                 default:
8897                         return 4096 * 2;
8898                 case 1:
8899                         return 1920 * 2;
8900                 case 2:
8901                         return 2560 * 2;
8902                 }
8903         }
8904
8905         /* controller not enabled, so no lb used */
8906         return 0;
8907 }
8908
8909 /**
8910  * cik_get_number_of_dram_channels - get the number of dram channels
8911  *
8912  * @rdev: radeon_device pointer
8913  *
8914  * Look up the number of video ram channels (CIK).
8915  * Used for display watermark bandwidth calculations
8916  * Returns the number of dram channels
8917  */
8918 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8919 {
8920         u32 tmp = RREG32(MC_SHARED_CHMAP);
8921
8922         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8923         case 0:
8924         default:
8925                 return 1;
8926         case 1:
8927                 return 2;
8928         case 2:
8929                 return 4;
8930         case 3:
8931                 return 8;
8932         case 4:
8933                 return 3;
8934         case 5:
8935                 return 6;
8936         case 6:
8937                 return 10;
8938         case 7:
8939                 return 12;
8940         case 8:
8941                 return 16;
8942         }
8943 }
8944
8945 struct dce8_wm_params {
8946         u32 dram_channels; /* number of dram channels */
8947         u32 yclk;          /* bandwidth per dram data pin in kHz */
8948         u32 sclk;          /* engine clock in kHz */
8949         u32 disp_clk;      /* display clock in kHz */
8950         u32 src_width;     /* viewport width */
8951         u32 active_time;   /* active display time in ns */
8952         u32 blank_time;    /* blank time in ns */
8953         bool interlaced;    /* mode is interlaced */
8954         fixed20_12 vsc;    /* vertical scale ratio */
8955         u32 num_heads;     /* number of active crtcs */
8956         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8957         u32 lb_size;       /* line buffer allocated to pipe */
8958         u32 vtaps;         /* vertical scaler taps */
8959 };
8960
8961 /**
8962  * dce8_dram_bandwidth - get the dram bandwidth
8963  *
8964  * @wm: watermark calculation data
8965  *
8966  * Calculate the raw dram bandwidth (CIK).
8967  * Used for display watermark bandwidth calculations
8968  * Returns the dram bandwidth in MBytes/s
8969  */
8970 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8971 {
8972         /* Calculate raw DRAM Bandwidth */
8973         fixed20_12 dram_efficiency; /* 0.7 */
8974         fixed20_12 yclk, dram_channels, bandwidth;
8975         fixed20_12 a;
8976
8977         a.full = dfixed_const(1000);
8978         yclk.full = dfixed_const(wm->yclk);
8979         yclk.full = dfixed_div(yclk, a);
8980         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8981         a.full = dfixed_const(10);
8982         dram_efficiency.full = dfixed_const(7);
8983         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8984         bandwidth.full = dfixed_mul(dram_channels, yclk);
8985         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8986
8987         return dfixed_trunc(bandwidth);
8988 }
8989
8990 /**
8991  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8992  *
8993  * @wm: watermark calculation data
8994  *
8995  * Calculate the dram bandwidth used for display (CIK).
8996  * Used for display watermark bandwidth calculations
8997  * Returns the dram bandwidth for display in MBytes/s
8998  */
8999 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9000 {
9001         /* Calculate DRAM Bandwidth and the part allocated to display. */
9002         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9003         fixed20_12 yclk, dram_channels, bandwidth;
9004         fixed20_12 a;
9005
9006         a.full = dfixed_const(1000);
9007         yclk.full = dfixed_const(wm->yclk);
9008         yclk.full = dfixed_div(yclk, a);
9009         dram_channels.full = dfixed_const(wm->dram_channels * 4);
9010         a.full = dfixed_const(10);
9011         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9012         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9013         bandwidth.full = dfixed_mul(dram_channels, yclk);
9014         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9015
9016         return dfixed_trunc(bandwidth);
9017 }
9018
9019 /**
9020  * dce8_data_return_bandwidth - get the data return bandwidth
9021  *
9022  * @wm: watermark calculation data
9023  *
9024  * Calculate the data return bandwidth used for display (CIK).
9025  * Used for display watermark bandwidth calculations
9026  * Returns the data return bandwidth in MBytes/s
9027  */
9028 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9029 {
9030         /* Calculate the display Data return Bandwidth */
9031         fixed20_12 return_efficiency; /* 0.8 */
9032         fixed20_12 sclk, bandwidth;
9033         fixed20_12 a;
9034
9035         a.full = dfixed_const(1000);
9036         sclk.full = dfixed_const(wm->sclk);
9037         sclk.full = dfixed_div(sclk, a);
9038         a.full = dfixed_const(10);
9039         return_efficiency.full = dfixed_const(8);
9040         return_efficiency.full = dfixed_div(return_efficiency, a);
9041         a.full = dfixed_const(32);
9042         bandwidth.full = dfixed_mul(a, sclk);
9043         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9044
9045         return dfixed_trunc(bandwidth);
9046 }
9047
9048 /**
9049  * dce8_dmif_request_bandwidth - get the dmif bandwidth
9050  *
9051  * @wm: watermark calculation data
9052  *
9053  * Calculate the dmif bandwidth used for display (CIK).
9054  * Used for display watermark bandwidth calculations
9055  * Returns the dmif bandwidth in MBytes/s
9056  */
9057 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9058 {
9059         /* Calculate the DMIF Request Bandwidth */
9060         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9061         fixed20_12 disp_clk, bandwidth;
9062         fixed20_12 a, b;
9063
9064         a.full = dfixed_const(1000);
9065         disp_clk.full = dfixed_const(wm->disp_clk);
9066         disp_clk.full = dfixed_div(disp_clk, a);
9067         a.full = dfixed_const(32);
9068         b.full = dfixed_mul(a, disp_clk);
9069
9070         a.full = dfixed_const(10);
9071         disp_clk_request_efficiency.full = dfixed_const(8);
9072         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9073
9074         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9075
9076         return dfixed_trunc(bandwidth);
9077 }
9078
9079 /**
9080  * dce8_available_bandwidth - get the min available bandwidth
9081  *
9082  * @wm: watermark calculation data
9083  *
9084  * Calculate the min available bandwidth used for display (CIK).
9085  * Used for display watermark bandwidth calculations
9086  * Returns the min available bandwidth in MBytes/s
9087  */
9088 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9089 {
9090         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9091         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9092         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9093         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9094
9095         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9096 }
9097
9098 /**
9099  * dce8_average_bandwidth - get the average available bandwidth
9100  *
9101  * @wm: watermark calculation data
9102  *
9103  * Calculate the average available bandwidth used for display (CIK).
9104  * Used for display watermark bandwidth calculations
9105  * Returns the average available bandwidth in MBytes/s
9106  */
9107 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9108 {
9109         /* Calculate the display mode Average Bandwidth
9110          * DisplayMode should contain the source and destination dimensions,
9111          * timing, etc.
9112          */
9113         fixed20_12 bpp;
9114         fixed20_12 line_time;
9115         fixed20_12 src_width;
9116         fixed20_12 bandwidth;
9117         fixed20_12 a;
9118
9119         a.full = dfixed_const(1000);
9120         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9121         line_time.full = dfixed_div(line_time, a);
9122         bpp.full = dfixed_const(wm->bytes_per_pixel);
9123         src_width.full = dfixed_const(wm->src_width);
9124         bandwidth.full = dfixed_mul(src_width, bpp);
9125         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9126         bandwidth.full = dfixed_div(bandwidth, line_time);
9127
9128         return dfixed_trunc(bandwidth);
9129 }
9130
9131 /**
9132  * dce8_latency_watermark - get the latency watermark
9133  *
9134  * @wm: watermark calculation data
9135  *
9136  * Calculate the latency watermark (CIK).
9137  * Used for display watermark bandwidth calculations
9138  * Returns the latency watermark in ns
9139  */
9140 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9141 {
9142         /* First calculate the latency in ns */
9143         u32 mc_latency = 2000; /* 2000 ns. */
9144         u32 available_bandwidth = dce8_available_bandwidth(wm);
9145         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9146         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9147         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9148         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9149                 (wm->num_heads * cursor_line_pair_return_time);
9150         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9151         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9152         u32 tmp, dmif_size = 12288;
9153         fixed20_12 a, b, c;
9154
9155         if (wm->num_heads == 0)
9156                 return 0;
9157
9158         a.full = dfixed_const(2);
9159         b.full = dfixed_const(1);
9160         if ((wm->vsc.full > a.full) ||
9161             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9162             (wm->vtaps >= 5) ||
9163             ((wm->vsc.full >= a.full) && wm->interlaced))
9164                 max_src_lines_per_dst_line = 4;
9165         else
9166                 max_src_lines_per_dst_line = 2;
9167
9168         a.full = dfixed_const(available_bandwidth);
9169         b.full = dfixed_const(wm->num_heads);
9170         a.full = dfixed_div(a, b);
9171
9172         b.full = dfixed_const(mc_latency + 512);
9173         c.full = dfixed_const(wm->disp_clk);
9174         b.full = dfixed_div(b, c);
9175
9176         c.full = dfixed_const(dmif_size);
9177         b.full = dfixed_div(c, b);
9178
9179         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9180
9181         b.full = dfixed_const(1000);
9182         c.full = dfixed_const(wm->disp_clk);
9183         b.full = dfixed_div(c, b);
9184         c.full = dfixed_const(wm->bytes_per_pixel);
9185         b.full = dfixed_mul(b, c);
9186
9187         lb_fill_bw = min(tmp, dfixed_trunc(b));
9188
9189         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9190         b.full = dfixed_const(1000);
9191         c.full = dfixed_const(lb_fill_bw);
9192         b.full = dfixed_div(c, b);
9193         a.full = dfixed_div(a, b);
9194         line_fill_time = dfixed_trunc(a);
9195
9196         if (line_fill_time < wm->active_time)
9197                 return latency;
9198         else
9199                 return latency + (line_fill_time - wm->active_time);
9200
9201 }
9202
9203 /**
9204  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9205  * average and available dram bandwidth
9206  *
9207  * @wm: watermark calculation data
9208  *
9209  * Check if the display average bandwidth fits in the display
9210  * dram bandwidth (CIK).
9211  * Used for display watermark bandwidth calculations
9212  * Returns true if the display fits, false if not.
9213  */
9214 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9215 {
9216         if (dce8_average_bandwidth(wm) <=
9217             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9218                 return true;
9219         else
9220                 return false;
9221 }
9222
9223 /**
9224  * dce8_average_bandwidth_vs_available_bandwidth - check
9225  * average and available bandwidth
9226  *
9227  * @wm: watermark calculation data
9228  *
9229  * Check if the display average bandwidth fits in the display
9230  * available bandwidth (CIK).
9231  * Used for display watermark bandwidth calculations
9232  * Returns true if the display fits, false if not.
9233  */
9234 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9235 {
9236         if (dce8_average_bandwidth(wm) <=
9237             (dce8_available_bandwidth(wm) / wm->num_heads))
9238                 return true;
9239         else
9240                 return false;
9241 }
9242
9243 /**
9244  * dce8_check_latency_hiding - check latency hiding
9245  *
9246  * @wm: watermark calculation data
9247  *
9248  * Check latency hiding (CIK).
9249  * Used for display watermark bandwidth calculations
9250  * Returns true if the display fits, false if not.
9251  */
9252 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9253 {
9254         u32 lb_partitions = wm->lb_size / wm->src_width;
9255         u32 line_time = wm->active_time + wm->blank_time;
9256         u32 latency_tolerant_lines;
9257         u32 latency_hiding;
9258         fixed20_12 a;
9259
9260         a.full = dfixed_const(1);
9261         if (wm->vsc.full > a.full)
9262                 latency_tolerant_lines = 1;
9263         else {
9264                 if (lb_partitions <= (wm->vtaps + 1))
9265                         latency_tolerant_lines = 1;
9266                 else
9267                         latency_tolerant_lines = 2;
9268         }
9269
9270         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9271
9272         if (dce8_latency_watermark(wm) <= latency_hiding)
9273                 return true;
9274         else
9275                 return false;
9276 }
9277
9278 /**
9279  * dce8_program_watermarks - program display watermarks
9280  *
9281  * @rdev: radeon_device pointer
9282  * @radeon_crtc: the selected display controller
9283  * @lb_size: line buffer size
9284  * @num_heads: number of display controllers in use
9285  *
9286  * Calculate and program the display watermarks for the
9287  * selected display controller (CIK).
9288  */
9289 static void dce8_program_watermarks(struct radeon_device *rdev,
9290                                     struct radeon_crtc *radeon_crtc,
9291                                     u32 lb_size, u32 num_heads)
9292 {
9293         struct drm_display_mode *mode = &radeon_crtc->base.mode;
9294         struct dce8_wm_params wm_low, wm_high;
9295         u32 pixel_period;
9296         u32 line_time = 0;
9297         u32 latency_watermark_a = 0, latency_watermark_b = 0;
9298         u32 tmp, wm_mask;
9299
9300         if (radeon_crtc->base.enabled && num_heads && mode) {
9301                 pixel_period = 1000000 / (u32)mode->clock;
9302                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9303
9304                 /* watermark for high clocks */
9305                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9306                     rdev->pm.dpm_enabled) {
9307                         wm_high.yclk =
9308                                 radeon_dpm_get_mclk(rdev, false) * 10;
9309                         wm_high.sclk =
9310                                 radeon_dpm_get_sclk(rdev, false) * 10;
9311                 } else {
9312                         wm_high.yclk = rdev->pm.current_mclk * 10;
9313                         wm_high.sclk = rdev->pm.current_sclk * 10;
9314                 }
9315
9316                 wm_high.disp_clk = mode->clock;
9317                 wm_high.src_width = mode->crtc_hdisplay;
9318                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9319                 wm_high.blank_time = line_time - wm_high.active_time;
9320                 wm_high.interlaced = false;
9321                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9322                         wm_high.interlaced = true;
9323                 wm_high.vsc = radeon_crtc->vsc;
9324                 wm_high.vtaps = 1;
9325                 if (radeon_crtc->rmx_type != RMX_OFF)
9326                         wm_high.vtaps = 2;
9327                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9328                 wm_high.lb_size = lb_size;
9329                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9330                 wm_high.num_heads = num_heads;
9331
9332                 /* set for high clocks */
9333                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9334
9335                 /* possibly force display priority to high */
9336                 /* should really do this at mode validation time... */
9337                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9338                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9339                     !dce8_check_latency_hiding(&wm_high) ||
9340                     (rdev->disp_priority == 2)) {
9341                         DRM_DEBUG_KMS("force priority to high\n");
9342                 }
9343
9344                 /* watermark for low clocks */
9345                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9346                     rdev->pm.dpm_enabled) {
9347                         wm_low.yclk =
9348                                 radeon_dpm_get_mclk(rdev, true) * 10;
9349                         wm_low.sclk =
9350                                 radeon_dpm_get_sclk(rdev, true) * 10;
9351                 } else {
9352                         wm_low.yclk = rdev->pm.current_mclk * 10;
9353                         wm_low.sclk = rdev->pm.current_sclk * 10;
9354                 }
9355
9356                 wm_low.disp_clk = mode->clock;
9357                 wm_low.src_width = mode->crtc_hdisplay;
9358                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9359                 wm_low.blank_time = line_time - wm_low.active_time;
9360                 wm_low.interlaced = false;
9361                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9362                         wm_low.interlaced = true;
9363                 wm_low.vsc = radeon_crtc->vsc;
9364                 wm_low.vtaps = 1;
9365                 if (radeon_crtc->rmx_type != RMX_OFF)
9366                         wm_low.vtaps = 2;
9367                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9368                 wm_low.lb_size = lb_size;
9369                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9370                 wm_low.num_heads = num_heads;
9371
9372                 /* set for low clocks */
9373                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9374
9375                 /* possibly force display priority to high */
9376                 /* should really do this at mode validation time... */
9377                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9378                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9379                     !dce8_check_latency_hiding(&wm_low) ||
9380                     (rdev->disp_priority == 2)) {
9381                         DRM_DEBUG_KMS("force priority to high\n");
9382                 }
9383         }
9384
9385         /* select wm A */
9386         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9387         tmp = wm_mask;
9388         tmp &= ~LATENCY_WATERMARK_MASK(3);
9389         tmp |= LATENCY_WATERMARK_MASK(1);
9390         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9391         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9392                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9393                 LATENCY_HIGH_WATERMARK(line_time)));
9394         /* select wm B */
9395         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9396         tmp &= ~LATENCY_WATERMARK_MASK(3);
9397         tmp |= LATENCY_WATERMARK_MASK(2);
9398         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9399         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9400                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9401                 LATENCY_HIGH_WATERMARK(line_time)));
9402         /* restore original selection */
9403         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9404
9405         /* save values for DPM */
9406         radeon_crtc->line_time = line_time;
9407         radeon_crtc->wm_high = latency_watermark_a;
9408         radeon_crtc->wm_low = latency_watermark_b;
9409 }
9410
9411 /**
9412  * dce8_bandwidth_update - program display watermarks
9413  *
9414  * @rdev: radeon_device pointer
9415  *
9416  * Calculate and program the display watermarks and line
9417  * buffer allocation (CIK).
9418  */
9419 void dce8_bandwidth_update(struct radeon_device *rdev)
9420 {
9421         struct drm_display_mode *mode = NULL;
9422         u32 num_heads = 0, lb_size;
9423         int i;
9424
9425         radeon_update_display_priority(rdev);
9426
9427         for (i = 0; i < rdev->num_crtc; i++) {
9428                 if (rdev->mode_info.crtcs[i]->base.enabled)
9429                         num_heads++;
9430         }
9431         for (i = 0; i < rdev->num_crtc; i++) {
9432                 mode = &rdev->mode_info.crtcs[i]->base.mode;
9433                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9434                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9435         }
9436 }
9437
9438 /**
9439  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9440  *
9441  * @rdev: radeon_device pointer
9442  *
9443  * Fetches a GPU clock counter snapshot (SI).
9444  * Returns the 64 bit clock counter snapshot.
9445  */
9446 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9447 {
9448         uint64_t clock;
9449
9450         mutex_lock(&rdev->gpu_clock_mutex);
9451         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9452         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9453                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9454         mutex_unlock(&rdev->gpu_clock_mutex);
9455         return clock;
9456 }
9457
9458 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9459                               u32 cntl_reg, u32 status_reg)
9460 {
9461         int r, i;
9462         struct atom_clock_dividers dividers;
9463         uint32_t tmp;
9464
9465         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9466                                            clock, false, &dividers);
9467         if (r)
9468                 return r;
9469
9470         tmp = RREG32_SMC(cntl_reg);
9471         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9472         tmp |= dividers.post_divider;
9473         WREG32_SMC(cntl_reg, tmp);
9474
9475         for (i = 0; i < 100; i++) {
9476                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9477                         break;
9478                 mdelay(10);
9479         }
9480         if (i == 100)
9481                 return -ETIMEDOUT;
9482
9483         return 0;
9484 }
9485
9486 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9487 {
9488         int r = 0;
9489
9490         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9491         if (r)
9492                 return r;
9493
9494         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9495         return r;
9496 }
9497
9498 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9499 {
9500         int r, i;
9501         struct atom_clock_dividers dividers;
9502         u32 tmp;
9503
9504         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9505                                            ecclk, false, &dividers);
9506         if (r)
9507                 return r;
9508
9509         for (i = 0; i < 100; i++) {
9510                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9511                         break;
9512                 mdelay(10);
9513         }
9514         if (i == 100)
9515                 return -ETIMEDOUT;
9516
9517         tmp = RREG32_SMC(CG_ECLK_CNTL);
9518         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9519         tmp |= dividers.post_divider;
9520         WREG32_SMC(CG_ECLK_CNTL, tmp);
9521
9522         for (i = 0; i < 100; i++) {
9523                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9524                         break;
9525                 mdelay(10);
9526         }
9527         if (i == 100)
9528                 return -ETIMEDOUT;
9529
9530         return 0;
9531 }
9532
9533 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9534 {
9535         struct pci_dev *root = rdev->pdev->bus->self;
9536         int bridge_pos, gpu_pos;
9537         u32 speed_cntl, mask, current_data_rate;
9538         int ret, i;
9539         u16 tmp16;
9540
9541         if (radeon_pcie_gen2 == 0)
9542                 return;
9543
9544         if (rdev->flags & RADEON_IS_IGP)
9545                 return;
9546
9547         if (!(rdev->flags & RADEON_IS_PCIE))
9548                 return;
9549
9550         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9551         if (ret != 0)
9552                 return;
9553
9554         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9555                 return;
9556
9557         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9558         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9559                 LC_CURRENT_DATA_RATE_SHIFT;
9560         if (mask & DRM_PCIE_SPEED_80) {
9561                 if (current_data_rate == 2) {
9562                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9563                         return;
9564                 }
9565                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9566         } else if (mask & DRM_PCIE_SPEED_50) {
9567                 if (current_data_rate == 1) {
9568                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9569                         return;
9570                 }
9571                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9572         }
9573
9574         bridge_pos = pci_pcie_cap(root);
9575         if (!bridge_pos)
9576                 return;
9577
9578         gpu_pos = pci_pcie_cap(rdev->pdev);
9579         if (!gpu_pos)
9580                 return;
9581
9582         if (mask & DRM_PCIE_SPEED_80) {
9583                 /* re-try equalization if gen3 is not already enabled */
9584                 if (current_data_rate != 2) {
9585                         u16 bridge_cfg, gpu_cfg;
9586                         u16 bridge_cfg2, gpu_cfg2;
9587                         u32 max_lw, current_lw, tmp;
9588
9589                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9590                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9591
9592                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9593                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9594
9595                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9596                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9597
9598                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9599                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9600                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9601
9602                         if (current_lw < max_lw) {
9603                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9604                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9605                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9606                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9607                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9608                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9609                                 }
9610                         }
9611
9612                         for (i = 0; i < 10; i++) {
9613                                 /* check status */
9614                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9615                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9616                                         break;
9617
9618                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9619                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9620
9621                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9622                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9623
9624                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9625                                 tmp |= LC_SET_QUIESCE;
9626                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9627
9628                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9629                                 tmp |= LC_REDO_EQ;
9630                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9631
9632                                 mdelay(100);
9633
9634                                 /* linkctl */
9635                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9636                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9637                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9638                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9639
9640                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9641                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9642                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9643                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9644
9645                                 /* linkctl2 */
9646                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9647                                 tmp16 &= ~((1 << 4) | (7 << 9));
9648                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9649                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9650
9651                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9652                                 tmp16 &= ~((1 << 4) | (7 << 9));
9653                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9654                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9655
9656                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9657                                 tmp &= ~LC_SET_QUIESCE;
9658                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9659                         }
9660                 }
9661         }
9662
9663         /* set the link speed */
9664         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9665         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9666         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9667
9668         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9669         tmp16 &= ~0xf;
9670         if (mask & DRM_PCIE_SPEED_80)
9671                 tmp16 |= 3; /* gen3 */
9672         else if (mask & DRM_PCIE_SPEED_50)
9673                 tmp16 |= 2; /* gen2 */
9674         else
9675                 tmp16 |= 1; /* gen1 */
9676         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9677
9678         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9679         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9680         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9681
9682         for (i = 0; i < rdev->usec_timeout; i++) {
9683                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9684                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9685                         break;
9686                 udelay(1);
9687         }
9688 }
9689
9690 static void cik_program_aspm(struct radeon_device *rdev)
9691 {
9692         u32 data, orig;
9693         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9694         bool disable_clkreq = false;
9695
9696         if (radeon_aspm == 0)
9697                 return;
9698
9699         /* XXX double check IGPs */
9700         if (rdev->flags & RADEON_IS_IGP)
9701                 return;
9702
9703         if (!(rdev->flags & RADEON_IS_PCIE))
9704                 return;
9705
9706         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9707         data &= ~LC_XMIT_N_FTS_MASK;
9708         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9709         if (orig != data)
9710                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9711
9712         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9713         data |= LC_GO_TO_RECOVERY;
9714         if (orig != data)
9715                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9716
9717         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9718         data |= P_IGNORE_EDB_ERR;
9719         if (orig != data)
9720                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9721
9722         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9723         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9724         data |= LC_PMI_TO_L1_DIS;
9725         if (!disable_l0s)
9726                 data |= LC_L0S_INACTIVITY(7);
9727
9728         if (!disable_l1) {
9729                 data |= LC_L1_INACTIVITY(7);
9730                 data &= ~LC_PMI_TO_L1_DIS;
9731                 if (orig != data)
9732                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9733
9734                 if (!disable_plloff_in_l1) {
9735                         bool clk_req_support;
9736
9737                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9738                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9739                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9740                         if (orig != data)
9741                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9742
9743                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9744                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9745                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9746                         if (orig != data)
9747                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9748
9749                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9750                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9751                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9752                         if (orig != data)
9753                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9754
9755                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9756                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9757                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9758                         if (orig != data)
9759                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9760
9761                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9762                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9763                         data |= LC_DYN_LANES_PWR_STATE(3);
9764                         if (orig != data)
9765                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9766
9767                         if (!disable_clkreq) {
9768                                 struct pci_dev *root = rdev->pdev->bus->self;
9769                                 u32 lnkcap;
9770
9771                                 clk_req_support = false;
9772                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9773                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9774                                         clk_req_support = true;
9775                         } else {
9776                                 clk_req_support = false;
9777                         }
9778
9779                         if (clk_req_support) {
9780                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9781                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9782                                 if (orig != data)
9783                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9784
9785                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9786                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9787                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9788                                 if (orig != data)
9789                                         WREG32_SMC(THM_CLK_CNTL, data);
9790
9791                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9792                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9793                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9794                                 if (orig != data)
9795                                         WREG32_SMC(MISC_CLK_CTRL, data);
9796
9797                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9798                                 data &= ~BCLK_AS_XCLK;
9799                                 if (orig != data)
9800                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9801
9802                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9803                                 data &= ~FORCE_BIF_REFCLK_EN;
9804                                 if (orig != data)
9805                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9806
9807                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9808                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9809                                 data |= MPLL_CLKOUT_SEL(4);
9810                                 if (orig != data)
9811                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9812                         }
9813                 }
9814         } else {
9815                 if (orig != data)
9816                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9817         }
9818
9819         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9820         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9821         if (orig != data)
9822                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9823
9824         if (!disable_l0s) {
9825                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9826                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9827                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9828                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9829                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9830                                 data &= ~LC_L0S_INACTIVITY_MASK;
9831                                 if (orig != data)
9832                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9833                         }
9834                 }
9835         }
9836 }