drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <drm/drmP.h>
  27 #include "radeon.h"
  28 #include "radeon_asic.h"
  29 #include <drm/radeon_drm.h>
  30 #include "evergreend.h"
  31 #include "atom.h"
  32 #include "avivod.h"
  33 #include "evergreen_reg.h"
  34 #include "evergreen_blit_shaders.h"
  35 #include "radeon_ucode.h"
  36
  37 static const u32 crtc_offsets[6] =
  38 {
  39         EVERGREEN_CRTC0_REGISTER_OFFSET,
  40         EVERGREEN_CRTC1_REGISTER_OFFSET,
  41         EVERGREEN_CRTC2_REGISTER_OFFSET,
  42         EVERGREEN_CRTC3_REGISTER_OFFSET,
  43         EVERGREEN_CRTC4_REGISTER_OFFSET,
  44         EVERGREEN_CRTC5_REGISTER_OFFSET
  45 };
  46
  47 #include "clearstate_evergreen.h"
  48
  49 static const u32 sumo_rlc_save_restore_register_list[] =
  50 {
  51         0x98fc,
  52         0x9830,
  53         0x9834,
  54         0x9838,
  55         0x9870,
  56         0x9874,
  57         0x8a14,
  58         0x8b24,
  59         0x8bcc,
  60         0x8b10,
  61         0x8d00,
  62         0x8d04,
  63         0x8c00,
  64         0x8c04,
  65         0x8c08,
  66         0x8c0c,
  67         0x8d8c,
  68         0x8c20,
  69         0x8c24,
  70         0x8c28,
  71         0x8c18,
  72         0x8c1c,
  73         0x8cf0,
  74         0x8e2c,
  75         0x8e38,
  76         0x8c30,
  77         0x9508,
  78         0x9688,
  79         0x9608,
  80         0x960c,
  81         0x9610,
  82         0x9614,
  83         0x88c4,
  84         0x88d4,
  85         0xa008,
  86         0x900c,
  87         0x9100,
  88         0x913c,
  89         0x98f8,
  90         0x98f4,
  91         0x9b7c,
  92         0x3f8c,
  93         0x8950,
  94         0x8954,
  95         0x8a18,
  96         0x8b28,
  97         0x9144,
  98         0x9148,
  99         0x914c,
 100         0x3f90,
 101         0x3f94,
 102         0x915c,
 103         0x9160,
 104         0x9178,
 105         0x917c,
 106         0x9180,
 107         0x918c,
 108         0x9190,
 109         0x9194,
 110         0x9198,
 111         0x919c,
 112         0x91a8,
 113         0x91ac,
 114         0x91b0,
 115         0x91b4,
 116         0x91b8,
 117         0x91c4,
 118         0x91c8,
 119         0x91cc,
 120         0x91d0,
 121         0x91d4,
 122         0x91e0,
 123         0x91e4,
 124         0x91ec,
 125         0x91f0,
 126         0x91f4,
 127         0x9200,
 128         0x9204,
 129         0x929c,
 130         0x9150,
 131         0x802c,
 132 };
 133
 134 static void evergreen_gpu_init(struct radeon_device *rdev);
 135 void evergreen_fini(struct radeon_device *rdev);
 136 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 137 void evergreen_program_aspm(struct radeon_device *rdev);
 138 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 139                                      int ring, u32 cp_int_cntl);
 140 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 141                                    u32 status, u32 addr);
 142 void cik_init_cp_pg_table(struct radeon_device *rdev);
 143
 144 extern u32 si_get_csb_size(struct radeon_device *rdev);
 145 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 146 extern u32 cik_get_csb_size(struct radeon_device *rdev);
 147 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
 148 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
 149
 150 static const u32 evergreen_golden_registers[] =
 151 {
 152         0x3f90, 0xffff0000, 0xff000000,
 153         0x9148, 0xffff0000, 0xff000000,
 154         0x3f94, 0xffff0000, 0xff000000,
 155         0x914c, 0xffff0000, 0xff000000,
 156         0x9b7c, 0xffffffff, 0x00000000,
 157         0x8a14, 0xffffffff, 0x00000007,
 158         0x8b10, 0xffffffff, 0x00000000,
 159         0x960c, 0xffffffff, 0x54763210,
 160         0x88c4, 0xffffffff, 0x000000c2,
 161         0x88d4, 0xffffffff, 0x00000010,
 162         0x8974, 0xffffffff, 0x00000000,
 163         0xc78, 0x00000080, 0x00000080,
 164         0x5eb4, 0xffffffff, 0x00000002,
 165         0x5e78, 0xffffffff, 0x001000f0,
 166         0x6104, 0x01000300, 0x00000000,
 167         0x5bc0, 0x00300000, 0x00000000,
 168         0x7030, 0xffffffff, 0x00000011,
 169         0x7c30, 0xffffffff, 0x00000011,
 170         0x10830, 0xffffffff, 0x00000011,
 171         0x11430, 0xffffffff, 0x00000011,
 172         0x12030, 0xffffffff, 0x00000011,
 173         0x12c30, 0xffffffff, 0x00000011,
 174         0xd02c, 0xffffffff, 0x08421000,
 175         0x240c, 0xffffffff, 0x00000380,
 176         0x8b24, 0xffffffff, 0x00ff0fff,
 177         0x28a4c, 0x06000000, 0x06000000,
 178         0x10c, 0x00000001, 0x00000001,
 179         0x8d00, 0xffffffff, 0x100e4848,
 180         0x8d04, 0xffffffff, 0x00164745,
 181         0x8c00, 0xffffffff, 0xe4000003,
 182         0x8c04, 0xffffffff, 0x40600060,
 183         0x8c08, 0xffffffff, 0x001c001c,
 184         0x8cf0, 0xffffffff, 0x08e00620,
 185         0x8c20, 0xffffffff, 0x00800080,
 186         0x8c24, 0xffffffff, 0x00800080,
 187         0x8c18, 0xffffffff, 0x20202078,
 188         0x8c1c, 0xffffffff, 0x00001010,
 189         0x28350, 0xffffffff, 0x00000000,
 190         0xa008, 0xffffffff, 0x00010000,
 191         0x5c4, 0xffffffff, 0x00000001,
 192         0x9508, 0xffffffff, 0x00000002,
 193         0x913c, 0x0000000f, 0x0000000a
 194 };
 195
 196 static const u32 evergreen_golden_registers2[] =
 197 {
 198         0x2f4c, 0xffffffff, 0x00000000,
 199         0x54f4, 0xffffffff, 0x00000000,
 200         0x54f0, 0xffffffff, 0x00000000,
 201         0x5498, 0xffffffff, 0x00000000,
 202         0x549c, 0xffffffff, 0x00000000,
 203         0x5494, 0xffffffff, 0x00000000,
 204         0x53cc, 0xffffffff, 0x00000000,
 205         0x53c8, 0xffffffff, 0x00000000,
 206         0x53c4, 0xffffffff, 0x00000000,
 207         0x53c0, 0xffffffff, 0x00000000,
 208         0x53bc, 0xffffffff, 0x00000000,
 209         0x53b8, 0xffffffff, 0x00000000,
 210         0x53b4, 0xffffffff, 0x00000000,
 211         0x53b0, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cypress_mgcg_init[] =
 215 {
 216         0x802c, 0xffffffff, 0xc0000000,
 217         0x5448, 0xffffffff, 0x00000100,
 218         0x55e4, 0xffffffff, 0x00000100,
 219         0x160c, 0xffffffff, 0x00000100,
 220         0x5644, 0xffffffff, 0x00000100,
 221         0xc164, 0xffffffff, 0x00000100,
 222         0x8a18, 0xffffffff, 0x00000100,
 223         0x897c, 0xffffffff, 0x06000100,
 224         0x8b28, 0xffffffff, 0x00000100,
 225         0x9144, 0xffffffff, 0x00000100,
 226         0x9a60, 0xffffffff, 0x00000100,
 227         0x9868, 0xffffffff, 0x00000100,
 228         0x8d58, 0xffffffff, 0x00000100,
 229         0x9510, 0xffffffff, 0x00000100,
 230         0x949c, 0xffffffff, 0x00000100,
 231         0x9654, 0xffffffff, 0x00000100,
 232         0x9030, 0xffffffff, 0x00000100,
 233         0x9034, 0xffffffff, 0x00000100,
 234         0x9038, 0xffffffff, 0x00000100,
 235         0x903c, 0xffffffff, 0x00000100,
 236         0x9040, 0xffffffff, 0x00000100,
 237         0xa200, 0xffffffff, 0x00000100,
 238         0xa204, 0xffffffff, 0x00000100,
 239         0xa208, 0xffffffff, 0x00000100,
 240         0xa20c, 0xffffffff, 0x00000100,
 241         0x971c, 0xffffffff, 0x00000100,
 242         0x977c, 0xffffffff, 0x00000100,
 243         0x3f80, 0xffffffff, 0x00000100,
 244         0xa210, 0xffffffff, 0x00000100,
 245         0xa214, 0xffffffff, 0x00000100,
 246         0x4d8, 0xffffffff, 0x00000100,
 247         0x9784, 0xffffffff, 0x00000100,
 248         0x9698, 0xffffffff, 0x00000100,
 249         0x4d4, 0xffffffff, 0x00000200,
 250         0x30cc, 0xffffffff, 0x00000100,
 251         0xd0c0, 0xffffffff, 0xff000100,
 252         0x802c, 0xffffffff, 0x40000000,
 253         0x915c, 0xffffffff, 0x00010000,
 254         0x9160, 0xffffffff, 0x00030002,
 255         0x9178, 0xffffffff, 0x00070000,
 256         0x917c, 0xffffffff, 0x00030002,
 257         0x9180, 0xffffffff, 0x00050004,
 258         0x918c, 0xffffffff, 0x00010006,
 259         0x9190, 0xffffffff, 0x00090008,
 260         0x9194, 0xffffffff, 0x00070000,
 261         0x9198, 0xffffffff, 0x00030002,
 262         0x919c, 0xffffffff, 0x00050004,
 263         0x91a8, 0xffffffff, 0x00010006,
 264         0x91ac, 0xffffffff, 0x00090008,
 265         0x91b0, 0xffffffff, 0x00070000,
 266         0x91b4, 0xffffffff, 0x00030002,
 267         0x91b8, 0xffffffff, 0x00050004,
 268         0x91c4, 0xffffffff, 0x00010006,
 269         0x91c8, 0xffffffff, 0x00090008,
 270         0x91cc, 0xffffffff, 0x00070000,
 271         0x91d0, 0xffffffff, 0x00030002,
 272         0x91d4, 0xffffffff, 0x00050004,
 273         0x91e0, 0xffffffff, 0x00010006,
 274         0x91e4, 0xffffffff, 0x00090008,
 275         0x91e8, 0xffffffff, 0x00000000,
 276         0x91ec, 0xffffffff, 0x00070000,
 277         0x91f0, 0xffffffff, 0x00030002,
 278         0x91f4, 0xffffffff, 0x00050004,
 279         0x9200, 0xffffffff, 0x00010006,
 280         0x9204, 0xffffffff, 0x00090008,
 281         0x9208, 0xffffffff, 0x00070000,
 282         0x920c, 0xffffffff, 0x00030002,
 283         0x9210, 0xffffffff, 0x00050004,
 284         0x921c, 0xffffffff, 0x00010006,
 285         0x9220, 0xffffffff, 0x00090008,
 286         0x9224, 0xffffffff, 0x00070000,
 287         0x9228, 0xffffffff, 0x00030002,
 288         0x922c, 0xffffffff, 0x00050004,
 289         0x9238, 0xffffffff, 0x00010006,
 290         0x923c, 0xffffffff, 0x00090008,
 291         0x9240, 0xffffffff, 0x00070000,
 292         0x9244, 0xffffffff, 0x00030002,
 293         0x9248, 0xffffffff, 0x00050004,
 294         0x9254, 0xffffffff, 0x00010006,
 295         0x9258, 0xffffffff, 0x00090008,
 296         0x925c, 0xffffffff, 0x00070000,
 297         0x9260, 0xffffffff, 0x00030002,
 298         0x9264, 0xffffffff, 0x00050004,
 299         0x9270, 0xffffffff, 0x00010006,
 300         0x9274, 0xffffffff, 0x00090008,
 301         0x9278, 0xffffffff, 0x00070000,
 302         0x927c, 0xffffffff, 0x00030002,
 303         0x9280, 0xffffffff, 0x00050004,
 304         0x928c, 0xffffffff, 0x00010006,
 305         0x9290, 0xffffffff, 0x00090008,
 306         0x9294, 0xffffffff, 0x00000000,
 307         0x929c, 0xffffffff, 0x00000001,
 308         0x802c, 0xffffffff, 0x40010000,
 309         0x915c, 0xffffffff, 0x00010000,
 310         0x9160, 0xffffffff, 0x00030002,
 311         0x9178, 0xffffffff, 0x00070000,
 312         0x917c, 0xffffffff, 0x00030002,
 313         0x9180, 0xffffffff, 0x00050004,
 314         0x918c, 0xffffffff, 0x00010006,
 315         0x9190, 0xffffffff, 0x00090008,
 316         0x9194, 0xffffffff, 0x00070000,
 317         0x9198, 0xffffffff, 0x00030002,
 318         0x919c, 0xffffffff, 0x00050004,
 319         0x91a8, 0xffffffff, 0x00010006,
 320         0x91ac, 0xffffffff, 0x00090008,
 321         0x91b0, 0xffffffff, 0x00070000,
 322         0x91b4, 0xffffffff, 0x00030002,
 323         0x91b8, 0xffffffff, 0x00050004,
 324         0x91c4, 0xffffffff, 0x00010006,
 325         0x91c8, 0xffffffff, 0x00090008,
 326         0x91cc, 0xffffffff, 0x00070000,
 327         0x91d0, 0xffffffff, 0x00030002,
 328         0x91d4, 0xffffffff, 0x00050004,
 329         0x91e0, 0xffffffff, 0x00010006,
 330         0x91e4, 0xffffffff, 0x00090008,
 331         0x91e8, 0xffffffff, 0x00000000,
 332         0x91ec, 0xffffffff, 0x00070000,
 333         0x91f0, 0xffffffff, 0x00030002,
 334         0x91f4, 0xffffffff, 0x00050004,
 335         0x9200, 0xffffffff, 0x00010006,
 336         0x9204, 0xffffffff, 0x00090008,
 337         0x9208, 0xffffffff, 0x00070000,
 338         0x920c, 0xffffffff, 0x00030002,
 339         0x9210, 0xffffffff, 0x00050004,
 340         0x921c, 0xffffffff, 0x00010006,
 341         0x9220, 0xffffffff, 0x00090008,
 342         0x9224, 0xffffffff, 0x00070000,
 343         0x9228, 0xffffffff, 0x00030002,
 344         0x922c, 0xffffffff, 0x00050004,
 345         0x9238, 0xffffffff, 0x00010006,
 346         0x923c, 0xffffffff, 0x00090008,
 347         0x9240, 0xffffffff, 0x00070000,
 348         0x9244, 0xffffffff, 0x00030002,
 349         0x9248, 0xffffffff, 0x00050004,
 350         0x9254, 0xffffffff, 0x00010006,
 351         0x9258, 0xffffffff, 0x00090008,
 352         0x925c, 0xffffffff, 0x00070000,
 353         0x9260, 0xffffffff, 0x00030002,
 354         0x9264, 0xffffffff, 0x00050004,
 355         0x9270, 0xffffffff, 0x00010006,
 356         0x9274, 0xffffffff, 0x00090008,
 357         0x9278, 0xffffffff, 0x00070000,
 358         0x927c, 0xffffffff, 0x00030002,
 359         0x9280, 0xffffffff, 0x00050004,
 360         0x928c, 0xffffffff, 0x00010006,
 361         0x9290, 0xffffffff, 0x00090008,
 362         0x9294, 0xffffffff, 0x00000000,
 363         0x929c, 0xffffffff, 0x00000001,
 364         0x802c, 0xffffffff, 0xc0000000
 365 };
 366
 367 static const u32 redwood_mgcg_init[] =
 368 {
 369         0x802c, 0xffffffff, 0xc0000000,
 370         0x5448, 0xffffffff, 0x00000100,
 371         0x55e4, 0xffffffff, 0x00000100,
 372         0x160c, 0xffffffff, 0x00000100,
 373         0x5644, 0xffffffff, 0x00000100,
 374         0xc164, 0xffffffff, 0x00000100,
 375         0x8a18, 0xffffffff, 0x00000100,
 376         0x897c, 0xffffffff, 0x06000100,
 377         0x8b28, 0xffffffff, 0x00000100,
 378         0x9144, 0xffffffff, 0x00000100,
 379         0x9a60, 0xffffffff, 0x00000100,
 380         0x9868, 0xffffffff, 0x00000100,
 381         0x8d58, 0xffffffff, 0x00000100,
 382         0x9510, 0xffffffff, 0x00000100,
 383         0x949c, 0xffffffff, 0x00000100,
 384         0x9654, 0xffffffff, 0x00000100,
 385         0x9030, 0xffffffff, 0x00000100,
 386         0x9034, 0xffffffff, 0x00000100,
 387         0x9038, 0xffffffff, 0x00000100,
 388         0x903c, 0xffffffff, 0x00000100,
 389         0x9040, 0xffffffff, 0x00000100,
 390         0xa200, 0xffffffff, 0x00000100,
 391         0xa204, 0xffffffff, 0x00000100,
 392         0xa208, 0xffffffff, 0x00000100,
 393         0xa20c, 0xffffffff, 0x00000100,
 394         0x971c, 0xffffffff, 0x00000100,
 395         0x977c, 0xffffffff, 0x00000100,
 396         0x3f80, 0xffffffff, 0x00000100,
 397         0xa210, 0xffffffff, 0x00000100,
 398         0xa214, 0xffffffff, 0x00000100,
 399         0x4d8, 0xffffffff, 0x00000100,
 400         0x9784, 0xffffffff, 0x00000100,
 401         0x9698, 0xffffffff, 0x00000100,
 402         0x4d4, 0xffffffff, 0x00000200,
 403         0x30cc, 0xffffffff, 0x00000100,
 404         0xd0c0, 0xffffffff, 0xff000100,
 405         0x802c, 0xffffffff, 0x40000000,
 406         0x915c, 0xffffffff, 0x00010000,
 407         0x9160, 0xffffffff, 0x00030002,
 408         0x9178, 0xffffffff, 0x00070000,
 409         0x917c, 0xffffffff, 0x00030002,
 410         0x9180, 0xffffffff, 0x00050004,
 411         0x918c, 0xffffffff, 0x00010006,
 412         0x9190, 0xffffffff, 0x00090008,
 413         0x9194, 0xffffffff, 0x00070000,
 414         0x9198, 0xffffffff, 0x00030002,
 415         0x919c, 0xffffffff, 0x00050004,
 416         0x91a8, 0xffffffff, 0x00010006,
 417         0x91ac, 0xffffffff, 0x00090008,
 418         0x91b0, 0xffffffff, 0x00070000,
 419         0x91b4, 0xffffffff, 0x00030002,
 420         0x91b8, 0xffffffff, 0x00050004,
 421         0x91c4, 0xffffffff, 0x00010006,
 422         0x91c8, 0xffffffff, 0x00090008,
 423         0x91cc, 0xffffffff, 0x00070000,
 424         0x91d0, 0xffffffff, 0x00030002,
 425         0x91d4, 0xffffffff, 0x00050004,
 426         0x91e0, 0xffffffff, 0x00010006,
 427         0x91e4, 0xffffffff, 0x00090008,
 428         0x91e8, 0xffffffff, 0x00000000,
 429         0x91ec, 0xffffffff, 0x00070000,
 430         0x91f0, 0xffffffff, 0x00030002,
 431         0x91f4, 0xffffffff, 0x00050004,
 432         0x9200, 0xffffffff, 0x00010006,
 433         0x9204, 0xffffffff, 0x00090008,
 434         0x9294, 0xffffffff, 0x00000000,
 435         0x929c, 0xffffffff, 0x00000001,
 436         0x802c, 0xffffffff, 0xc0000000
 437 };
 438
 439 static const u32 cedar_golden_registers[] =
 440 {
 441         0x3f90, 0xffff0000, 0xff000000,
 442         0x9148, 0xffff0000, 0xff000000,
 443         0x3f94, 0xffff0000, 0xff000000,
 444         0x914c, 0xffff0000, 0xff000000,
 445         0x9b7c, 0xffffffff, 0x00000000,
 446         0x8a14, 0xffffffff, 0x00000007,
 447         0x8b10, 0xffffffff, 0x00000000,
 448         0x960c, 0xffffffff, 0x54763210,
 449         0x88c4, 0xffffffff, 0x000000c2,
 450         0x88d4, 0xffffffff, 0x00000000,
 451         0x8974, 0xffffffff, 0x00000000,
 452         0xc78, 0x00000080, 0x00000080,
 453         0x5eb4, 0xffffffff, 0x00000002,
 454         0x5e78, 0xffffffff, 0x001000f0,
 455         0x6104, 0x01000300, 0x00000000,
 456         0x5bc0, 0x00300000, 0x00000000,
 457         0x7030, 0xffffffff, 0x00000011,
 458         0x7c30, 0xffffffff, 0x00000011,
 459         0x10830, 0xffffffff, 0x00000011,
 460         0x11430, 0xffffffff, 0x00000011,
 461         0xd02c, 0xffffffff, 0x08421000,
 462         0x240c, 0xffffffff, 0x00000380,
 463         0x8b24, 0xffffffff, 0x00ff0fff,
 464         0x28a4c, 0x06000000, 0x06000000,
 465         0x10c, 0x00000001, 0x00000001,
 466         0x8d00, 0xffffffff, 0x100e4848,
 467         0x8d04, 0xffffffff, 0x00164745,
 468         0x8c00, 0xffffffff, 0xe4000003,
 469         0x8c04, 0xffffffff, 0x40600060,
 470         0x8c08, 0xffffffff, 0x001c001c,
 471         0x8cf0, 0xffffffff, 0x08e00410,
 472         0x8c20, 0xffffffff, 0x00800080,
 473         0x8c24, 0xffffffff, 0x00800080,
 474         0x8c18, 0xffffffff, 0x20202078,
 475         0x8c1c, 0xffffffff, 0x00001010,
 476         0x28350, 0xffffffff, 0x00000000,
 477         0xa008, 0xffffffff, 0x00010000,
 478         0x5c4, 0xffffffff, 0x00000001,
 479         0x9508, 0xffffffff, 0x00000002
 480 };
 481
 482 static const u32 cedar_mgcg_init[] =
 483 {
 484         0x802c, 0xffffffff, 0xc0000000,
 485         0x5448, 0xffffffff, 0x00000100,
 486         0x55e4, 0xffffffff, 0x00000100,
 487         0x160c, 0xffffffff, 0x00000100,
 488         0x5644, 0xffffffff, 0x00000100,
 489         0xc164, 0xffffffff, 0x00000100,
 490         0x8a18, 0xffffffff, 0x00000100,
 491         0x897c, 0xffffffff, 0x06000100,
 492         0x8b28, 0xffffffff, 0x00000100,
 493         0x9144, 0xffffffff, 0x00000100,
 494         0x9a60, 0xffffffff, 0x00000100,
 495         0x9868, 0xffffffff, 0x00000100,
 496         0x8d58, 0xffffffff, 0x00000100,
 497         0x9510, 0xffffffff, 0x00000100,
 498         0x949c, 0xffffffff, 0x00000100,
 499         0x9654, 0xffffffff, 0x00000100,
 500         0x9030, 0xffffffff, 0x00000100,
 501         0x9034, 0xffffffff, 0x00000100,
 502         0x9038, 0xffffffff, 0x00000100,
 503         0x903c, 0xffffffff, 0x00000100,
 504         0x9040, 0xffffffff, 0x00000100,
 505         0xa200, 0xffffffff, 0x00000100,
 506         0xa204, 0xffffffff, 0x00000100,
 507         0xa208, 0xffffffff, 0x00000100,
 508         0xa20c, 0xffffffff, 0x00000100,
 509         0x971c, 0xffffffff, 0x00000100,
 510         0x977c, 0xffffffff, 0x00000100,
 511         0x3f80, 0xffffffff, 0x00000100,
 512         0xa210, 0xffffffff, 0x00000100,
 513         0xa214, 0xffffffff, 0x00000100,
 514         0x4d8, 0xffffffff, 0x00000100,
 515         0x9784, 0xffffffff, 0x00000100,
 516         0x9698, 0xffffffff, 0x00000100,
 517         0x4d4, 0xffffffff, 0x00000200,
 518         0x30cc, 0xffffffff, 0x00000100,
 519         0xd0c0, 0xffffffff, 0xff000100,
 520         0x802c, 0xffffffff, 0x40000000,
 521         0x915c, 0xffffffff, 0x00010000,
 522         0x9178, 0xffffffff, 0x00050000,
 523         0x917c, 0xffffffff, 0x00030002,
 524         0x918c, 0xffffffff, 0x00010004,
 525         0x9190, 0xffffffff, 0x00070006,
 526         0x9194, 0xffffffff, 0x00050000,
 527         0x9198, 0xffffffff, 0x00030002,
 528         0x91a8, 0xffffffff, 0x00010004,
 529         0x91ac, 0xffffffff, 0x00070006,
 530         0x91e8, 0xffffffff, 0x00000000,
 531         0x9294, 0xffffffff, 0x00000000,
 532         0x929c, 0xffffffff, 0x00000001,
 533         0x802c, 0xffffffff, 0xc0000000
 534 };
 535
 536 static const u32 juniper_mgcg_init[] =
 537 {
 538         0x802c, 0xffffffff, 0xc0000000,
 539         0x5448, 0xffffffff, 0x00000100,
 540         0x55e4, 0xffffffff, 0x00000100,
 541         0x160c, 0xffffffff, 0x00000100,
 542         0x5644, 0xffffffff, 0x00000100,
 543         0xc164, 0xffffffff, 0x00000100,
 544         0x8a18, 0xffffffff, 0x00000100,
 545         0x897c, 0xffffffff, 0x06000100,
 546         0x8b28, 0xffffffff, 0x00000100,
 547         0x9144, 0xffffffff, 0x00000100,
 548         0x9a60, 0xffffffff, 0x00000100,
 549         0x9868, 0xffffffff, 0x00000100,
 550         0x8d58, 0xffffffff, 0x00000100,
 551         0x9510, 0xffffffff, 0x00000100,
 552         0x949c, 0xffffffff, 0x00000100,
 553         0x9654, 0xffffffff, 0x00000100,
 554         0x9030, 0xffffffff, 0x00000100,
 555         0x9034, 0xffffffff, 0x00000100,
 556         0x9038, 0xffffffff, 0x00000100,
 557         0x903c, 0xffffffff, 0x00000100,
 558         0x9040, 0xffffffff, 0x00000100,
 559         0xa200, 0xffffffff, 0x00000100,
 560         0xa204, 0xffffffff, 0x00000100,
 561         0xa208, 0xffffffff, 0x00000100,
 562         0xa20c, 0xffffffff, 0x00000100,
 563         0x971c, 0xffffffff, 0x00000100,
 564         0xd0c0, 0xffffffff, 0xff000100,
 565         0x802c, 0xffffffff, 0x40000000,
 566         0x915c, 0xffffffff, 0x00010000,
 567         0x9160, 0xffffffff, 0x00030002,
 568         0x9178, 0xffffffff, 0x00070000,
 569         0x917c, 0xffffffff, 0x00030002,
 570         0x9180, 0xffffffff, 0x00050004,
 571         0x918c, 0xffffffff, 0x00010006,
 572         0x9190, 0xffffffff, 0x00090008,
 573         0x9194, 0xffffffff, 0x00070000,
 574         0x9198, 0xffffffff, 0x00030002,
 575         0x919c, 0xffffffff, 0x00050004,
 576         0x91a8, 0xffffffff, 0x00010006,
 577         0x91ac, 0xffffffff, 0x00090008,
 578         0x91b0, 0xffffffff, 0x00070000,
 579         0x91b4, 0xffffffff, 0x00030002,
 580         0x91b8, 0xffffffff, 0x00050004,
 581         0x91c4, 0xffffffff, 0x00010006,
 582         0x91c8, 0xffffffff, 0x00090008,
 583         0x91cc, 0xffffffff, 0x00070000,
 584         0x91d0, 0xffffffff, 0x00030002,
 585         0x91d4, 0xffffffff, 0x00050004,
 586         0x91e0, 0xffffffff, 0x00010006,
 587         0x91e4, 0xffffffff, 0x00090008,
 588         0x91e8, 0xffffffff, 0x00000000,
 589         0x91ec, 0xffffffff, 0x00070000,
 590         0x91f0, 0xffffffff, 0x00030002,
 591         0x91f4, 0xffffffff, 0x00050004,
 592         0x9200, 0xffffffff, 0x00010006,
 593         0x9204, 0xffffffff, 0x00090008,
 594         0x9208, 0xffffffff, 0x00070000,
 595         0x920c, 0xffffffff, 0x00030002,
 596         0x9210, 0xffffffff, 0x00050004,
 597         0x921c, 0xffffffff, 0x00010006,
 598         0x9220, 0xffffffff, 0x00090008,
 599         0x9224, 0xffffffff, 0x00070000,
 600         0x9228, 0xffffffff, 0x00030002,
 601         0x922c, 0xffffffff, 0x00050004,
 602         0x9238, 0xffffffff, 0x00010006,
 603         0x923c, 0xffffffff, 0x00090008,
 604         0x9240, 0xffffffff, 0x00070000,
 605         0x9244, 0xffffffff, 0x00030002,
 606         0x9248, 0xffffffff, 0x00050004,
 607         0x9254, 0xffffffff, 0x00010006,
 608         0x9258, 0xffffffff, 0x00090008,
 609         0x925c, 0xffffffff, 0x00070000,
 610         0x9260, 0xffffffff, 0x00030002,
 611         0x9264, 0xffffffff, 0x00050004,
 612         0x9270, 0xffffffff, 0x00010006,
 613         0x9274, 0xffffffff, 0x00090008,
 614         0x9278, 0xffffffff, 0x00070000,
 615         0x927c, 0xffffffff, 0x00030002,
 616         0x9280, 0xffffffff, 0x00050004,
 617         0x928c, 0xffffffff, 0x00010006,
 618         0x9290, 0xffffffff, 0x00090008,
 619         0x9294, 0xffffffff, 0x00000000,
 620         0x929c, 0xffffffff, 0x00000001,
 621         0x802c, 0xffffffff, 0xc0000000,
 622         0x977c, 0xffffffff, 0x00000100,
 623         0x3f80, 0xffffffff, 0x00000100,
 624         0xa210, 0xffffffff, 0x00000100,
 625         0xa214, 0xffffffff, 0x00000100,
 626         0x4d8, 0xffffffff, 0x00000100,
 627         0x9784, 0xffffffff, 0x00000100,
 628         0x9698, 0xffffffff, 0x00000100,
 629         0x4d4, 0xffffffff, 0x00000200,
 630         0x30cc, 0xffffffff, 0x00000100,
 631         0x802c, 0xffffffff, 0xc0000000
 632 };
 633
 634 static const u32 supersumo_golden_registers[] =
 635 {
 636         0x5eb4, 0xffffffff, 0x00000002,
 637         0x5c4, 0xffffffff, 0x00000001,
 638         0x7030, 0xffffffff, 0x00000011,
 639         0x7c30, 0xffffffff, 0x00000011,
 640         0x6104, 0x01000300, 0x00000000,
 641         0x5bc0, 0x00300000, 0x00000000,
 642         0x8c04, 0xffffffff, 0x40600060,
 643         0x8c08, 0xffffffff, 0x001c001c,
 644         0x8c20, 0xffffffff, 0x00800080,
 645         0x8c24, 0xffffffff, 0x00800080,
 646         0x8c18, 0xffffffff, 0x20202078,
 647         0x8c1c, 0xffffffff, 0x00001010,
 648         0x918c, 0xffffffff, 0x00010006,
 649         0x91a8, 0xffffffff, 0x00010006,
 650         0x91c4, 0xffffffff, 0x00010006,
 651         0x91e0, 0xffffffff, 0x00010006,
 652         0x9200, 0xffffffff, 0x00010006,
 653         0x9150, 0xffffffff, 0x6e944040,
 654         0x917c, 0xffffffff, 0x00030002,
 655         0x9180, 0xffffffff, 0x00050004,
 656         0x9198, 0xffffffff, 0x00030002,
 657         0x919c, 0xffffffff, 0x00050004,
 658         0x91b4, 0xffffffff, 0x00030002,
 659         0x91b8, 0xffffffff, 0x00050004,
 660         0x91d0, 0xffffffff, 0x00030002,
 661         0x91d4, 0xffffffff, 0x00050004,
 662         0x91f0, 0xffffffff, 0x00030002,
 663         0x91f4, 0xffffffff, 0x00050004,
 664         0x915c, 0xffffffff, 0x00010000,
 665         0x9160, 0xffffffff, 0x00030002,
 666         0x3f90, 0xffff0000, 0xff000000,
 667         0x9178, 0xffffffff, 0x00070000,
 668         0x9194, 0xffffffff, 0x00070000,
 669         0x91b0, 0xffffffff, 0x00070000,
 670         0x91cc, 0xffffffff, 0x00070000,
 671         0x91ec, 0xffffffff, 0x00070000,
 672         0x9148, 0xffff0000, 0xff000000,
 673         0x9190, 0xffffffff, 0x00090008,
 674         0x91ac, 0xffffffff, 0x00090008,
 675         0x91c8, 0xffffffff, 0x00090008,
 676         0x91e4, 0xffffffff, 0x00090008,
 677         0x9204, 0xffffffff, 0x00090008,
 678         0x3f94, 0xffff0000, 0xff000000,
 679         0x914c, 0xffff0000, 0xff000000,
 680         0x929c, 0xffffffff, 0x00000001,
 681         0x8a18, 0xffffffff, 0x00000100,
 682         0x8b28, 0xffffffff, 0x00000100,
 683         0x9144, 0xffffffff, 0x00000100,
 684         0x5644, 0xffffffff, 0x00000100,
 685         0x9b7c, 0xffffffff, 0x00000000,
 686         0x8030, 0xffffffff, 0x0000100a,
 687         0x8a14, 0xffffffff, 0x00000007,
 688         0x8b24, 0xffffffff, 0x00ff0fff,
 689         0x8b10, 0xffffffff, 0x00000000,
 690         0x28a4c, 0x06000000, 0x06000000,
 691         0x4d8, 0xffffffff, 0x00000100,
 692         0x913c, 0xffff000f, 0x0100000a,
 693         0x960c, 0xffffffff, 0x54763210,
 694         0x88c4, 0xffffffff, 0x000000c2,
 695         0x88d4, 0xffffffff, 0x00000010,
 696         0x8974, 0xffffffff, 0x00000000,
 697         0xc78, 0x00000080, 0x00000080,
 698         0x5e78, 0xffffffff, 0x001000f0,
 699         0xd02c, 0xffffffff, 0x08421000,
 700         0xa008, 0xffffffff, 0x00010000,
 701         0x8d00, 0xffffffff, 0x100e4848,
 702         0x8d04, 0xffffffff, 0x00164745,
 703         0x8c00, 0xffffffff, 0xe4000003,
 704         0x8cf0, 0x1fffffff, 0x08e00620,
 705         0x28350, 0xffffffff, 0x00000000,
 706         0x9508, 0xffffffff, 0x00000002
 707 };
 708
 709 static const u32 sumo_golden_registers[] =
 710 {
 711         0x900c, 0x00ffffff, 0x0017071f,
 712         0x8c18, 0xffffffff, 0x10101060,
 713         0x8c1c, 0xffffffff, 0x00001010,
 714         0x8c30, 0x0000000f, 0x00000005,
 715         0x9688, 0x0000000f, 0x00000007
 716 };
 717
 718 static const u32 wrestler_golden_registers[] =
 719 {
 720         0x5eb4, 0xffffffff, 0x00000002,
 721         0x5c4, 0xffffffff, 0x00000001,
 722         0x7030, 0xffffffff, 0x00000011,
 723         0x7c30, 0xffffffff, 0x00000011,
 724         0x6104, 0x01000300, 0x00000000,
 725         0x5bc0, 0x00300000, 0x00000000,
 726         0x918c, 0xffffffff, 0x00010006,
 727         0x91a8, 0xffffffff, 0x00010006,
 728         0x9150, 0xffffffff, 0x6e944040,
 729         0x917c, 0xffffffff, 0x00030002,
 730         0x9198, 0xffffffff, 0x00030002,
 731         0x915c, 0xffffffff, 0x00010000,
 732         0x3f90, 0xffff0000, 0xff000000,
 733         0x9178, 0xffffffff, 0x00070000,
 734         0x9194, 0xffffffff, 0x00070000,
 735         0x9148, 0xffff0000, 0xff000000,
 736         0x9190, 0xffffffff, 0x00090008,
 737         0x91ac, 0xffffffff, 0x00090008,
 738         0x3f94, 0xffff0000, 0xff000000,
 739         0x914c, 0xffff0000, 0xff000000,
 740         0x929c, 0xffffffff, 0x00000001,
 741         0x8a18, 0xffffffff, 0x00000100,
 742         0x8b28, 0xffffffff, 0x00000100,
 743         0x9144, 0xffffffff, 0x00000100,
 744         0x9b7c, 0xffffffff, 0x00000000,
 745         0x8030, 0xffffffff, 0x0000100a,
 746         0x8a14, 0xffffffff, 0x00000001,
 747         0x8b24, 0xffffffff, 0x00ff0fff,
 748         0x8b10, 0xffffffff, 0x00000000,
 749         0x28a4c, 0x06000000, 0x06000000,
 750         0x4d8, 0xffffffff, 0x00000100,
 751         0x913c, 0xffff000f, 0x0100000a,
 752         0x960c, 0xffffffff, 0x54763210,
 753         0x88c4, 0xffffffff, 0x000000c2,
 754         0x88d4, 0xffffffff, 0x00000010,
 755         0x8974, 0xffffffff, 0x00000000,
 756         0xc78, 0x00000080, 0x00000080,
 757         0x5e78, 0xffffffff, 0x001000f0,
 758         0xd02c, 0xffffffff, 0x08421000,
 759         0xa008, 0xffffffff, 0x00010000,
 760         0x8d00, 0xffffffff, 0x100e4848,
 761         0x8d04, 0xffffffff, 0x00164745,
 762         0x8c00, 0xffffffff, 0xe4000003,
 763         0x8cf0, 0x1fffffff, 0x08e00410,
 764         0x28350, 0xffffffff, 0x00000000,
 765         0x9508, 0xffffffff, 0x00000002,
 766         0x900c, 0xffffffff, 0x0017071f,
 767         0x8c18, 0xffffffff, 0x10101060,
 768         0x8c1c, 0xffffffff, 0x00001010
 769 };
 770
 771 static const u32 barts_golden_registers[] =
 772 {
 773         0x5eb4, 0xffffffff, 0x00000002,
 774         0x5e78, 0x8f311ff1, 0x001000f0,
 775         0x3f90, 0xffff0000, 0xff000000,
 776         0x9148, 0xffff0000, 0xff000000,
 777         0x3f94, 0xffff0000, 0xff000000,
 778         0x914c, 0xffff0000, 0xff000000,
 779         0xc78, 0x00000080, 0x00000080,
 780         0xbd4, 0x70073777, 0x00010001,
 781         0xd02c, 0xbfffff1f, 0x08421000,
 782         0xd0b8, 0x03773777, 0x02011003,
 783         0x5bc0, 0x00200000, 0x50100000,
 784         0x98f8, 0x33773777, 0x02011003,
 785         0x98fc, 0xffffffff, 0x76543210,
 786         0x7030, 0x31000311, 0x00000011,
 787         0x2f48, 0x00000007, 0x02011003,
 788         0x6b28, 0x00000010, 0x00000012,
 789         0x7728, 0x00000010, 0x00000012,
 790         0x10328, 0x00000010, 0x00000012,
 791         0x10f28, 0x00000010, 0x00000012,
 792         0x11b28, 0x00000010, 0x00000012,
 793         0x12728, 0x00000010, 0x00000012,
 794         0x240c, 0x000007ff, 0x00000380,
 795         0x8a14, 0xf000001f, 0x00000007,
 796         0x8b24, 0x3fff3fff, 0x00ff0fff,
 797         0x8b10, 0x0000ff0f, 0x00000000,
 798         0x28a4c, 0x07ffffff, 0x06000000,
 799         0x10c, 0x00000001, 0x00010003,
 800         0xa02c, 0xffffffff, 0x0000009b,
 801         0x913c, 0x0000000f, 0x0100000a,
 802         0x8d00, 0xffff7f7f, 0x100e4848,
 803         0x8d04, 0x00ffffff, 0x00164745,
 804         0x8c00, 0xfffc0003, 0xe4000003,
 805         0x8c04, 0xf8ff00ff, 0x40600060,
 806         0x8c08, 0x00ff00ff, 0x001c001c,
 807         0x8cf0, 0x1fff1fff, 0x08e00620,
 808         0x8c20, 0x0fff0fff, 0x00800080,
 809         0x8c24, 0x0fff0fff, 0x00800080,
 810         0x8c18, 0xffffffff, 0x20202078,
 811         0x8c1c, 0x0000ffff, 0x00001010,
 812         0x28350, 0x00000f01, 0x00000000,
 813         0x9508, 0x3700001f, 0x00000002,
 814         0x960c, 0xffffffff, 0x54763210,
 815         0x88c4, 0x001f3ae3, 0x000000c2,
 816         0x88d4, 0x0000001f, 0x00000010,
 817         0x8974, 0xffffffff, 0x00000000
 818 };
 819
 820 static const u32 turks_golden_registers[] =
 821 {
 822         0x5eb4, 0xffffffff, 0x00000002,
 823         0x5e78, 0x8f311ff1, 0x001000f0,
 824         0x8c8, 0x00003000, 0x00001070,
 825         0x8cc, 0x000fffff, 0x00040035,
 826         0x3f90, 0xffff0000, 0xfff00000,
 827         0x9148, 0xffff0000, 0xfff00000,
 828         0x3f94, 0xffff0000, 0xfff00000,
 829         0x914c, 0xffff0000, 0xfff00000,
 830         0xc78, 0x00000080, 0x00000080,
 831         0xbd4, 0x00073007, 0x00010002,
 832         0xd02c, 0xbfffff1f, 0x08421000,
 833         0xd0b8, 0x03773777, 0x02010002,
 834         0x5bc0, 0x00200000, 0x50100000,
 835         0x98f8, 0x33773777, 0x00010002,
 836         0x98fc, 0xffffffff, 0x33221100,
 837         0x7030, 0x31000311, 0x00000011,
 838         0x2f48, 0x33773777, 0x00010002,
 839         0x6b28, 0x00000010, 0x00000012,
 840         0x7728, 0x00000010, 0x00000012,
 841         0x10328, 0x00000010, 0x00000012,
 842         0x10f28, 0x00000010, 0x00000012,
 843         0x11b28, 0x00000010, 0x00000012,
 844         0x12728, 0x00000010, 0x00000012,
 845         0x240c, 0x000007ff, 0x00000380,
 846         0x8a14, 0xf000001f, 0x00000007,
 847         0x8b24, 0x3fff3fff, 0x00ff0fff,
 848         0x8b10, 0x0000ff0f, 0x00000000,
 849         0x28a4c, 0x07ffffff, 0x06000000,
 850         0x10c, 0x00000001, 0x00010003,
 851         0xa02c, 0xffffffff, 0x0000009b,
 852         0x913c, 0x0000000f, 0x0100000a,
 853         0x8d00, 0xffff7f7f, 0x100e4848,
 854         0x8d04, 0x00ffffff, 0x00164745,
 855         0x8c00, 0xfffc0003, 0xe4000003,
 856         0x8c04, 0xf8ff00ff, 0x40600060,
 857         0x8c08, 0x00ff00ff, 0x001c001c,
 858         0x8cf0, 0x1fff1fff, 0x08e00410,
 859         0x8c20, 0x0fff0fff, 0x00800080,
 860         0x8c24, 0x0fff0fff, 0x00800080,
 861         0x8c18, 0xffffffff, 0x20202078,
 862         0x8c1c, 0x0000ffff, 0x00001010,
 863         0x28350, 0x00000f01, 0x00000000,
 864         0x9508, 0x3700001f, 0x00000002,
 865         0x960c, 0xffffffff, 0x54763210,
 866         0x88c4, 0x001f3ae3, 0x000000c2,
 867         0x88d4, 0x0000001f, 0x00000010,
 868         0x8974, 0xffffffff, 0x00000000
 869 };
 870
 871 static const u32 caicos_golden_registers[] =
 872 {
 873         0x5eb4, 0xffffffff, 0x00000002,
 874         0x5e78, 0x8f311ff1, 0x001000f0,
 875         0x8c8, 0x00003420, 0x00001450,
 876         0x8cc, 0x000fffff, 0x00040035,
 877         0x3f90, 0xffff0000, 0xfffc0000,
 878         0x9148, 0xffff0000, 0xfffc0000,
 879         0x3f94, 0xffff0000, 0xfffc0000,
 880         0x914c, 0xffff0000, 0xfffc0000,
 881         0xc78, 0x00000080, 0x00000080,
 882         0xbd4, 0x00073007, 0x00010001,
 883         0xd02c, 0xbfffff1f, 0x08421000,
 884         0xd0b8, 0x03773777, 0x02010001,
 885         0x5bc0, 0x00200000, 0x50100000,
 886         0x98f8, 0x33773777, 0x02010001,
 887         0x98fc, 0xffffffff, 0x33221100,
 888         0x7030, 0x31000311, 0x00000011,
 889         0x2f48, 0x33773777, 0x02010001,
 890         0x6b28, 0x00000010, 0x00000012,
 891         0x7728, 0x00000010, 0x00000012,
 892         0x10328, 0x00000010, 0x00000012,
 893         0x10f28, 0x00000010, 0x00000012,
 894         0x11b28, 0x00000010, 0x00000012,
 895         0x12728, 0x00000010, 0x00000012,
 896         0x240c, 0x000007ff, 0x00000380,
 897         0x8a14, 0xf000001f, 0x00000001,
 898         0x8b24, 0x3fff3fff, 0x00ff0fff,
 899         0x8b10, 0x0000ff0f, 0x00000000,
 900         0x28a4c, 0x07ffffff, 0x06000000,
 901         0x10c, 0x00000001, 0x00010003,
 902         0xa02c, 0xffffffff, 0x0000009b,
 903         0x913c, 0x0000000f, 0x0100000a,
 904         0x8d00, 0xffff7f7f, 0x100e4848,
 905         0x8d04, 0x00ffffff, 0x00164745,
 906         0x8c00, 0xfffc0003, 0xe4000003,
 907         0x8c04, 0xf8ff00ff, 0x40600060,
 908         0x8c08, 0x00ff00ff, 0x001c001c,
 909         0x8cf0, 0x1fff1fff, 0x08e00410,
 910         0x8c20, 0x0fff0fff, 0x00800080,
 911         0x8c24, 0x0fff0fff, 0x00800080,
 912         0x8c18, 0xffffffff, 0x20202078,
 913         0x8c1c, 0x0000ffff, 0x00001010,
 914         0x28350, 0x00000f01, 0x00000000,
 915         0x9508, 0x3700001f, 0x00000002,
 916         0x960c, 0xffffffff, 0x54763210,
 917         0x88c4, 0x001f3ae3, 0x000000c2,
 918         0x88d4, 0x0000001f, 0x00000010,
 919         0x8974, 0xffffffff, 0x00000000
 920 };
 921
 922 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 923 {
 924         switch (rdev->family) {
 925         case CHIP_CYPRESS:
 926         case CHIP_HEMLOCK:
 927                 radeon_program_register_sequence(rdev,
 928                                                  evergreen_golden_registers,
 929                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 930                 radeon_program_register_sequence(rdev,
 931                                                  evergreen_golden_registers2,
 932                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 933                 radeon_program_register_sequence(rdev,
 934                                                  cypress_mgcg_init,
 935                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 936                 break;
 937         case CHIP_JUNIPER:
 938                 radeon_program_register_sequence(rdev,
 939                                                  evergreen_golden_registers,
 940                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 941                 radeon_program_register_sequence(rdev,
 942                                                  evergreen_golden_registers2,
 943                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 944                 radeon_program_register_sequence(rdev,
 945                                                  juniper_mgcg_init,
 946                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
 947                 break;
 948         case CHIP_REDWOOD:
 949                 radeon_program_register_sequence(rdev,
 950                                                  evergreen_golden_registers,
 951                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 952                 radeon_program_register_sequence(rdev,
 953                                                  evergreen_golden_registers2,
 954                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 955                 radeon_program_register_sequence(rdev,
 956                                                  redwood_mgcg_init,
 957                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
 958                 break;
 959         case CHIP_CEDAR:
 960                 radeon_program_register_sequence(rdev,
 961                                                  cedar_golden_registers,
 962                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
 963                 radeon_program_register_sequence(rdev,
 964                                                  evergreen_golden_registers2,
 965                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 966                 radeon_program_register_sequence(rdev,
 967                                                  cedar_mgcg_init,
 968                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
 969                 break;
 970         case CHIP_PALM:
 971                 radeon_program_register_sequence(rdev,
 972                                                  wrestler_golden_registers,
 973                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
 974                 break;
 975         case CHIP_SUMO:
 976                 radeon_program_register_sequence(rdev,
 977                                                  supersumo_golden_registers,
 978                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 979                 break;
 980         case CHIP_SUMO2:
 981                 radeon_program_register_sequence(rdev,
 982                                                  supersumo_golden_registers,
 983                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 984                 radeon_program_register_sequence(rdev,
 985                                                  sumo_golden_registers,
 986                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
 987                 break;
 988         case CHIP_BARTS:
 989                 radeon_program_register_sequence(rdev,
 990                                                  barts_golden_registers,
 991                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
 992                 break;
 993         case CHIP_TURKS:
 994                 radeon_program_register_sequence(rdev,
 995                                                  turks_golden_registers,
 996                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
 997                 break;
 998         case CHIP_CAICOS:
 999                 radeon_program_register_sequence(rdev,
1000                                                  caicos_golden_registers,
1001                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
1002                 break;
1003         default:
1004                 break;
1005         }
1006 }
1007
1008 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1009                              unsigned *bankh, unsigned *mtaspect,
1010                              unsigned *tile_split)
1011 {
1012         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1013         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1014         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1015         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1016         switch (*bankw) {
1017         default:
1018         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1019         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1020         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1021         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1022         }
1023         switch (*bankh) {
1024         default:
1025         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1026         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1027         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1028         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1029         }
1030         switch (*mtaspect) {
1031         default:
1032         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1033         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1034         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1035         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1036         }
1037 }
1038
1039 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1040                               u32 cntl_reg, u32 status_reg)
1041 {
1042         int r, i;
1043         struct atom_clock_dividers dividers;
1044
1045         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1046                                            clock, false, &dividers);
1047         if (r)
1048                 return r;
1049
1050         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1051
1052         for (i = 0; i < 100; i++) {
1053                 if (RREG32(status_reg) & DCLK_STATUS)
1054                         break;
1055                 mdelay(10);
1056         }
1057         if (i == 100)
1058                 return -ETIMEDOUT;
1059
1060         return 0;
1061 }
1062
1063 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1064 {
1065         int r = 0;
1066         u32 cg_scratch = RREG32(CG_SCRATCH1);
1067
1068         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1069         if (r)
1070                 goto done;
1071         cg_scratch &= 0xffff0000;
1072         cg_scratch |= vclk / 100; /* Mhz */
1073
1074         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1075         if (r)
1076                 goto done;
1077         cg_scratch &= 0x0000ffff;
1078         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1079
1080 done:
1081         WREG32(CG_SCRATCH1, cg_scratch);
1082
1083         return r;
1084 }
1085
1086 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1087 {
1088         /* start off with something large */
1089         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1090         int r;
1091
1092         /* bypass vclk and dclk with bclk */
1093         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1094                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1095                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1096
1097         /* put PLL in bypass mode */
1098         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1099
1100         if (!vclk || !dclk) {
1101                 /* keep the Bypass mode, put PLL to sleep */
1102                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1103                 return 0;
1104         }
1105
1106         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1107                                           16384, 0x03FFFFFF, 0, 128, 5,
1108                                           &fb_div, &vclk_div, &dclk_div);
1109         if (r)
1110                 return r;
1111
1112         /* set VCO_MODE to 1 */
1113         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1114
1115         /* toggle UPLL_SLEEP to 1 then back to 0 */
1116         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1117         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1118
1119         /* deassert UPLL_RESET */
1120         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1121
1122         mdelay(1);
1123
1124         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1125         if (r)
1126                 return r;
1127
1128         /* assert UPLL_RESET again */
1129         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1130
1131         /* disable spread spectrum. */
1132         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1133
1134         /* set feedback divider */
1135         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1136
1137         /* set ref divider to 0 */
1138         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1139
1140         if (fb_div < 307200)
1141                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1142         else
1143                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1144
1145         /* set PDIV_A and PDIV_B */
1146         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1147                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1148                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1149
1150         /* give the PLL some time to settle */
1151         mdelay(15);
1152
1153         /* deassert PLL_RESET */
1154         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1155
1156         mdelay(15);
1157
1158         /* switch from bypass mode to normal mode */
1159         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1160
1161         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1162         if (r)
1163                 return r;
1164
1165         /* switch VCLK and DCLK selection */
1166         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1167                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1168                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1169
1170         mdelay(100);
1171
1172         return 0;
1173 }
1174
1175 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1176 {
1177         int readrq;
1178         u16 v;
1179
1180         readrq = pcie_get_readrq(rdev->pdev);
1181         v = ffs(readrq) - 8;
1182         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1183          * to avoid hangs or perfomance issues
1184          */
1185         if ((v == 0) || (v == 6) || (v == 7))
1186                 pcie_set_readrq(rdev->pdev, 512);
1187 }
1188
1189 void dce4_program_fmt(struct drm_encoder *encoder)
1190 {
1191         struct drm_device *dev = encoder->dev;
1192         struct radeon_device *rdev = dev->dev_private;
1193         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1194         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1195         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1196         int bpc = 0;
1197         u32 tmp = 0;
1198         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1199
1200         if (connector) {
1201                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1202                 bpc = radeon_get_monitor_bpc(connector);
1203                 dither = radeon_connector->dither;
1204         }
1205
1206         /* LVDS/eDP FMT is set up by atom */
1207         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1208                 return;
1209
1210         /* not needed for analog */
1211         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1212             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1213                 return;
1214
1215         if (bpc == 0)
1216                 return;
1217
1218         switch (bpc) {
1219         case 6:
1220                 if (dither == RADEON_FMT_DITHER_ENABLE)
1221                         /* XXX sort out optimal dither settings */
1222                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1223                                 FMT_SPATIAL_DITHER_EN);
1224                 else
1225                         tmp |= FMT_TRUNCATE_EN;
1226                 break;
1227         case 8:
1228                 if (dither == RADEON_FMT_DITHER_ENABLE)
1229                         /* XXX sort out optimal dither settings */
1230                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1231                                 FMT_RGB_RANDOM_ENABLE |
1232                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1233                 else
1234                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1235                 break;
1236         case 10:
1237         default:
1238                 /* not needed */
1239                 break;
1240         }
1241
1242         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1243 }
1244
1245 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1246 {
1247         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1248                 return true;
1249         else
1250                 return false;
1251 }
1252
1253 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1254 {
1255         u32 pos1, pos2;
1256
1257         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1258         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259
1260         if (pos1 != pos2)
1261                 return true;
1262         else
1263                 return false;
1264 }
1265
1266 /**
1267  * dce4_wait_for_vblank - vblank wait asic callback.
1268  *
1269  * @rdev: radeon_device pointer
1270  * @crtc: crtc to wait for vblank on
1271  *
1272  * Wait for vblank on the requested crtc (evergreen+).
1273  */
1274 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1275 {
1276         unsigned i = 0;
1277
1278         if (crtc >= rdev->num_crtc)
1279                 return;
1280
1281         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1282                 return;
1283
1284         /* depending on when we hit vblank, we may be close to active; if so,
1285          * wait for another frame.
1286          */
1287         while (dce4_is_in_vblank(rdev, crtc)) {
1288                 if (i++ % 100 == 0) {
1289                         if (!dce4_is_counter_moving(rdev, crtc))
1290                                 break;
1291                 }
1292         }
1293
1294         while (!dce4_is_in_vblank(rdev, crtc)) {
1295                 if (i++ % 100 == 0) {
1296                         if (!dce4_is_counter_moving(rdev, crtc))
1297                                 break;
1298                 }
1299         }
1300 }
1301
1302 /**
1303  * evergreen_page_flip - pageflip callback.
1304  *
1305  * @rdev: radeon_device pointer
1306  * @crtc_id: crtc to cleanup pageflip on
1307  * @crtc_base: new address of the crtc (GPU MC address)
1308  *
1309  * Does the actual pageflip (evergreen+).
1310  * During vblank we take the crtc lock and wait for the update_pending
1311  * bit to go high, when it does, we release the lock, and allow the
1312  * double buffered update to take place.
1313  * Returns the current update pending status.
1314  */
1315 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1316 {
1317         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1318         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1319         int i;
1320
1321         /* Lock the graphics update lock */
1322         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1323         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1324
1325         /* update the scanout addresses */
1326         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1327                upper_32_bits(crtc_base));
1328         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1329                (u32)crtc_base);
1330
1331         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1332                upper_32_bits(crtc_base));
1333         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1334                (u32)crtc_base);
1335
1336         /* Wait for update_pending to go high. */
1337         for (i = 0; i < rdev->usec_timeout; i++) {
1338                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1339                         break;
1340                 udelay(1);
1341         }
1342         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1343
1344         /* Unlock the lock, so double-buffering can take place inside vblank */
1345         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1346         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1347 }
1348
1349 /**
1350  * evergreen_page_flip_pending - check if page flip is still pending
1351  *
1352  * @rdev: radeon_device pointer
1353  * @crtc_id: crtc to check
1354  *
1355  * Returns the current update pending status.
1356  */
1357 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1358 {
1359         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1360
1361         /* Return current update_pending status: */
1362         return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1363                 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1364 }
1365
1366 /* get temperature in millidegrees */
1367 int evergreen_get_temp(struct radeon_device *rdev)
1368 {
1369         u32 temp, toffset;
1370         int actual_temp = 0;
1371
1372         if (rdev->family == CHIP_JUNIPER) {
1373                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1374                         TOFFSET_SHIFT;
1375                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1376                         TS0_ADC_DOUT_SHIFT;
1377
1378                 if (toffset & 0x100)
1379                         actual_temp = temp / 2 - (0x200 - toffset);
1380                 else
1381                         actual_temp = temp / 2 + toffset;
1382
1383                 actual_temp = actual_temp * 1000;
1384
1385         } else {
1386                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1387                         ASIC_T_SHIFT;
1388
1389                 if (temp & 0x400)
1390                         actual_temp = -256;
1391                 else if (temp & 0x200)
1392                         actual_temp = 255;
1393                 else if (temp & 0x100) {
1394                         actual_temp = temp & 0x1ff;
1395                         actual_temp |= ~0x1ff;
1396                 } else
1397                         actual_temp = temp & 0xff;
1398
1399                 actual_temp = (actual_temp * 1000) / 2;
1400         }
1401
1402         return actual_temp;
1403 }
1404
1405 int sumo_get_temp(struct radeon_device *rdev)
1406 {
1407         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1408         int actual_temp = temp - 49;
1409
1410         return actual_temp * 1000;
1411 }
1412
1413 /**
1414  * sumo_pm_init_profile - Initialize power profiles callback.
1415  *
1416  * @rdev: radeon_device pointer
1417  *
1418  * Initialize the power states used in profile mode
1419  * (sumo, trinity, SI).
1420  * Used for profile mode only.
1421  */
1422 void sumo_pm_init_profile(struct radeon_device *rdev)
1423 {
1424         int idx;
1425
1426         /* default */
1427         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1428         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1429         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1430         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1431
1432         /* low,mid sh/mh */
1433         if (rdev->flags & RADEON_IS_MOBILITY)
1434                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1435         else
1436                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1437
1438         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1439         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1440         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1441         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1442
1443         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1444         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1445         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1446         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1447
1448         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1449         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1450         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1451         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1452
1453         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1454         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1455         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1456         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1457
1458         /* high sh/mh */
1459         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1460         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1461         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1462         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1463         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1464                 rdev->pm.power_state[idx].num_clock_modes - 1;
1465
1466         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1467         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1468         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1469         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1470                 rdev->pm.power_state[idx].num_clock_modes - 1;
1471 }
1472
1473 /**
1474  * btc_pm_init_profile - Initialize power profiles callback.
1475  *
1476  * @rdev: radeon_device pointer
1477  *
1478  * Initialize the power states used in profile mode
1479  * (BTC, cayman).
1480  * Used for profile mode only.
1481  */
1482 void btc_pm_init_profile(struct radeon_device *rdev)
1483 {
1484         int idx;
1485
1486         /* default */
1487         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1488         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1489         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1490         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1491         /* starting with BTC, there is one state that is used for both
1492          * MH and SH.  Difference is that we always use the high clock index for
1493          * mclk.
1494          */
1495         if (rdev->flags & RADEON_IS_MOBILITY)
1496                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1497         else
1498                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1499         /* low sh */
1500         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1501         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1502         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1503         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1504         /* mid sh */
1505         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1506         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1507         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1508         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1509         /* high sh */
1510         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1511         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1512         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1513         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1514         /* low mh */
1515         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1516         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1517         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1518         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1519         /* mid mh */
1520         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1521         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1522         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1523         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1524         /* high mh */
1525         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1526         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1527         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1528         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1529 }
1530
1531 /**
1532  * evergreen_pm_misc - set additional pm hw parameters callback.
1533  *
1534  * @rdev: radeon_device pointer
1535  *
1536  * Set non-clock parameters associated with a power state
1537  * (voltage, etc.) (evergreen+).
1538  */
1539 void evergreen_pm_misc(struct radeon_device *rdev)
1540 {
1541         int req_ps_idx = rdev->pm.requested_power_state_index;
1542         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1543         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1544         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1545
1546         if (voltage->type == VOLTAGE_SW) {
1547                 /* 0xff0x are flags rather then an actual voltage */
1548                 if ((voltage->voltage & 0xff00) == 0xff00)
1549                         return;
1550                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1551                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1552                         rdev->pm.current_vddc = voltage->voltage;
1553                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1554                 }
1555
1556                 /* starting with BTC, there is one state that is used for both
1557                  * MH and SH.  Difference is that we always use the high clock index for
1558                  * mclk and vddci.
1559                  */
1560                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1561                     (rdev->family >= CHIP_BARTS) &&
1562                     rdev->pm.active_crtc_count &&
1563                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1564                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1565                         voltage = &rdev->pm.power_state[req_ps_idx].
1566                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1567
1568                 /* 0xff0x are flags rather then an actual voltage */
1569                 if ((voltage->vddci & 0xff00) == 0xff00)
1570                         return;
1571                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1572                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1573                         rdev->pm.current_vddci = voltage->vddci;
1574                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1575                 }
1576         }
1577 }
1578
1579 /**
1580  * evergreen_pm_prepare - pre-power state change callback.
1581  *
1582  * @rdev: radeon_device pointer
1583  *
1584  * Prepare for a power state change (evergreen+).
1585  */
1586 void evergreen_pm_prepare(struct radeon_device *rdev)
1587 {
1588         struct drm_device *ddev = rdev->ddev;
1589         struct drm_crtc *crtc;
1590         struct radeon_crtc *radeon_crtc;
1591         u32 tmp;
1592
1593         /* disable any active CRTCs */
1594         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1595                 radeon_crtc = to_radeon_crtc(crtc);
1596                 if (radeon_crtc->enabled) {
1597                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1598                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1599                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1600                 }
1601         }
1602 }
1603
1604 /**
1605  * evergreen_pm_finish - post-power state change callback.
1606  *
1607  * @rdev: radeon_device pointer
1608  *
1609  * Clean up after a power state change (evergreen+).
1610  */
1611 void evergreen_pm_finish(struct radeon_device *rdev)
1612 {
1613         struct drm_device *ddev = rdev->ddev;
1614         struct drm_crtc *crtc;
1615         struct radeon_crtc *radeon_crtc;
1616         u32 tmp;
1617
1618         /* enable any active CRTCs */
1619         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1620                 radeon_crtc = to_radeon_crtc(crtc);
1621                 if (radeon_crtc->enabled) {
1622                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1623                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1624                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1625                 }
1626         }
1627 }
1628
1629 /**
1630  * evergreen_hpd_sense - hpd sense callback.
1631  *
1632  * @rdev: radeon_device pointer
1633  * @hpd: hpd (hotplug detect) pin
1634  *
1635  * Checks if a digital monitor is connected (evergreen+).
1636  * Returns true if connected, false if not connected.
1637  */
1638 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1639 {
1640         bool connected = false;
1641
1642         switch (hpd) {
1643         case RADEON_HPD_1:
1644                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1645                         connected = true;
1646                 break;
1647         case RADEON_HPD_2:
1648                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1649                         connected = true;
1650                 break;
1651         case RADEON_HPD_3:
1652                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1653                         connected = true;
1654                 break;
1655         case RADEON_HPD_4:
1656                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1657                         connected = true;
1658                 break;
1659         case RADEON_HPD_5:
1660                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1661                         connected = true;
1662                 break;
1663         case RADEON_HPD_6:
1664                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1665                         connected = true;
1666                 break;
1667         default:
1668                 break;
1669         }
1670
1671         return connected;
1672 }
1673
1674 /**
1675  * evergreen_hpd_set_polarity - hpd set polarity callback.
1676  *
1677  * @rdev: radeon_device pointer
1678  * @hpd: hpd (hotplug detect) pin
1679  *
1680  * Set the polarity of the hpd pin (evergreen+).
1681  */
1682 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1683                                 enum radeon_hpd_id hpd)
1684 {
1685         u32 tmp;
1686         bool connected = evergreen_hpd_sense(rdev, hpd);
1687
1688         switch (hpd) {
1689         case RADEON_HPD_1:
1690                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1691                 if (connected)
1692                         tmp &= ~DC_HPDx_INT_POLARITY;
1693                 else
1694                         tmp |= DC_HPDx_INT_POLARITY;
1695                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1696                 break;
1697         case RADEON_HPD_2:
1698                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1699                 if (connected)
1700                         tmp &= ~DC_HPDx_INT_POLARITY;
1701                 else
1702                         tmp |= DC_HPDx_INT_POLARITY;
1703                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1704                 break;
1705         case RADEON_HPD_3:
1706                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1707                 if (connected)
1708                         tmp &= ~DC_HPDx_INT_POLARITY;
1709                 else
1710                         tmp |= DC_HPDx_INT_POLARITY;
1711                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1712                 break;
1713         case RADEON_HPD_4:
1714                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1715                 if (connected)
1716                         tmp &= ~DC_HPDx_INT_POLARITY;
1717                 else
1718                         tmp |= DC_HPDx_INT_POLARITY;
1719                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1720                 break;
1721         case RADEON_HPD_5:
1722                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1723                 if (connected)
1724                         tmp &= ~DC_HPDx_INT_POLARITY;
1725                 else
1726                         tmp |= DC_HPDx_INT_POLARITY;
1727                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1728                         break;
1729         case RADEON_HPD_6:
1730                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1731                 if (connected)
1732                         tmp &= ~DC_HPDx_INT_POLARITY;
1733                 else
1734                         tmp |= DC_HPDx_INT_POLARITY;
1735                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1736                 break;
1737         default:
1738                 break;
1739         }
1740 }
1741
1742 /**
1743  * evergreen_hpd_init - hpd setup callback.
1744  *
1745  * @rdev: radeon_device pointer
1746  *
1747  * Setup the hpd pins used by the card (evergreen+).
1748  * Enable the pin, set the polarity, and enable the hpd interrupts.
1749  */
1750 void evergreen_hpd_init(struct radeon_device *rdev)
1751 {
1752         struct drm_device *dev = rdev->ddev;
1753         struct drm_connector *connector;
1754         unsigned enabled = 0;
1755         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1756                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1757
1758         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1759                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1760
1761                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1762                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1763                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1764                          * aux dp channel on imac and help (but not completely fix)
1765                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1766                          * also avoid interrupt storms during dpms.
1767                          */
1768                         continue;
1769                 }
1770                 switch (radeon_connector->hpd.hpd) {
1771                 case RADEON_HPD_1:
1772                         WREG32(DC_HPD1_CONTROL, tmp);
1773                         break;
1774                 case RADEON_HPD_2:
1775                         WREG32(DC_HPD2_CONTROL, tmp);
1776                         break;
1777                 case RADEON_HPD_3:
1778                         WREG32(DC_HPD3_CONTROL, tmp);
1779                         break;
1780                 case RADEON_HPD_4:
1781                         WREG32(DC_HPD4_CONTROL, tmp);
1782                         break;
1783                 case RADEON_HPD_5:
1784                         WREG32(DC_HPD5_CONTROL, tmp);
1785                         break;
1786                 case RADEON_HPD_6:
1787                         WREG32(DC_HPD6_CONTROL, tmp);
1788                         break;
1789                 default:
1790                         break;
1791                 }
1792                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1793                 enabled |= 1 << radeon_connector->hpd.hpd;
1794         }
1795         radeon_irq_kms_enable_hpd(rdev, enabled);
1796 }
1797
1798 /**
1799  * evergreen_hpd_fini - hpd tear down callback.
1800  *
1801  * @rdev: radeon_device pointer
1802  *
1803  * Tear down the hpd pins used by the card (evergreen+).
1804  * Disable the hpd interrupts.
1805  */
1806 void evergreen_hpd_fini(struct radeon_device *rdev)
1807 {
1808         struct drm_device *dev = rdev->ddev;
1809         struct drm_connector *connector;
1810         unsigned disabled = 0;
1811
1812         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1813                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1814                 switch (radeon_connector->hpd.hpd) {
1815                 case RADEON_HPD_1:
1816                         WREG32(DC_HPD1_CONTROL, 0);
1817                         break;
1818                 case RADEON_HPD_2:
1819                         WREG32(DC_HPD2_CONTROL, 0);
1820                         break;
1821                 case RADEON_HPD_3:
1822                         WREG32(DC_HPD3_CONTROL, 0);
1823                         break;
1824                 case RADEON_HPD_4:
1825                         WREG32(DC_HPD4_CONTROL, 0);
1826                         break;
1827                 case RADEON_HPD_5:
1828                         WREG32(DC_HPD5_CONTROL, 0);
1829                         break;
1830                 case RADEON_HPD_6:
1831                         WREG32(DC_HPD6_CONTROL, 0);
1832                         break;
1833                 default:
1834                         break;
1835                 }
1836                 disabled |= 1 << radeon_connector->hpd.hpd;
1837         }
1838         radeon_irq_kms_disable_hpd(rdev, disabled);
1839 }
1840
1841 /* watermark setup */
1842
1843 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1844                                         struct radeon_crtc *radeon_crtc,
1845                                         struct drm_display_mode *mode,
1846                                         struct drm_display_mode *other_mode)
1847 {
1848         u32 tmp, buffer_alloc, i;
1849         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1850         /*
1851          * Line Buffer Setup
1852          * There are 3 line buffers, each one shared by 2 display controllers.
1853          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1854          * the display controllers.  The paritioning is done via one of four
1855          * preset allocations specified in bits 2:0:
1856          * first display controller
1857          *  0 - first half of lb (3840 * 2)
1858          *  1 - first 3/4 of lb (5760 * 2)
1859          *  2 - whole lb (7680 * 2), other crtc must be disabled
1860          *  3 - first 1/4 of lb (1920 * 2)
1861          * second display controller
1862          *  4 - second half of lb (3840 * 2)
1863          *  5 - second 3/4 of lb (5760 * 2)
1864          *  6 - whole lb (7680 * 2), other crtc must be disabled
1865          *  7 - last 1/4 of lb (1920 * 2)
1866          */
1867         /* this can get tricky if we have two large displays on a paired group
1868          * of crtcs.  Ideally for multiple large displays we'd assign them to
1869          * non-linked crtcs for maximum line buffer allocation.
1870          */
1871         if (radeon_crtc->base.enabled && mode) {
1872                 if (other_mode) {
1873                         tmp = 0; /* 1/2 */
1874                         buffer_alloc = 1;
1875                 } else {
1876                         tmp = 2; /* whole */
1877                         buffer_alloc = 2;
1878                 }
1879         } else {
1880                 tmp = 0;
1881                 buffer_alloc = 0;
1882         }
1883
1884         /* second controller of the pair uses second half of the lb */
1885         if (radeon_crtc->crtc_id % 2)
1886                 tmp += 4;
1887         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1888
1889         if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1890                 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1891                        DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1892                 for (i = 0; i < rdev->usec_timeout; i++) {
1893                         if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1894                             DMIF_BUFFERS_ALLOCATED_COMPLETED)
1895                                 break;
1896                         udelay(1);
1897                 }
1898         }
1899
1900         if (radeon_crtc->base.enabled && mode) {
1901                 switch (tmp) {
1902                 case 0:
1903                 case 4:
1904                 default:
1905                         if (ASIC_IS_DCE5(rdev))
1906                                 return 4096 * 2;
1907                         else
1908                                 return 3840 * 2;
1909                 case 1:
1910                 case 5:
1911                         if (ASIC_IS_DCE5(rdev))
1912                                 return 6144 * 2;
1913                         else
1914                                 return 5760 * 2;
1915                 case 2:
1916                 case 6:
1917                         if (ASIC_IS_DCE5(rdev))
1918                                 return 8192 * 2;
1919                         else
1920                                 return 7680 * 2;
1921                 case 3:
1922                 case 7:
1923                         if (ASIC_IS_DCE5(rdev))
1924                                 return 2048 * 2;
1925                         else
1926                                 return 1920 * 2;
1927                 }
1928         }
1929
1930         /* controller not enabled, so no lb used */
1931         return 0;
1932 }
1933
1934 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1935 {
1936         u32 tmp = RREG32(MC_SHARED_CHMAP);
1937
1938         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1939         case 0:
1940         default:
1941                 return 1;
1942         case 1:
1943                 return 2;
1944         case 2:
1945                 return 4;
1946         case 3:
1947                 return 8;
1948         }
1949 }
1950
1951 struct evergreen_wm_params {
1952         u32 dram_channels; /* number of dram channels */
1953         u32 yclk;          /* bandwidth per dram data pin in kHz */
1954         u32 sclk;          /* engine clock in kHz */
1955         u32 disp_clk;      /* display clock in kHz */
1956         u32 src_width;     /* viewport width */
1957         u32 active_time;   /* active display time in ns */
1958         u32 blank_time;    /* blank time in ns */
1959         bool interlaced;    /* mode is interlaced */
1960         fixed20_12 vsc;    /* vertical scale ratio */
1961         u32 num_heads;     /* number of active crtcs */
1962         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1963         u32 lb_size;       /* line buffer allocated to pipe */
1964         u32 vtaps;         /* vertical scaler taps */
1965 };
1966
1967 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1968 {
1969         /* Calculate DRAM Bandwidth and the part allocated to display. */
1970         fixed20_12 dram_efficiency; /* 0.7 */
1971         fixed20_12 yclk, dram_channels, bandwidth;
1972         fixed20_12 a;
1973
1974         a.full = dfixed_const(1000);
1975         yclk.full = dfixed_const(wm->yclk);
1976         yclk.full = dfixed_div(yclk, a);
1977         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1978         a.full = dfixed_const(10);
1979         dram_efficiency.full = dfixed_const(7);
1980         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1981         bandwidth.full = dfixed_mul(dram_channels, yclk);
1982         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1983
1984         return dfixed_trunc(bandwidth);
1985 }
1986
1987 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1988 {
1989         /* Calculate DRAM Bandwidth and the part allocated to display. */
1990         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1991         fixed20_12 yclk, dram_channels, bandwidth;
1992         fixed20_12 a;
1993
1994         a.full = dfixed_const(1000);
1995         yclk.full = dfixed_const(wm->yclk);
1996         yclk.full = dfixed_div(yclk, a);
1997         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1998         a.full = dfixed_const(10);
1999         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2000         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2001         bandwidth.full = dfixed_mul(dram_channels, yclk);
2002         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2003
2004         return dfixed_trunc(bandwidth);
2005 }
2006
2007 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2008 {
2009         /* Calculate the display Data return Bandwidth */
2010         fixed20_12 return_efficiency; /* 0.8 */
2011         fixed20_12 sclk, bandwidth;
2012         fixed20_12 a;
2013
2014         a.full = dfixed_const(1000);
2015         sclk.full = dfixed_const(wm->sclk);
2016         sclk.full = dfixed_div(sclk, a);
2017         a.full = dfixed_const(10);
2018         return_efficiency.full = dfixed_const(8);
2019         return_efficiency.full = dfixed_div(return_efficiency, a);
2020         a.full = dfixed_const(32);
2021         bandwidth.full = dfixed_mul(a, sclk);
2022         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2023
2024         return dfixed_trunc(bandwidth);
2025 }
2026
2027 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2028 {
2029         /* Calculate the DMIF Request Bandwidth */
2030         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2031         fixed20_12 disp_clk, bandwidth;
2032         fixed20_12 a;
2033
2034         a.full = dfixed_const(1000);
2035         disp_clk.full = dfixed_const(wm->disp_clk);
2036         disp_clk.full = dfixed_div(disp_clk, a);
2037         a.full = dfixed_const(10);
2038         disp_clk_request_efficiency.full = dfixed_const(8);
2039         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2040         a.full = dfixed_const(32);
2041         bandwidth.full = dfixed_mul(a, disp_clk);
2042         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2043
2044         return dfixed_trunc(bandwidth);
2045 }
2046
2047 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2048 {
2049         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2050         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2051         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2052         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2053
2054         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2055 }
2056
2057 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2058 {
2059         /* Calculate the display mode Average Bandwidth
2060          * DisplayMode should contain the source and destination dimensions,
2061          * timing, etc.
2062          */
2063         fixed20_12 bpp;
2064         fixed20_12 line_time;
2065         fixed20_12 src_width;
2066         fixed20_12 bandwidth;
2067         fixed20_12 a;
2068
2069         a.full = dfixed_const(1000);
2070         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2071         line_time.full = dfixed_div(line_time, a);
2072         bpp.full = dfixed_const(wm->bytes_per_pixel);
2073         src_width.full = dfixed_const(wm->src_width);
2074         bandwidth.full = dfixed_mul(src_width, bpp);
2075         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2076         bandwidth.full = dfixed_div(bandwidth, line_time);
2077
2078         return dfixed_trunc(bandwidth);
2079 }
2080
2081 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2082 {
2083         /* First calcualte the latency in ns */
2084         u32 mc_latency = 2000; /* 2000 ns. */
2085         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2086         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2087         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2088         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2089         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2090                 (wm->num_heads * cursor_line_pair_return_time);
2091         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2092         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2093         fixed20_12 a, b, c;
2094
2095         if (wm->num_heads == 0)
2096                 return 0;
2097
2098         a.full = dfixed_const(2);
2099         b.full = dfixed_const(1);
2100         if ((wm->vsc.full > a.full) ||
2101             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2102             (wm->vtaps >= 5) ||
2103             ((wm->vsc.full >= a.full) && wm->interlaced))
2104                 max_src_lines_per_dst_line = 4;
2105         else
2106                 max_src_lines_per_dst_line = 2;
2107
2108         a.full = dfixed_const(available_bandwidth);
2109         b.full = dfixed_const(wm->num_heads);
2110         a.full = dfixed_div(a, b);
2111
2112         b.full = dfixed_const(1000);
2113         c.full = dfixed_const(wm->disp_clk);
2114         b.full = dfixed_div(c, b);
2115         c.full = dfixed_const(wm->bytes_per_pixel);
2116         b.full = dfixed_mul(b, c);
2117
2118         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2119
2120         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2121         b.full = dfixed_const(1000);
2122         c.full = dfixed_const(lb_fill_bw);
2123         b.full = dfixed_div(c, b);
2124         a.full = dfixed_div(a, b);
2125         line_fill_time = dfixed_trunc(a);
2126
2127         if (line_fill_time < wm->active_time)
2128                 return latency;
2129         else
2130                 return latency + (line_fill_time - wm->active_time);
2131
2132 }
2133
2134 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2135 {
2136         if (evergreen_average_bandwidth(wm) <=
2137             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2138                 return true;
2139         else
2140                 return false;
2141 };
2142
2143 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2144 {
2145         if (evergreen_average_bandwidth(wm) <=
2146             (evergreen_available_bandwidth(wm) / wm->num_heads))
2147                 return true;
2148         else
2149                 return false;
2150 };
2151
2152 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2153 {
2154         u32 lb_partitions = wm->lb_size / wm->src_width;
2155         u32 line_time = wm->active_time + wm->blank_time;
2156         u32 latency_tolerant_lines;
2157         u32 latency_hiding;
2158         fixed20_12 a;
2159
2160         a.full = dfixed_const(1);
2161         if (wm->vsc.full > a.full)
2162                 latency_tolerant_lines = 1;
2163         else {
2164                 if (lb_partitions <= (wm->vtaps + 1))
2165                         latency_tolerant_lines = 1;
2166                 else
2167                         latency_tolerant_lines = 2;
2168         }
2169
2170         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2171
2172         if (evergreen_latency_watermark(wm) <= latency_hiding)
2173                 return true;
2174         else
2175                 return false;
2176 }
2177
2178 static void evergreen_program_watermarks(struct radeon_device *rdev,
2179                                          struct radeon_crtc *radeon_crtc,
2180                                          u32 lb_size, u32 num_heads)
2181 {
2182         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2183         struct evergreen_wm_params wm_low, wm_high;
2184         u32 dram_channels;
2185         u32 pixel_period;
2186         u32 line_time = 0;
2187         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2188         u32 priority_a_mark = 0, priority_b_mark = 0;
2189         u32 priority_a_cnt = PRIORITY_OFF;
2190         u32 priority_b_cnt = PRIORITY_OFF;
2191         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2192         u32 tmp, arb_control3;
2193         fixed20_12 a, b, c;
2194
2195         if (radeon_crtc->base.enabled && num_heads && mode) {
2196                 pixel_period = 1000000 / (u32)mode->clock;
2197                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2198                 priority_a_cnt = 0;
2199                 priority_b_cnt = 0;
2200                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2201
2202                 /* watermark for high clocks */
2203                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2204                         wm_high.yclk =
2205                                 radeon_dpm_get_mclk(rdev, false) * 10;
2206                         wm_high.sclk =
2207                                 radeon_dpm_get_sclk(rdev, false) * 10;
2208                 } else {
2209                         wm_high.yclk = rdev->pm.current_mclk * 10;
2210                         wm_high.sclk = rdev->pm.current_sclk * 10;
2211                 }
2212
2213                 wm_high.disp_clk = mode->clock;
2214                 wm_high.src_width = mode->crtc_hdisplay;
2215                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2216                 wm_high.blank_time = line_time - wm_high.active_time;
2217                 wm_high.interlaced = false;
2218                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2219                         wm_high.interlaced = true;
2220                 wm_high.vsc = radeon_crtc->vsc;
2221                 wm_high.vtaps = 1;
2222                 if (radeon_crtc->rmx_type != RMX_OFF)
2223                         wm_high.vtaps = 2;
2224                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2225                 wm_high.lb_size = lb_size;
2226                 wm_high.dram_channels = dram_channels;
2227                 wm_high.num_heads = num_heads;
2228
2229                 /* watermark for low clocks */
2230                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2231                         wm_low.yclk =
2232                                 radeon_dpm_get_mclk(rdev, true) * 10;
2233                         wm_low.sclk =
2234                                 radeon_dpm_get_sclk(rdev, true) * 10;
2235                 } else {
2236                         wm_low.yclk = rdev->pm.current_mclk * 10;
2237                         wm_low.sclk = rdev->pm.current_sclk * 10;
2238                 }
2239
2240                 wm_low.disp_clk = mode->clock;
2241                 wm_low.src_width = mode->crtc_hdisplay;
2242                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2243                 wm_low.blank_time = line_time - wm_low.active_time;
2244                 wm_low.interlaced = false;
2245                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2246                         wm_low.interlaced = true;
2247                 wm_low.vsc = radeon_crtc->vsc;
2248                 wm_low.vtaps = 1;
2249                 if (radeon_crtc->rmx_type != RMX_OFF)
2250                         wm_low.vtaps = 2;
2251                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2252                 wm_low.lb_size = lb_size;
2253                 wm_low.dram_channels = dram_channels;
2254                 wm_low.num_heads = num_heads;
2255
2256                 /* set for high clocks */
2257                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2258                 /* set for low clocks */
2259                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2260
2261                 /* possibly force display priority to high */
2262                 /* should really do this at mode validation time... */
2263                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2264                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2265                     !evergreen_check_latency_hiding(&wm_high) ||
2266                     (rdev->disp_priority == 2)) {
2267                         DRM_DEBUG_KMS("force priority a to high\n");
2268                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2269                 }
2270                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2271                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2272                     !evergreen_check_latency_hiding(&wm_low) ||
2273                     (rdev->disp_priority == 2)) {
2274                         DRM_DEBUG_KMS("force priority b to high\n");
2275                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2276                 }
2277
2278                 a.full = dfixed_const(1000);
2279                 b.full = dfixed_const(mode->clock);
2280                 b.full = dfixed_div(b, a);
2281                 c.full = dfixed_const(latency_watermark_a);
2282                 c.full = dfixed_mul(c, b);
2283                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2284                 c.full = dfixed_div(c, a);
2285                 a.full = dfixed_const(16);
2286                 c.full = dfixed_div(c, a);
2287                 priority_a_mark = dfixed_trunc(c);
2288                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2289
2290                 a.full = dfixed_const(1000);
2291                 b.full = dfixed_const(mode->clock);
2292                 b.full = dfixed_div(b, a);
2293                 c.full = dfixed_const(latency_watermark_b);
2294                 c.full = dfixed_mul(c, b);
2295                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2296                 c.full = dfixed_div(c, a);
2297                 a.full = dfixed_const(16);
2298                 c.full = dfixed_div(c, a);
2299                 priority_b_mark = dfixed_trunc(c);
2300                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2301         }
2302
2303         /* select wm A */
2304         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2305         tmp = arb_control3;
2306         tmp &= ~LATENCY_WATERMARK_MASK(3);
2307         tmp |= LATENCY_WATERMARK_MASK(1);
2308         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2309         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2310                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2311                 LATENCY_HIGH_WATERMARK(line_time)));
2312         /* select wm B */
2313         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2314         tmp &= ~LATENCY_WATERMARK_MASK(3);
2315         tmp |= LATENCY_WATERMARK_MASK(2);
2316         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2317         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2318                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2319                 LATENCY_HIGH_WATERMARK(line_time)));
2320         /* restore original selection */
2321         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2322
2323         /* write the priority marks */
2324         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2325         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2326
2327         /* save values for DPM */
2328         radeon_crtc->line_time = line_time;
2329         radeon_crtc->wm_high = latency_watermark_a;
2330         radeon_crtc->wm_low = latency_watermark_b;
2331 }
2332
2333 /**
2334  * evergreen_bandwidth_update - update display watermarks callback.
2335  *
2336  * @rdev: radeon_device pointer
2337  *
2338  * Update the display watermarks based on the requested mode(s)
2339  * (evergreen+).
2340  */
2341 void evergreen_bandwidth_update(struct radeon_device *rdev)
2342 {
2343         struct drm_display_mode *mode0 = NULL;
2344         struct drm_display_mode *mode1 = NULL;
2345         u32 num_heads = 0, lb_size;
2346         int i;
2347
2348         radeon_update_display_priority(rdev);
2349
2350         for (i = 0; i < rdev->num_crtc; i++) {
2351                 if (rdev->mode_info.crtcs[i]->base.enabled)
2352                         num_heads++;
2353         }
2354         for (i = 0; i < rdev->num_crtc; i += 2) {
2355                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2356                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2357                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2358                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2359                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2360                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2361         }
2362 }
2363
2364 /**
2365  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2366  *
2367  * @rdev: radeon_device pointer
2368  *
2369  * Wait for the MC (memory controller) to be idle.
2370  * (evergreen+).
2371  * Returns 0 if the MC is idle, -1 if not.
2372  */
2373 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2374 {
2375         unsigned i;
2376         u32 tmp;
2377
2378         for (i = 0; i < rdev->usec_timeout; i++) {
2379                 /* read MC_STATUS */
2380                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2381                 if (!tmp)
2382                         return 0;
2383                 udelay(1);
2384         }
2385         return -1;
2386 }
2387
2388 /*
2389  * GART
2390  */
2391 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2392 {
2393         unsigned i;
2394         u32 tmp;
2395
2396         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2397
2398         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2399         for (i = 0; i < rdev->usec_timeout; i++) {
2400                 /* read MC_STATUS */
2401                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2402                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2403                 if (tmp == 2) {
2404                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2405                         return;
2406                 }
2407                 if (tmp) {
2408                         return;
2409                 }
2410                 udelay(1);
2411         }
2412 }
2413
2414 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2415 {
2416         u32 tmp;
2417         int r;
2418
2419         if (rdev->gart.robj == NULL) {
2420                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2421                 return -EINVAL;
2422         }
2423         r = radeon_gart_table_vram_pin(rdev);
2424         if (r)
2425                 return r;
2426         /* Setup L2 cache */
2427         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2428                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2429                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2430         WREG32(VM_L2_CNTL2, 0);
2431         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2432         /* Setup TLB control */
2433         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2434                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2435                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2436                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2437         if (rdev->flags & RADEON_IS_IGP) {
2438                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2439                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2440                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2441         } else {
2442                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2443                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2444                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2445                 if ((rdev->family == CHIP_JUNIPER) ||
2446                     (rdev->family == CHIP_CYPRESS) ||
2447                     (rdev->family == CHIP_HEMLOCK) ||
2448                     (rdev->family == CHIP_BARTS))
2449                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2450         }
2451         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2452         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2453         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2454         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2455         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2456         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2457         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2458         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2459                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2460         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2461                         (u32)(rdev->dummy_page.addr >> 12));
2462         WREG32(VM_CONTEXT1_CNTL, 0);
2463
2464         evergreen_pcie_gart_tlb_flush(rdev);
2465         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2466                  (unsigned)(rdev->mc.gtt_size >> 20),
2467                  (unsigned long long)rdev->gart.table_addr);
2468         rdev->gart.ready = true;
2469         return 0;
2470 }
2471
2472 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2473 {
2474         u32 tmp;
2475
2476         /* Disable all tables */
2477         WREG32(VM_CONTEXT0_CNTL, 0);
2478         WREG32(VM_CONTEXT1_CNTL, 0);
2479
2480         /* Setup L2 cache */
2481         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2482                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2483         WREG32(VM_L2_CNTL2, 0);
2484         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2485         /* Setup TLB control */
2486         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2487         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2488         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2489         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2490         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2491         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2492         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2493         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2494         radeon_gart_table_vram_unpin(rdev);
2495 }
2496
2497 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2498 {
2499         evergreen_pcie_gart_disable(rdev);
2500         radeon_gart_table_vram_free(rdev);
2501         radeon_gart_fini(rdev);
2502 }
2503
2504
2505 static void evergreen_agp_enable(struct radeon_device *rdev)
2506 {
2507         u32 tmp;
2508
2509         /* Setup L2 cache */
2510         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2511                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2512                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2513         WREG32(VM_L2_CNTL2, 0);
2514         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2515         /* Setup TLB control */
2516         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2517                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2518                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2519                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2520         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2521         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2522         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2523         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2524         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2525         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2526         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2527         WREG32(VM_CONTEXT0_CNTL, 0);
2528         WREG32(VM_CONTEXT1_CNTL, 0);
2529 }
2530
2531 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2532 {
2533         u32 crtc_enabled, tmp, frame_count, blackout;
2534         int i, j;
2535
2536         if (!ASIC_IS_NODCE(rdev)) {
2537                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2538                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2539
2540                 /* disable VGA render */
2541                 WREG32(VGA_RENDER_CONTROL, 0);
2542         }
2543         /* blank the display controllers */
2544         for (i = 0; i < rdev->num_crtc; i++) {
2545                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2546                 if (crtc_enabled) {
2547                         save->crtc_enabled[i] = true;
2548                         if (ASIC_IS_DCE6(rdev)) {
2549                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2550                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2551                                         radeon_wait_for_vblank(rdev, i);
2552                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2553                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2554                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2555                                 }
2556                         } else {
2557                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2558                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2559                                         radeon_wait_for_vblank(rdev, i);
2560                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2561                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2562                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2563                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2564                                 }
2565                         }
2566                         /* wait for the next frame */
2567                         frame_count = radeon_get_vblank_counter(rdev, i);
2568                         for (j = 0; j < rdev->usec_timeout; j++) {
2569                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2570                                         break;
2571                                 udelay(1);
2572                         }
2573
2574                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2575                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2576                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2577                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2578                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2579                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2580                         save->crtc_enabled[i] = false;
2581                         /* ***** */
2582                 } else {
2583                         save->crtc_enabled[i] = false;
2584                 }
2585         }
2586
2587         radeon_mc_wait_for_idle(rdev);
2588
2589         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2590         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2591                 /* Block CPU access */
2592                 WREG32(BIF_FB_EN, 0);
2593                 /* blackout the MC */
2594                 blackout &= ~BLACKOUT_MODE_MASK;
2595                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2596         }
2597         /* wait for the MC to settle */
2598         udelay(100);
2599
2600         /* lock double buffered regs */
2601         for (i = 0; i < rdev->num_crtc; i++) {
2602                 if (save->crtc_enabled[i]) {
2603                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2604                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2605                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2606                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2607                         }
2608                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2609                         if (!(tmp & 1)) {
2610                                 tmp |= 1;
2611                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2612                         }
2613                 }
2614         }
2615 }
2616
2617 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2618 {
2619         u32 tmp, frame_count;
2620         int i, j;
2621
2622         /* update crtc base addresses */
2623         for (i = 0; i < rdev->num_crtc; i++) {
2624                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2625                        upper_32_bits(rdev->mc.vram_start));
2626                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2627                        upper_32_bits(rdev->mc.vram_start));
2628                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2629                        (u32)rdev->mc.vram_start);
2630                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2631                        (u32)rdev->mc.vram_start);
2632         }
2633
2634         if (!ASIC_IS_NODCE(rdev)) {
2635                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2636                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2637         }
2638
2639         /* unlock regs and wait for update */
2640         for (i = 0; i < rdev->num_crtc; i++) {
2641                 if (save->crtc_enabled[i]) {
2642                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2643                         if ((tmp & 0x7) != 3) {
2644                                 tmp &= ~0x7;
2645                                 tmp |= 0x3;
2646                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2647                         }
2648                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2649                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2650                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2651                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2652                         }
2653                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2654                         if (tmp & 1) {
2655                                 tmp &= ~1;
2656                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2657                         }
2658                         for (j = 0; j < rdev->usec_timeout; j++) {
2659                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2660                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2661                                         break;
2662                                 udelay(1);
2663                         }
2664                 }
2665         }
2666
2667         /* unblackout the MC */
2668         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2669         tmp &= ~BLACKOUT_MODE_MASK;
2670         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2671         /* allow CPU access */
2672         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2673
2674         for (i = 0; i < rdev->num_crtc; i++) {
2675                 if (save->crtc_enabled[i]) {
2676                         if (ASIC_IS_DCE6(rdev)) {
2677                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2678                                 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2679                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2680                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2681                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2682                         } else {
2683                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2684                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2685                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2687                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2688                         }
2689                         /* wait for the next frame */
2690                         frame_count = radeon_get_vblank_counter(rdev, i);
2691                         for (j = 0; j < rdev->usec_timeout; j++) {
2692                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2693                                         break;
2694                                 udelay(1);
2695                         }
2696                 }
2697         }
2698         if (!ASIC_IS_NODCE(rdev)) {
2699                 /* Unlock vga access */
2700                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2701                 mdelay(1);
2702                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2703         }
2704 }
2705
2706 void evergreen_mc_program(struct radeon_device *rdev)
2707 {
2708         struct evergreen_mc_save save;
2709         u32 tmp;
2710         int i, j;
2711
2712         /* Initialize HDP */
2713         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2714                 WREG32((0x2c14 + j), 0x00000000);
2715                 WREG32((0x2c18 + j), 0x00000000);
2716                 WREG32((0x2c1c + j), 0x00000000);
2717                 WREG32((0x2c20 + j), 0x00000000);
2718                 WREG32((0x2c24 + j), 0x00000000);
2719         }
2720         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2721
2722         evergreen_mc_stop(rdev, &save);
2723         if (evergreen_mc_wait_for_idle(rdev)) {
2724                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2725         }
2726         /* Lockout access through VGA aperture*/
2727         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2728         /* Update configuration */
2729         if (rdev->flags & RADEON_IS_AGP) {
2730                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2731                         /* VRAM before AGP */
2732                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2733                                 rdev->mc.vram_start >> 12);
2734                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2735                                 rdev->mc.gtt_end >> 12);
2736                 } else {
2737                         /* VRAM after AGP */
2738                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2739                                 rdev->mc.gtt_start >> 12);
2740                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2741                                 rdev->mc.vram_end >> 12);
2742                 }
2743         } else {
2744                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2745                         rdev->mc.vram_start >> 12);
2746                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2747                         rdev->mc.vram_end >> 12);
2748         }
2749         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2750         /* llano/ontario only */
2751         if ((rdev->family == CHIP_PALM) ||
2752             (rdev->family == CHIP_SUMO) ||
2753             (rdev->family == CHIP_SUMO2)) {
2754                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2755                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2756                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2757                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2758         }
2759         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2760         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2761         WREG32(MC_VM_FB_LOCATION, tmp);
2762         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2763         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2764         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2765         if (rdev->flags & RADEON_IS_AGP) {
2766                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2767                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2768                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2769         } else {
2770                 WREG32(MC_VM_AGP_BASE, 0);
2771                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2772                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2773         }
2774         if (evergreen_mc_wait_for_idle(rdev)) {
2775                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2776         }
2777         evergreen_mc_resume(rdev, &save);
2778         /* we need to own VRAM, so turn off the VGA renderer here
2779          * to stop it overwriting our objects */
2780         rv515_vga_render_disable(rdev);
2781 }
2782
2783 /*
2784  * CP.
2785  */
2786 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2787 {
2788         struct radeon_ring *ring = &rdev->ring[ib->ring];
2789         u32 next_rptr;
2790
2791         /* set to DX10/11 mode */
2792         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2793         radeon_ring_write(ring, 1);
2794
2795         if (ring->rptr_save_reg) {
2796                 next_rptr = ring->wptr + 3 + 4;
2797                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2798                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2799                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2800                 radeon_ring_write(ring, next_rptr);
2801         } else if (rdev->wb.enabled) {
2802                 next_rptr = ring->wptr + 5 + 4;
2803                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2804                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2805                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2806                 radeon_ring_write(ring, next_rptr);
2807                 radeon_ring_write(ring, 0);
2808         }
2809
2810         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2811         radeon_ring_write(ring,
2812 #ifdef __BIG_ENDIAN
2813                           (2 << 0) |
2814 #endif
2815                           (ib->gpu_addr & 0xFFFFFFFC));
2816         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2817         radeon_ring_write(ring, ib->length_dw);
2818 }
2819
2820
2821 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2822 {
2823         const __be32 *fw_data;
2824         int i;
2825
2826         if (!rdev->me_fw || !rdev->pfp_fw)
2827                 return -EINVAL;
2828
2829         r700_cp_stop(rdev);
2830         WREG32(CP_RB_CNTL,
2831 #ifdef __BIG_ENDIAN
2832                BUF_SWAP_32BIT |
2833 #endif
2834                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2835
2836         fw_data = (const __be32 *)rdev->pfp_fw->data;
2837         WREG32(CP_PFP_UCODE_ADDR, 0);
2838         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2839                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2840         WREG32(CP_PFP_UCODE_ADDR, 0);
2841
2842         fw_data = (const __be32 *)rdev->me_fw->data;
2843         WREG32(CP_ME_RAM_WADDR, 0);
2844         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2845                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2846
2847         WREG32(CP_PFP_UCODE_ADDR, 0);
2848         WREG32(CP_ME_RAM_WADDR, 0);
2849         WREG32(CP_ME_RAM_RADDR, 0);
2850         return 0;
2851 }
2852
2853 static int evergreen_cp_start(struct radeon_device *rdev)
2854 {
2855         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2856         int r, i;
2857         uint32_t cp_me;
2858
2859         r = radeon_ring_lock(rdev, ring, 7);
2860         if (r) {
2861                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2862                 return r;
2863         }
2864         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2865         radeon_ring_write(ring, 0x1);
2866         radeon_ring_write(ring, 0x0);
2867         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2868         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2869         radeon_ring_write(ring, 0);
2870         radeon_ring_write(ring, 0);
2871         radeon_ring_unlock_commit(rdev, ring, false);
2872
2873         cp_me = 0xff;
2874         WREG32(CP_ME_CNTL, cp_me);
2875
2876         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2877         if (r) {
2878                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2879                 return r;
2880         }
2881
2882         /* setup clear context state */
2883         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2884         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2885
2886         for (i = 0; i < evergreen_default_size; i++)
2887                 radeon_ring_write(ring, evergreen_default_state[i]);
2888
2889         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2890         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2891
2892         /* set clear context state */
2893         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2894         radeon_ring_write(ring, 0);
2895
2896         /* SQ_VTX_BASE_VTX_LOC */
2897         radeon_ring_write(ring, 0xc0026f00);
2898         radeon_ring_write(ring, 0x00000000);
2899         radeon_ring_write(ring, 0x00000000);
2900         radeon_ring_write(ring, 0x00000000);
2901
2902         /* Clear consts */
2903         radeon_ring_write(ring, 0xc0036f00);
2904         radeon_ring_write(ring, 0x00000bc4);
2905         radeon_ring_write(ring, 0xffffffff);
2906         radeon_ring_write(ring, 0xffffffff);
2907         radeon_ring_write(ring, 0xffffffff);
2908
2909         radeon_ring_write(ring, 0xc0026900);
2910         radeon_ring_write(ring, 0x00000316);
2911         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2912         radeon_ring_write(ring, 0x00000010); /*  */
2913
2914         radeon_ring_unlock_commit(rdev, ring, false);
2915
2916         return 0;
2917 }
2918
2919 static int evergreen_cp_resume(struct radeon_device *rdev)
2920 {
2921         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2922         u32 tmp;
2923         u32 rb_bufsz;
2924         int r;
2925
2926         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2927         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2928                                  SOFT_RESET_PA |
2929                                  SOFT_RESET_SH |
2930                                  SOFT_RESET_VGT |
2931                                  SOFT_RESET_SPI |
2932                                  SOFT_RESET_SX));
2933         RREG32(GRBM_SOFT_RESET);
2934         mdelay(15);
2935         WREG32(GRBM_SOFT_RESET, 0);
2936         RREG32(GRBM_SOFT_RESET);
2937
2938         /* Set ring buffer size */
2939         rb_bufsz = order_base_2(ring->ring_size / 8);
2940         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2941 #ifdef __BIG_ENDIAN
2942         tmp |= BUF_SWAP_32BIT;
2943 #endif
2944         WREG32(CP_RB_CNTL, tmp);
2945         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2946         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2947
2948         /* Set the write pointer delay */
2949         WREG32(CP_RB_WPTR_DELAY, 0);
2950
2951         /* Initialize the ring buffer's read and write pointers */
2952         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2953         WREG32(CP_RB_RPTR_WR, 0);
2954         ring->wptr = 0;
2955         WREG32(CP_RB_WPTR, ring->wptr);
2956
2957         /* set the wb address whether it's enabled or not */
2958         WREG32(CP_RB_RPTR_ADDR,
2959                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2960         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2961         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2962
2963         if (rdev->wb.enabled)
2964                 WREG32(SCRATCH_UMSK, 0xff);
2965         else {
2966                 tmp |= RB_NO_UPDATE;
2967                 WREG32(SCRATCH_UMSK, 0);
2968         }
2969
2970         mdelay(1);
2971         WREG32(CP_RB_CNTL, tmp);
2972
2973         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2974         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2975
2976         evergreen_cp_start(rdev);
2977         ring->ready = true;
2978         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2979         if (r) {
2980                 ring->ready = false;
2981                 return r;
2982         }
2983         return 0;
2984 }
2985
2986 /*
2987  * Core functions
2988  */
2989 static void evergreen_gpu_init(struct radeon_device *rdev)
2990 {
2991         u32 gb_addr_config;
2992         u32 mc_shared_chmap, mc_arb_ramcfg;
2993         u32 sx_debug_1;
2994         u32 smx_dc_ctl0;
2995         u32 sq_config;
2996         u32 sq_lds_resource_mgmt;
2997         u32 sq_gpr_resource_mgmt_1;
2998         u32 sq_gpr_resource_mgmt_2;
2999         u32 sq_gpr_resource_mgmt_3;
3000         u32 sq_thread_resource_mgmt;
3001         u32 sq_thread_resource_mgmt_2;
3002         u32 sq_stack_resource_mgmt_1;
3003         u32 sq_stack_resource_mgmt_2;
3004         u32 sq_stack_resource_mgmt_3;
3005         u32 vgt_cache_invalidation;
3006         u32 hdp_host_path_cntl, tmp;
3007         u32 disabled_rb_mask;
3008         int i, j, num_shader_engines, ps_thread_count;
3009
3010         switch (rdev->family) {
3011         case CHIP_CYPRESS:
3012         case CHIP_HEMLOCK:
3013                 rdev->config.evergreen.num_ses = 2;
3014                 rdev->config.evergreen.max_pipes = 4;
3015                 rdev->config.evergreen.max_tile_pipes = 8;
3016                 rdev->config.evergreen.max_simds = 10;
3017                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3018                 rdev->config.evergreen.max_gprs = 256;
3019                 rdev->config.evergreen.max_threads = 248;
3020                 rdev->config.evergreen.max_gs_threads = 32;
3021                 rdev->config.evergreen.max_stack_entries = 512;
3022                 rdev->config.evergreen.sx_num_of_sets = 4;
3023                 rdev->config.evergreen.sx_max_export_size = 256;
3024                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3025                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3026                 rdev->config.evergreen.max_hw_contexts = 8;
3027                 rdev->config.evergreen.sq_num_cf_insts = 2;
3028
3029                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3030                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3031                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3032                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3033                 break;
3034         case CHIP_JUNIPER:
3035                 rdev->config.evergreen.num_ses = 1;
3036                 rdev->config.evergreen.max_pipes = 4;
3037                 rdev->config.evergreen.max_tile_pipes = 4;
3038                 rdev->config.evergreen.max_simds = 10;
3039                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3040                 rdev->config.evergreen.max_gprs = 256;
3041                 rdev->config.evergreen.max_threads = 248;
3042                 rdev->config.evergreen.max_gs_threads = 32;
3043                 rdev->config.evergreen.max_stack_entries = 512;
3044                 rdev->config.evergreen.sx_num_of_sets = 4;
3045                 rdev->config.evergreen.sx_max_export_size = 256;
3046                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3047                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3048                 rdev->config.evergreen.max_hw_contexts = 8;
3049                 rdev->config.evergreen.sq_num_cf_insts = 2;
3050
3051                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3052                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3053                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3054                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3055                 break;
3056         case CHIP_REDWOOD:
3057                 rdev->config.evergreen.num_ses = 1;
3058                 rdev->config.evergreen.max_pipes = 4;
3059                 rdev->config.evergreen.max_tile_pipes = 4;
3060                 rdev->config.evergreen.max_simds = 5;
3061                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3062                 rdev->config.evergreen.max_gprs = 256;
3063                 rdev->config.evergreen.max_threads = 248;
3064                 rdev->config.evergreen.max_gs_threads = 32;
3065                 rdev->config.evergreen.max_stack_entries = 256;
3066                 rdev->config.evergreen.sx_num_of_sets = 4;
3067                 rdev->config.evergreen.sx_max_export_size = 256;
3068                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3069                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3070                 rdev->config.evergreen.max_hw_contexts = 8;
3071                 rdev->config.evergreen.sq_num_cf_insts = 2;
3072
3073                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3074                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3075                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3076                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3077                 break;
3078         case CHIP_CEDAR:
3079         default:
3080                 rdev->config.evergreen.num_ses = 1;
3081                 rdev->config.evergreen.max_pipes = 2;
3082                 rdev->config.evergreen.max_tile_pipes = 2;
3083                 rdev->config.evergreen.max_simds = 2;
3084                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3085                 rdev->config.evergreen.max_gprs = 256;
3086                 rdev->config.evergreen.max_threads = 192;
3087                 rdev->config.evergreen.max_gs_threads = 16;
3088                 rdev->config.evergreen.max_stack_entries = 256;
3089                 rdev->config.evergreen.sx_num_of_sets = 4;
3090                 rdev->config.evergreen.sx_max_export_size = 128;
3091                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3092                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3093                 rdev->config.evergreen.max_hw_contexts = 4;
3094                 rdev->config.evergreen.sq_num_cf_insts = 1;
3095
3096                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3097                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3098                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3099                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3100                 break;
3101         case CHIP_PALM:
3102                 rdev->config.evergreen.num_ses = 1;
3103                 rdev->config.evergreen.max_pipes = 2;
3104                 rdev->config.evergreen.max_tile_pipes = 2;
3105                 rdev->config.evergreen.max_simds = 2;
3106                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3107                 rdev->config.evergreen.max_gprs = 256;
3108                 rdev->config.evergreen.max_threads = 192;
3109                 rdev->config.evergreen.max_gs_threads = 16;
3110                 rdev->config.evergreen.max_stack_entries = 256;
3111                 rdev->config.evergreen.sx_num_of_sets = 4;
3112                 rdev->config.evergreen.sx_max_export_size = 128;
3113                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3114                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3115                 rdev->config.evergreen.max_hw_contexts = 4;
3116                 rdev->config.evergreen.sq_num_cf_insts = 1;
3117
3118                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3119                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3120                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3121                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3122                 break;
3123         case CHIP_SUMO:
3124                 rdev->config.evergreen.num_ses = 1;
3125                 rdev->config.evergreen.max_pipes = 4;
3126                 rdev->config.evergreen.max_tile_pipes = 4;
3127                 if (rdev->pdev->device == 0x9648)
3128                         rdev->config.evergreen.max_simds = 3;
3129                 else if ((rdev->pdev->device == 0x9647) ||
3130                          (rdev->pdev->device == 0x964a))
3131                         rdev->config.evergreen.max_simds = 4;
3132                 else
3133                         rdev->config.evergreen.max_simds = 5;
3134                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3135                 rdev->config.evergreen.max_gprs = 256;
3136                 rdev->config.evergreen.max_threads = 248;
3137                 rdev->config.evergreen.max_gs_threads = 32;
3138                 rdev->config.evergreen.max_stack_entries = 256;
3139                 rdev->config.evergreen.sx_num_of_sets = 4;
3140                 rdev->config.evergreen.sx_max_export_size = 256;
3141                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3142                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3143                 rdev->config.evergreen.max_hw_contexts = 8;
3144                 rdev->config.evergreen.sq_num_cf_insts = 2;
3145
3146                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3147                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3148                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3149                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3150                 break;
3151         case CHIP_SUMO2:
3152                 rdev->config.evergreen.num_ses = 1;
3153                 rdev->config.evergreen.max_pipes = 4;
3154                 rdev->config.evergreen.max_tile_pipes = 4;
3155                 rdev->config.evergreen.max_simds = 2;
3156                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3157                 rdev->config.evergreen.max_gprs = 256;
3158                 rdev->config.evergreen.max_threads = 248;
3159                 rdev->config.evergreen.max_gs_threads = 32;
3160                 rdev->config.evergreen.max_stack_entries = 512;
3161                 rdev->config.evergreen.sx_num_of_sets = 4;
3162                 rdev->config.evergreen.sx_max_export_size = 256;
3163                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3164                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3165                 rdev->config.evergreen.max_hw_contexts = 4;
3166                 rdev->config.evergreen.sq_num_cf_insts = 2;
3167
3168                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3169                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3170                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3171                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3172                 break;
3173         case CHIP_BARTS:
3174                 rdev->config.evergreen.num_ses = 2;
3175                 rdev->config.evergreen.max_pipes = 4;
3176                 rdev->config.evergreen.max_tile_pipes = 8;
3177                 rdev->config.evergreen.max_simds = 7;
3178                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3179                 rdev->config.evergreen.max_gprs = 256;
3180                 rdev->config.evergreen.max_threads = 248;
3181                 rdev->config.evergreen.max_gs_threads = 32;
3182                 rdev->config.evergreen.max_stack_entries = 512;
3183                 rdev->config.evergreen.sx_num_of_sets = 4;
3184                 rdev->config.evergreen.sx_max_export_size = 256;
3185                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3186                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3187                 rdev->config.evergreen.max_hw_contexts = 8;
3188                 rdev->config.evergreen.sq_num_cf_insts = 2;
3189
3190                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3191                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3192                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3193                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3194                 break;
3195         case CHIP_TURKS:
3196                 rdev->config.evergreen.num_ses = 1;
3197                 rdev->config.evergreen.max_pipes = 4;
3198                 rdev->config.evergreen.max_tile_pipes = 4;
3199                 rdev->config.evergreen.max_simds = 6;
3200                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3201                 rdev->config.evergreen.max_gprs = 256;
3202                 rdev->config.evergreen.max_threads = 248;
3203                 rdev->config.evergreen.max_gs_threads = 32;
3204                 rdev->config.evergreen.max_stack_entries = 256;
3205                 rdev->config.evergreen.sx_num_of_sets = 4;
3206                 rdev->config.evergreen.sx_max_export_size = 256;
3207                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3208                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3209                 rdev->config.evergreen.max_hw_contexts = 8;
3210                 rdev->config.evergreen.sq_num_cf_insts = 2;
3211
3212                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3213                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3214                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3215                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3216                 break;
3217         case CHIP_CAICOS:
3218                 rdev->config.evergreen.num_ses = 1;
3219                 rdev->config.evergreen.max_pipes = 2;
3220                 rdev->config.evergreen.max_tile_pipes = 2;
3221                 rdev->config.evergreen.max_simds = 2;
3222                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3223                 rdev->config.evergreen.max_gprs = 256;
3224                 rdev->config.evergreen.max_threads = 192;
3225                 rdev->config.evergreen.max_gs_threads = 16;
3226                 rdev->config.evergreen.max_stack_entries = 256;
3227                 rdev->config.evergreen.sx_num_of_sets = 4;
3228                 rdev->config.evergreen.sx_max_export_size = 128;
3229                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3230                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3231                 rdev->config.evergreen.max_hw_contexts = 4;
3232                 rdev->config.evergreen.sq_num_cf_insts = 1;
3233
3234                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3235                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3236                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3237                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3238                 break;
3239         }
3240
3241         /* Initialize HDP */
3242         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3243                 WREG32((0x2c14 + j), 0x00000000);
3244                 WREG32((0x2c18 + j), 0x00000000);
3245                 WREG32((0x2c1c + j), 0x00000000);
3246                 WREG32((0x2c20 + j), 0x00000000);
3247                 WREG32((0x2c24 + j), 0x00000000);
3248         }
3249
3250         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3251
3252         evergreen_fix_pci_max_read_req_size(rdev);
3253
3254         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3255         if ((rdev->family == CHIP_PALM) ||
3256             (rdev->family == CHIP_SUMO) ||
3257             (rdev->family == CHIP_SUMO2))
3258                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3259         else
3260                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3261
3262         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3263          * not have bank info, so create a custom tiling dword.
3264          * bits 3:0   num_pipes
3265          * bits 7:4   num_banks
3266          * bits 11:8  group_size
3267          * bits 15:12 row_size
3268          */
3269         rdev->config.evergreen.tile_config = 0;
3270         switch (rdev->config.evergreen.max_tile_pipes) {
3271         case 1:
3272         default:
3273                 rdev->config.evergreen.tile_config |= (0 << 0);
3274                 break;
3275         case 2:
3276                 rdev->config.evergreen.tile_config |= (1 << 0);
3277                 break;
3278         case 4:
3279                 rdev->config.evergreen.tile_config |= (2 << 0);
3280                 break;
3281         case 8:
3282                 rdev->config.evergreen.tile_config |= (3 << 0);
3283                 break;
3284         }
3285         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3286         if (rdev->flags & RADEON_IS_IGP)
3287                 rdev->config.evergreen.tile_config |= 1 << 4;
3288         else {
3289                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3290                 case 0: /* four banks */
3291                         rdev->config.evergreen.tile_config |= 0 << 4;
3292                         break;
3293                 case 1: /* eight banks */
3294                         rdev->config.evergreen.tile_config |= 1 << 4;
3295                         break;
3296                 case 2: /* sixteen banks */
3297                 default:
3298                         rdev->config.evergreen.tile_config |= 2 << 4;
3299                         break;
3300                 }
3301         }
3302         rdev->config.evergreen.tile_config |= 0 << 8;
3303         rdev->config.evergreen.tile_config |=
3304                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3305
3306         num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3307
3308         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3309                 u32 efuse_straps_4;
3310                 u32 efuse_straps_3;
3311
3312                 efuse_straps_4 = RREG32_RCU(0x204);
3313                 efuse_straps_3 = RREG32_RCU(0x203);
3314                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3315                       ((efuse_straps_3 & 0xf0000000) >> 28));
3316         } else {
3317                 tmp = 0;
3318                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3319                         u32 rb_disable_bitmap;
3320
3321                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3322                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3323                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3324                         tmp <<= 4;
3325                         tmp |= rb_disable_bitmap;
3326                 }
3327         }
3328         /* enabled rb are just the one not disabled :) */
3329         disabled_rb_mask = tmp;
3330         tmp = 0;
3331         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3332                 tmp |= (1 << i);
3333         /* if all the backends are disabled, fix it up here */
3334         if ((disabled_rb_mask & tmp) == tmp) {
3335                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3336                         disabled_rb_mask &= ~(1 << i);
3337         }
3338
3339         for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3340                 u32 simd_disable_bitmap;
3341
3342                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3343                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3344                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3345                 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3346                 tmp <<= 16;
3347                 tmp |= simd_disable_bitmap;
3348         }
3349         rdev->config.evergreen.active_simds = hweight32(~tmp);
3350
3351         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3352         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3353
3354         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3355         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3356         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3357         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3358         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3359         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3360         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3361
3362         if ((rdev->config.evergreen.max_backends == 1) &&
3363             (rdev->flags & RADEON_IS_IGP)) {
3364                 if ((disabled_rb_mask & 3) == 1) {
3365                         /* RB0 disabled, RB1 enabled */
3366                         tmp = 0x11111111;
3367                 } else {
3368                         /* RB1 disabled, RB0 enabled */
3369                         tmp = 0x00000000;
3370                 }
3371         } else {
3372                 tmp = gb_addr_config & NUM_PIPES_MASK;
3373                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3374                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3375         }
3376         WREG32(GB_BACKEND_MAP, tmp);
3377
3378         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3379         WREG32(CGTS_TCC_DISABLE, 0);
3380         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3381         WREG32(CGTS_USER_TCC_DISABLE, 0);
3382
3383         /* set HW defaults for 3D engine */
3384         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3385                                      ROQ_IB2_START(0x2b)));
3386
3387         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3388
3389         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3390                              SYNC_GRADIENT |
3391                              SYNC_WALKER |
3392                              SYNC_ALIGNER));
3393
3394         sx_debug_1 = RREG32(SX_DEBUG_1);
3395         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3396         WREG32(SX_DEBUG_1, sx_debug_1);
3397
3398
3399         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3400         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3401         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3402         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3403
3404         if (rdev->family <= CHIP_SUMO2)
3405                 WREG32(SMX_SAR_CTL0, 0x00010000);
3406
3407         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3408                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3409                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3410
3411         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3412                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3413                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3414
3415         WREG32(VGT_NUM_INSTANCES, 1);
3416         WREG32(SPI_CONFIG_CNTL, 0);
3417         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3418         WREG32(CP_PERFMON_CNTL, 0);
3419
3420         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3421                                   FETCH_FIFO_HIWATER(0x4) |
3422                                   DONE_FIFO_HIWATER(0xe0) |
3423                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3424
3425         sq_config = RREG32(SQ_CONFIG);
3426         sq_config &= ~(PS_PRIO(3) |
3427                        VS_PRIO(3) |
3428                        GS_PRIO(3) |
3429                        ES_PRIO(3));
3430         sq_config |= (VC_ENABLE |
3431                       EXPORT_SRC_C |
3432                       PS_PRIO(0) |
3433                       VS_PRIO(1) |
3434                       GS_PRIO(2) |
3435                       ES_PRIO(3));
3436
3437         switch (rdev->family) {
3438         case CHIP_CEDAR:
3439         case CHIP_PALM:
3440         case CHIP_SUMO:
3441         case CHIP_SUMO2:
3442         case CHIP_CAICOS:
3443                 /* no vertex cache */
3444                 sq_config &= ~VC_ENABLE;
3445                 break;
3446         default:
3447                 break;
3448         }
3449
3450         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3451
3452         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3453         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3454         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3455         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3456         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3457         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3458         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3459
3460         switch (rdev->family) {
3461         case CHIP_CEDAR:
3462         case CHIP_PALM:
3463         case CHIP_SUMO:
3464         case CHIP_SUMO2:
3465                 ps_thread_count = 96;
3466                 break;
3467         default:
3468                 ps_thread_count = 128;
3469                 break;
3470         }
3471
3472         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3473         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3474         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3475         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3476         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3477         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3478
3479         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3480         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3481         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3482         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3483         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3484         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3485
3486         WREG32(SQ_CONFIG, sq_config);
3487         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3488         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3489         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3490         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3491         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3492         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3493         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3494         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3495         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3496         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3497
3498         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3499                                           FORCE_EOV_MAX_REZ_CNT(255)));
3500
3501         switch (rdev->family) {
3502         case CHIP_CEDAR:
3503         case CHIP_PALM:
3504         case CHIP_SUMO:
3505         case CHIP_SUMO2:
3506         case CHIP_CAICOS:
3507                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3508                 break;
3509         default:
3510                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3511                 break;
3512         }
3513         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3514         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3515
3516         WREG32(VGT_GS_VERTEX_REUSE, 16);
3517         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3518         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3519
3520         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3521         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3522
3523         WREG32(CB_PERF_CTR0_SEL_0, 0);
3524         WREG32(CB_PERF_CTR0_SEL_1, 0);
3525         WREG32(CB_PERF_CTR1_SEL_0, 0);
3526         WREG32(CB_PERF_CTR1_SEL_1, 0);
3527         WREG32(CB_PERF_CTR2_SEL_0, 0);
3528         WREG32(CB_PERF_CTR2_SEL_1, 0);
3529         WREG32(CB_PERF_CTR3_SEL_0, 0);
3530         WREG32(CB_PERF_CTR3_SEL_1, 0);
3531
3532         /* clear render buffer base addresses */
3533         WREG32(CB_COLOR0_BASE, 0);
3534         WREG32(CB_COLOR1_BASE, 0);
3535         WREG32(CB_COLOR2_BASE, 0);
3536         WREG32(CB_COLOR3_BASE, 0);
3537         WREG32(CB_COLOR4_BASE, 0);
3538         WREG32(CB_COLOR5_BASE, 0);
3539         WREG32(CB_COLOR6_BASE, 0);
3540         WREG32(CB_COLOR7_BASE, 0);
3541         WREG32(CB_COLOR8_BASE, 0);
3542         WREG32(CB_COLOR9_BASE, 0);
3543         WREG32(CB_COLOR10_BASE, 0);
3544         WREG32(CB_COLOR11_BASE, 0);
3545
3546         /* set the shader const cache sizes to 0 */
3547         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3548                 WREG32(i, 0);
3549         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3550                 WREG32(i, 0);
3551
3552         tmp = RREG32(HDP_MISC_CNTL);
3553         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3554         WREG32(HDP_MISC_CNTL, tmp);
3555
3556         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3557         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3558
3559         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3560
3561         udelay(50);
3562
3563 }
3564
3565 int evergreen_mc_init(struct radeon_device *rdev)
3566 {
3567         u32 tmp;
3568         int chansize, numchan;
3569
3570         /* Get VRAM informations */
3571         rdev->mc.vram_is_ddr = true;
3572         if ((rdev->family == CHIP_PALM) ||
3573             (rdev->family == CHIP_SUMO) ||
3574             (rdev->family == CHIP_SUMO2))
3575                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3576         else
3577                 tmp = RREG32(MC_ARB_RAMCFG);
3578         if (tmp & CHANSIZE_OVERRIDE) {
3579                 chansize = 16;
3580         } else if (tmp & CHANSIZE_MASK) {
3581                 chansize = 64;
3582         } else {
3583                 chansize = 32;
3584         }
3585         tmp = RREG32(MC_SHARED_CHMAP);
3586         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3587         case 0:
3588         default:
3589                 numchan = 1;
3590                 break;
3591         case 1:
3592                 numchan = 2;
3593                 break;
3594         case 2:
3595                 numchan = 4;
3596                 break;
3597         case 3:
3598                 numchan = 8;
3599                 break;
3600         }
3601         rdev->mc.vram_width = numchan * chansize;
3602         /* Could aper size report 0 ? */
3603         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3604         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3605         /* Setup GPU memory space */
3606         if ((rdev->family == CHIP_PALM) ||
3607             (rdev->family == CHIP_SUMO) ||
3608             (rdev->family == CHIP_SUMO2)) {
3609                 /* size in bytes on fusion */
3610                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3611                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3612         } else {
3613                 /* size in MB on evergreen/cayman/tn */
3614                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3615                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3616         }
3617         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3618         r700_vram_gtt_location(rdev, &rdev->mc);
3619         radeon_update_bandwidth_info(rdev);
3620
3621         return 0;
3622 }
3623
3624 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3625 {
3626         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3627                 RREG32(GRBM_STATUS));
3628         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3629                 RREG32(GRBM_STATUS_SE0));
3630         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3631                 RREG32(GRBM_STATUS_SE1));
3632         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3633                 RREG32(SRBM_STATUS));
3634         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3635                 RREG32(SRBM_STATUS2));
3636         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3637                 RREG32(CP_STALLED_STAT1));
3638         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3639                 RREG32(CP_STALLED_STAT2));
3640         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3641                 RREG32(CP_BUSY_STAT));
3642         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3643                 RREG32(CP_STAT));
3644         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3645                 RREG32(DMA_STATUS_REG));
3646         if (rdev->family >= CHIP_CAYMAN) {
3647                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3648                          RREG32(DMA_STATUS_REG + 0x800));
3649         }
3650 }
3651
3652 bool evergreen_is_display_hung(struct radeon_device *rdev)
3653 {
3654         u32 crtc_hung = 0;
3655         u32 crtc_status[6];
3656         u32 i, j, tmp;
3657
3658         for (i = 0; i < rdev->num_crtc; i++) {
3659                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3660                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3661                         crtc_hung |= (1 << i);
3662                 }
3663         }
3664
3665         for (j = 0; j < 10; j++) {
3666                 for (i = 0; i < rdev->num_crtc; i++) {
3667                         if (crtc_hung & (1 << i)) {
3668                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3669                                 if (tmp != crtc_status[i])
3670                                         crtc_hung &= ~(1 << i);
3671                         }
3672                 }
3673                 if (crtc_hung == 0)
3674                         return false;
3675                 udelay(100);
3676         }
3677
3678         return true;
3679 }
3680
3681 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3682 {
3683         u32 reset_mask = 0;
3684         u32 tmp;
3685
3686         /* GRBM_STATUS */
3687         tmp = RREG32(GRBM_STATUS);
3688         if (tmp & (PA_BUSY | SC_BUSY |
3689                    SH_BUSY | SX_BUSY |
3690                    TA_BUSY | VGT_BUSY |
3691                    DB_BUSY | CB_BUSY |
3692                    SPI_BUSY | VGT_BUSY_NO_DMA))
3693                 reset_mask |= RADEON_RESET_GFX;
3694
3695         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3696                    CP_BUSY | CP_COHERENCY_BUSY))
3697                 reset_mask |= RADEON_RESET_CP;
3698
3699         if (tmp & GRBM_EE_BUSY)
3700                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3701
3702         /* DMA_STATUS_REG */
3703         tmp = RREG32(DMA_STATUS_REG);
3704         if (!(tmp & DMA_IDLE))
3705                 reset_mask |= RADEON_RESET_DMA;
3706
3707         /* SRBM_STATUS2 */
3708         tmp = RREG32(SRBM_STATUS2);
3709         if (tmp & DMA_BUSY)
3710                 reset_mask |= RADEON_RESET_DMA;
3711
3712         /* SRBM_STATUS */
3713         tmp = RREG32(SRBM_STATUS);
3714         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3715                 reset_mask |= RADEON_RESET_RLC;
3716
3717         if (tmp & IH_BUSY)
3718                 reset_mask |= RADEON_RESET_IH;
3719
3720         if (tmp & SEM_BUSY)
3721                 reset_mask |= RADEON_RESET_SEM;
3722
3723         if (tmp & GRBM_RQ_PENDING)
3724                 reset_mask |= RADEON_RESET_GRBM;
3725
3726         if (tmp & VMC_BUSY)
3727                 reset_mask |= RADEON_RESET_VMC;
3728
3729         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3730                    MCC_BUSY | MCD_BUSY))
3731                 reset_mask |= RADEON_RESET_MC;
3732
3733         if (evergreen_is_display_hung(rdev))
3734                 reset_mask |= RADEON_RESET_DISPLAY;
3735
3736         /* VM_L2_STATUS */
3737         tmp = RREG32(VM_L2_STATUS);
3738         if (tmp & L2_BUSY)
3739                 reset_mask |= RADEON_RESET_VMC;
3740
3741         /* Skip MC reset as it's mostly likely not hung, just busy */
3742         if (reset_mask & RADEON_RESET_MC) {
3743                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3744                 reset_mask &= ~RADEON_RESET_MC;
3745         }
3746
3747         return reset_mask;
3748 }
3749
3750 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3751 {
3752         struct evergreen_mc_save save;
3753         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3754         u32 tmp;
3755
3756         if (reset_mask == 0)
3757                 return;
3758
3759         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3760
3761         evergreen_print_gpu_status_regs(rdev);
3762
3763         /* Disable CP parsing/prefetching */
3764         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3765
3766         if (reset_mask & RADEON_RESET_DMA) {
3767                 /* Disable DMA */
3768                 tmp = RREG32(DMA_RB_CNTL);
3769                 tmp &= ~DMA_RB_ENABLE;
3770                 WREG32(DMA_RB_CNTL, tmp);
3771         }
3772
3773         udelay(50);
3774
3775         evergreen_mc_stop(rdev, &save);
3776         if (evergreen_mc_wait_for_idle(rdev)) {
3777                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3778         }
3779
3780         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3781                 grbm_soft_reset |= SOFT_RESET_DB |
3782                         SOFT_RESET_CB |
3783                         SOFT_RESET_PA |
3784                         SOFT_RESET_SC |
3785                         SOFT_RESET_SPI |
3786                         SOFT_RESET_SX |
3787                         SOFT_RESET_SH |
3788                         SOFT_RESET_TC |
3789                         SOFT_RESET_TA |
3790                         SOFT_RESET_VC |
3791                         SOFT_RESET_VGT;
3792         }
3793
3794         if (reset_mask & RADEON_RESET_CP) {
3795                 grbm_soft_reset |= SOFT_RESET_CP |
3796                         SOFT_RESET_VGT;
3797
3798                 srbm_soft_reset |= SOFT_RESET_GRBM;
3799         }
3800
3801         if (reset_mask & RADEON_RESET_DMA)
3802                 srbm_soft_reset |= SOFT_RESET_DMA;
3803
3804         if (reset_mask & RADEON_RESET_DISPLAY)
3805                 srbm_soft_reset |= SOFT_RESET_DC;
3806
3807         if (reset_mask & RADEON_RESET_RLC)
3808                 srbm_soft_reset |= SOFT_RESET_RLC;
3809
3810         if (reset_mask & RADEON_RESET_SEM)
3811                 srbm_soft_reset |= SOFT_RESET_SEM;
3812
3813         if (reset_mask & RADEON_RESET_IH)
3814                 srbm_soft_reset |= SOFT_RESET_IH;
3815
3816         if (reset_mask & RADEON_RESET_GRBM)
3817                 srbm_soft_reset |= SOFT_RESET_GRBM;
3818
3819         if (reset_mask & RADEON_RESET_VMC)
3820                 srbm_soft_reset |= SOFT_RESET_VMC;
3821
3822         if (!(rdev->flags & RADEON_IS_IGP)) {
3823                 if (reset_mask & RADEON_RESET_MC)
3824                         srbm_soft_reset |= SOFT_RESET_MC;
3825         }
3826
3827         if (grbm_soft_reset) {
3828                 tmp = RREG32(GRBM_SOFT_RESET);
3829                 tmp |= grbm_soft_reset;
3830                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3831                 WREG32(GRBM_SOFT_RESET, tmp);
3832                 tmp = RREG32(GRBM_SOFT_RESET);
3833
3834                 udelay(50);
3835
3836                 tmp &= ~grbm_soft_reset;
3837                 WREG32(GRBM_SOFT_RESET, tmp);
3838                 tmp = RREG32(GRBM_SOFT_RESET);
3839         }
3840
3841         if (srbm_soft_reset) {
3842                 tmp = RREG32(SRBM_SOFT_RESET);
3843                 tmp |= srbm_soft_reset;
3844                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3845                 WREG32(SRBM_SOFT_RESET, tmp);
3846                 tmp = RREG32(SRBM_SOFT_RESET);
3847
3848                 udelay(50);
3849
3850                 tmp &= ~srbm_soft_reset;
3851                 WREG32(SRBM_SOFT_RESET, tmp);
3852                 tmp = RREG32(SRBM_SOFT_RESET);
3853         }
3854
3855         /* Wait a little for things to settle down */
3856         udelay(50);
3857
3858         evergreen_mc_resume(rdev, &save);
3859         udelay(50);
3860
3861         evergreen_print_gpu_status_regs(rdev);
3862 }
3863
3864 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3865 {
3866         struct evergreen_mc_save save;
3867         u32 tmp, i;
3868
3869         dev_info(rdev->dev, "GPU pci config reset\n");
3870
3871         /* disable dpm? */
3872
3873         /* Disable CP parsing/prefetching */
3874         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3875         udelay(50);
3876         /* Disable DMA */
3877         tmp = RREG32(DMA_RB_CNTL);
3878         tmp &= ~DMA_RB_ENABLE;
3879         WREG32(DMA_RB_CNTL, tmp);
3880         /* XXX other engines? */
3881
3882         /* halt the rlc */
3883         r600_rlc_stop(rdev);
3884
3885         udelay(50);
3886
3887         /* set mclk/sclk to bypass */
3888         rv770_set_clk_bypass_mode(rdev);
3889         /* disable BM */
3890         pci_clear_master(rdev->pdev);
3891         /* disable mem access */
3892         evergreen_mc_stop(rdev, &save);
3893         if (evergreen_mc_wait_for_idle(rdev)) {
3894                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3895         }
3896         /* reset */
3897         radeon_pci_config_reset(rdev);
3898         /* wait for asic to come out of reset */
3899         for (i = 0; i < rdev->usec_timeout; i++) {
3900                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3901                         break;
3902                 udelay(1);
3903         }
3904 }
3905
3906 int evergreen_asic_reset(struct radeon_device *rdev)
3907 {
3908         u32 reset_mask;
3909
3910         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3911
3912         if (reset_mask)
3913                 r600_set_bios_scratch_engine_hung(rdev, true);
3914
3915         /* try soft reset */
3916         evergreen_gpu_soft_reset(rdev, reset_mask);
3917
3918         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3919
3920         /* try pci config reset */
3921         if (reset_mask && radeon_hard_reset)
3922                 evergreen_gpu_pci_config_reset(rdev);
3923
3924         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3925
3926         if (!reset_mask)
3927                 r600_set_bios_scratch_engine_hung(rdev, false);
3928
3929         return 0;
3930 }
3931
3932 /**
3933  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3934  *
3935  * @rdev: radeon_device pointer
3936  * @ring: radeon_ring structure holding ring information
3937  *
3938  * Check if the GFX engine is locked up.
3939  * Returns true if the engine appears to be locked up, false if not.
3940  */
3941 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3942 {
3943         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3944
3945         if (!(reset_mask & (RADEON_RESET_GFX |
3946                             RADEON_RESET_COMPUTE |
3947                             RADEON_RESET_CP))) {
3948                 radeon_ring_lockup_update(rdev, ring);
3949                 return false;
3950         }
3951         return radeon_ring_test_lockup(rdev, ring);
3952 }
3953
3954 /*
3955  * RLC
3956  */
3957 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
3958 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
3959
3960 void sumo_rlc_fini(struct radeon_device *rdev)
3961 {
3962         int r;
3963
3964         /* save restore block */
3965         if (rdev->rlc.save_restore_obj) {
3966                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3967                 if (unlikely(r != 0))
3968                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3969                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3970                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3971
3972                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3973                 rdev->rlc.save_restore_obj = NULL;
3974         }
3975
3976         /* clear state block */
3977         if (rdev->rlc.clear_state_obj) {
3978                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3979                 if (unlikely(r != 0))
3980                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3981                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3982                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3983
3984                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3985                 rdev->rlc.clear_state_obj = NULL;
3986         }
3987
3988         /* clear state block */
3989         if (rdev->rlc.cp_table_obj) {
3990                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
3991                 if (unlikely(r != 0))
3992                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
3993                 radeon_bo_unpin(rdev->rlc.cp_table_obj);
3994                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
3995
3996                 radeon_bo_unref(&rdev->rlc.cp_table_obj);
3997                 rdev->rlc.cp_table_obj = NULL;
3998         }
3999 }
4000
4001 #define CP_ME_TABLE_SIZE    96
4002
4003 int sumo_rlc_init(struct radeon_device *rdev)
4004 {
4005         const u32 *src_ptr;
4006         volatile u32 *dst_ptr;
4007         u32 dws, data, i, j, k, reg_num;
4008         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4009         u64 reg_list_mc_addr;
4010         const struct cs_section_def *cs_data;
4011         int r;
4012
4013         src_ptr = rdev->rlc.reg_list;
4014         dws = rdev->rlc.reg_list_size;
4015         if (rdev->family >= CHIP_BONAIRE) {
4016                 dws += (5 * 16) + 48 + 48 + 64;
4017         }
4018         cs_data = rdev->rlc.cs_data;
4019
4020         if (src_ptr) {
4021                 /* save restore block */
4022                 if (rdev->rlc.save_restore_obj == NULL) {
4023                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4024                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4025                                              NULL, &rdev->rlc.save_restore_obj);
4026                         if (r) {
4027                                 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4028                                 return r;
4029                         }
4030                 }
4031
4032                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4033                 if (unlikely(r != 0)) {
4034                         sumo_rlc_fini(rdev);
4035                         return r;
4036                 }
4037                 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4038                                   &rdev->rlc.save_restore_gpu_addr);
4039                 if (r) {
4040                         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4041                         dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4042                         sumo_rlc_fini(rdev);
4043                         return r;
4044                 }
4045
4046                 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4047                 if (r) {
4048                         dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4049                         sumo_rlc_fini(rdev);
4050                         return r;
4051                 }
4052                 /* write the sr buffer */
4053                 dst_ptr = rdev->rlc.sr_ptr;
4054                 if (rdev->family >= CHIP_TAHITI) {
4055                         /* SI */
4056                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
4057                                 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4058                 } else {
4059                         /* ON/LN/TN */
4060                         /* format:
4061                          * dw0: (reg2 << 16) | reg1
4062                          * dw1: reg1 save space
4063                          * dw2: reg2 save space
4064                          */
4065                         for (i = 0; i < dws; i++) {
4066                                 data = src_ptr[i] >> 2;
4067                                 i++;
4068                                 if (i < dws)
4069                                         data |= (src_ptr[i] >> 2) << 16;
4070                                 j = (((i - 1) * 3) / 2);
4071                                 dst_ptr[j] = cpu_to_le32(data);
4072                         }
4073                         j = ((i * 3) / 2);
4074                         dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4075                 }
4076                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4077                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4078         }
4079
4080         if (cs_data) {
4081                 /* clear state block */
4082                 if (rdev->family >= CHIP_BONAIRE) {
4083                         rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4084                 } else if (rdev->family >= CHIP_TAHITI) {
4085                         rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4086                         dws = rdev->rlc.clear_state_size + (256 / 4);
4087                 } else {
4088                         reg_list_num = 0;
4089                         dws = 0;
4090                         for (i = 0; cs_data[i].section != NULL; i++) {
4091                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4092                                         reg_list_num++;
4093                                         dws += cs_data[i].section[j].reg_count;
4094                                 }
4095                         }
4096                         reg_list_blk_index = (3 * reg_list_num + 2);
4097                         dws += reg_list_blk_index;
4098                         rdev->rlc.clear_state_size = dws;
4099                 }
4100
4101                 if (rdev->rlc.clear_state_obj == NULL) {
4102                         r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4103                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4104                                              NULL, &rdev->rlc.clear_state_obj);
4105                         if (r) {
4106                                 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4107                                 sumo_rlc_fini(rdev);
4108                                 return r;
4109                         }
4110                 }
4111                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4112                 if (unlikely(r != 0)) {
4113                         sumo_rlc_fini(rdev);
4114                         return r;
4115                 }
4116                 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4117                                   &rdev->rlc.clear_state_gpu_addr);
4118                 if (r) {
4119                         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4120                         dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4121                         sumo_rlc_fini(rdev);
4122                         return r;
4123                 }
4124
4125                 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4126                 if (r) {
4127                         dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4128                         sumo_rlc_fini(rdev);
4129                         return r;
4130                 }
4131                 /* set up the cs buffer */
4132                 dst_ptr = rdev->rlc.cs_ptr;
4133                 if (rdev->family >= CHIP_BONAIRE) {
4134                         cik_get_csb_buffer(rdev, dst_ptr);
4135                 } else if (rdev->family >= CHIP_TAHITI) {
4136                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4137                         dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4138                         dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4139                         dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4140                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4141                 } else {
4142                         reg_list_hdr_blk_index = 0;
4143                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4144                         data = upper_32_bits(reg_list_mc_addr);
4145                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4146                         reg_list_hdr_blk_index++;
4147                         for (i = 0; cs_data[i].section != NULL; i++) {
4148                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4149                                         reg_num = cs_data[i].section[j].reg_count;
4150                                         data = reg_list_mc_addr & 0xffffffff;
4151                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4152                                         reg_list_hdr_blk_index++;
4153
4154                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4155                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4156                                         reg_list_hdr_blk_index++;
4157
4158                                         data = 0x08000000 | (reg_num * 4);
4159                                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4160                                         reg_list_hdr_blk_index++;
4161
4162                                         for (k = 0; k < reg_num; k++) {
4163                                                 data = cs_data[i].section[j].extent[k];
4164                                                 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4165                                         }
4166                                         reg_list_mc_addr += reg_num * 4;
4167                                         reg_list_blk_index += reg_num;
4168                                 }
4169                         }
4170                         dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4171                 }
4172                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4173                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4174         }
4175
4176         if (rdev->rlc.cp_table_size) {
4177                 if (rdev->rlc.cp_table_obj == NULL) {
4178                         r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4179                                              PAGE_SIZE, true,
4180                                              RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4181                                              NULL, &rdev->rlc.cp_table_obj);
4182                         if (r) {
4183                                 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4184                                 sumo_rlc_fini(rdev);
4185                                 return r;
4186                         }
4187                 }
4188
4189                 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4190                 if (unlikely(r != 0)) {
4191                         dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4192                         sumo_rlc_fini(rdev);
4193                         return r;
4194                 }
4195                 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4196                                   &rdev->rlc.cp_table_gpu_addr);
4197                 if (r) {
4198                         radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4199                         dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4200                         sumo_rlc_fini(rdev);
4201                         return r;
4202                 }
4203                 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4204                 if (r) {
4205                         dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4206                         sumo_rlc_fini(rdev);
4207                         return r;
4208                 }
4209
4210                 cik_init_cp_pg_table(rdev);
4211
4212                 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4213                 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4214
4215         }
4216
4217         return 0;
4218 }
4219
4220 static void evergreen_rlc_start(struct radeon_device *rdev)
4221 {
4222         u32 mask = RLC_ENABLE;
4223
4224         if (rdev->flags & RADEON_IS_IGP) {
4225                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4226         }
4227
4228         WREG32(RLC_CNTL, mask);
4229 }
4230
4231 int evergreen_rlc_resume(struct radeon_device *rdev)
4232 {
4233         u32 i;
4234         const __be32 *fw_data;
4235
4236         if (!rdev->rlc_fw)
4237                 return -EINVAL;
4238
4239         r600_rlc_stop(rdev);
4240
4241         WREG32(RLC_HB_CNTL, 0);
4242
4243         if (rdev->flags & RADEON_IS_IGP) {
4244                 if (rdev->family == CHIP_ARUBA) {
4245                         u32 always_on_bitmap =
4246                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4247                         /* find out the number of active simds */
4248                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4249                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4250                         tmp = hweight32(~tmp);
4251                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4252                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4253                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4254                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4255                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4256                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4257                         }
4258                 } else {
4259                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4260                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4261                 }
4262                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4263                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4264         } else {
4265                 WREG32(RLC_HB_BASE, 0);
4266                 WREG32(RLC_HB_RPTR, 0);
4267                 WREG32(RLC_HB_WPTR, 0);
4268                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4269                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4270         }
4271         WREG32(RLC_MC_CNTL, 0);
4272         WREG32(RLC_UCODE_CNTL, 0);
4273
4274         fw_data = (const __be32 *)rdev->rlc_fw->data;
4275         if (rdev->family >= CHIP_ARUBA) {
4276                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4277                         WREG32(RLC_UCODE_ADDR, i);
4278                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4279                 }
4280         } else if (rdev->family >= CHIP_CAYMAN) {
4281                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4282                         WREG32(RLC_UCODE_ADDR, i);
4283                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4284                 }
4285         } else {
4286                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4287                         WREG32(RLC_UCODE_ADDR, i);
4288                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289                 }
4290         }
4291         WREG32(RLC_UCODE_ADDR, 0);
4292
4293         evergreen_rlc_start(rdev);
4294
4295         return 0;
4296 }
4297
4298 /* Interrupts */
4299
4300 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4301 {
4302         if (crtc >= rdev->num_crtc)
4303                 return 0;
4304         else
4305                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4306 }
4307
4308 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4309 {
4310         u32 tmp;
4311
4312         if (rdev->family >= CHIP_CAYMAN) {
4313                 cayman_cp_int_cntl_setup(rdev, 0,
4314                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4315                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4316                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4317                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4318                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4319         } else
4320                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4321         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4322         WREG32(DMA_CNTL, tmp);
4323         WREG32(GRBM_INT_CNTL, 0);
4324         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4325         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4326         if (rdev->num_crtc >= 4) {
4327                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4328                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4329         }
4330         if (rdev->num_crtc >= 6) {
4331                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4332                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4333         }
4334
4335         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4336         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4337         if (rdev->num_crtc >= 4) {
4338                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4339                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4340         }
4341         if (rdev->num_crtc >= 6) {
4342                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4343                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4344         }
4345
4346         /* only one DAC on DCE5 */
4347         if (!ASIC_IS_DCE5(rdev))
4348                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4349         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4350
4351         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4352         WREG32(DC_HPD1_INT_CONTROL, tmp);
4353         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4354         WREG32(DC_HPD2_INT_CONTROL, tmp);
4355         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4356         WREG32(DC_HPD3_INT_CONTROL, tmp);
4357         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4358         WREG32(DC_HPD4_INT_CONTROL, tmp);
4359         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4360         WREG32(DC_HPD5_INT_CONTROL, tmp);
4361         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4362         WREG32(DC_HPD6_INT_CONTROL, tmp);
4363
4364 }
4365
4366 int evergreen_irq_set(struct radeon_device *rdev)
4367 {
4368         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4369         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4370         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4371         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4372         u32 grbm_int_cntl = 0;
4373         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4374         u32 dma_cntl, dma_cntl1 = 0;
4375         u32 thermal_int = 0;
4376
4377         if (!rdev->irq.installed) {
4378                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4379                 return -EINVAL;
4380         }
4381         /* don't enable anything if the ih is disabled */
4382         if (!rdev->ih.enabled) {
4383                 r600_disable_interrupts(rdev);
4384                 /* force the active interrupt state to all disabled */
4385                 evergreen_disable_interrupt_state(rdev);
4386                 return 0;
4387         }
4388
4389         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4390         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4393         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4394         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4395         if (rdev->family == CHIP_ARUBA)
4396                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4397                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4398         else
4399                 thermal_int = RREG32(CG_THERMAL_INT) &
4400                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4401
4402         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4403         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4404         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4405         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4406         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4407         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4408
4409         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4410
4411         if (rdev->family >= CHIP_CAYMAN) {
4412                 /* enable CP interrupts on all rings */
4413                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4414                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4415                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4416                 }
4417                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4418                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4419                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4420                 }
4421                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4422                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4423                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4424                 }
4425         } else {
4426                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4427                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4428                         cp_int_cntl |= RB_INT_ENABLE;
4429                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4430                 }
4431         }
4432
4433         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4434                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4435                 dma_cntl |= TRAP_ENABLE;
4436         }
4437
4438         if (rdev->family >= CHIP_CAYMAN) {
4439                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4440                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4441                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4442                         dma_cntl1 |= TRAP_ENABLE;
4443                 }
4444         }
4445
4446         if (rdev->irq.dpm_thermal) {
4447                 DRM_DEBUG("dpm thermal\n");
4448                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4449         }
4450
4451         if (rdev->irq.crtc_vblank_int[0] ||
4452             atomic_read(&rdev->irq.pflip[0])) {
4453                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4454                 crtc1 |= VBLANK_INT_MASK;
4455         }
4456         if (rdev->irq.crtc_vblank_int[1] ||
4457             atomic_read(&rdev->irq.pflip[1])) {
4458                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4459                 crtc2 |= VBLANK_INT_MASK;
4460         }
4461         if (rdev->irq.crtc_vblank_int[2] ||
4462             atomic_read(&rdev->irq.pflip[2])) {
4463                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4464                 crtc3 |= VBLANK_INT_MASK;
4465         }
4466         if (rdev->irq.crtc_vblank_int[3] ||
4467             atomic_read(&rdev->irq.pflip[3])) {
4468                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4469                 crtc4 |= VBLANK_INT_MASK;
4470         }
4471         if (rdev->irq.crtc_vblank_int[4] ||
4472             atomic_read(&rdev->irq.pflip[4])) {
4473                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4474                 crtc5 |= VBLANK_INT_MASK;
4475         }
4476         if (rdev->irq.crtc_vblank_int[5] ||
4477             atomic_read(&rdev->irq.pflip[5])) {
4478                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4479                 crtc6 |= VBLANK_INT_MASK;
4480         }
4481         if (rdev->irq.hpd[0]) {
4482                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4483                 hpd1 |= DC_HPDx_INT_EN;
4484         }
4485         if (rdev->irq.hpd[1]) {
4486                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4487                 hpd2 |= DC_HPDx_INT_EN;
4488         }
4489         if (rdev->irq.hpd[2]) {
4490                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4491                 hpd3 |= DC_HPDx_INT_EN;
4492         }
4493         if (rdev->irq.hpd[3]) {
4494                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4495                 hpd4 |= DC_HPDx_INT_EN;
4496         }
4497         if (rdev->irq.hpd[4]) {
4498                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4499                 hpd5 |= DC_HPDx_INT_EN;
4500         }
4501         if (rdev->irq.hpd[5]) {
4502                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4503                 hpd6 |= DC_HPDx_INT_EN;
4504         }
4505         if (rdev->irq.afmt[0]) {
4506                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4507                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4508         }
4509         if (rdev->irq.afmt[1]) {
4510                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4511                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4512         }
4513         if (rdev->irq.afmt[2]) {
4514                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4515                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4516         }
4517         if (rdev->irq.afmt[3]) {
4518                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4519                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4520         }
4521         if (rdev->irq.afmt[4]) {
4522                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4523                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4524         }
4525         if (rdev->irq.afmt[5]) {
4526                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4527                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4528         }
4529
4530         if (rdev->family >= CHIP_CAYMAN) {
4531                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4532                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4533                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4534         } else
4535                 WREG32(CP_INT_CNTL, cp_int_cntl);
4536
4537         WREG32(DMA_CNTL, dma_cntl);
4538
4539         if (rdev->family >= CHIP_CAYMAN)
4540                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4541
4542         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4543
4544         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4545         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4546         if (rdev->num_crtc >= 4) {
4547                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4548                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4549         }
4550         if (rdev->num_crtc >= 6) {
4551                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4552                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4553         }
4554
4555         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4556                GRPH_PFLIP_INT_MASK);
4557         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4558                GRPH_PFLIP_INT_MASK);
4559         if (rdev->num_crtc >= 4) {
4560                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4561                        GRPH_PFLIP_INT_MASK);
4562                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4563                        GRPH_PFLIP_INT_MASK);
4564         }
4565         if (rdev->num_crtc >= 6) {
4566                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4567                        GRPH_PFLIP_INT_MASK);
4568                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4569                        GRPH_PFLIP_INT_MASK);
4570         }
4571
4572         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4573         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4574         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4575         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4576         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4577         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4578         if (rdev->family == CHIP_ARUBA)
4579                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4580         else
4581                 WREG32(CG_THERMAL_INT, thermal_int);
4582
4583         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4584         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4585         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4586         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4587         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4588         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4589
4590         return 0;
4591 }
4592
4593 static void evergreen_irq_ack(struct radeon_device *rdev)
4594 {
4595         u32 tmp;
4596
4597         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4598         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4599         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4600         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4601         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4602         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4603         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4604         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4605         if (rdev->num_crtc >= 4) {
4606                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4607                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4608         }
4609         if (rdev->num_crtc >= 6) {
4610                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4611                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4612         }
4613
4614         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4615         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4616         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4617         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4618         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4619         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4620
4621         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4622                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4623         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4624                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4625         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4626                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4627         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4628                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4629         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4630                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4631         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4632                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4633
4634         if (rdev->num_crtc >= 4) {
4635                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4636                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4637                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4638                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4639                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4640                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4641                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4642                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4643                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4644                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4645                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4646                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4647         }
4648
4649         if (rdev->num_crtc >= 6) {
4650                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4651                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4652                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4653                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4654                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4655                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4656                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4657                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4658                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4659                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4660                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4661                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4662         }
4663
4664         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4665                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4666                 tmp |= DC_HPDx_INT_ACK;
4667                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4668         }
4669         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4670                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4671                 tmp |= DC_HPDx_INT_ACK;
4672                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4673         }
4674         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4675                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4676                 tmp |= DC_HPDx_INT_ACK;
4677                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4678         }
4679         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4680                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4681                 tmp |= DC_HPDx_INT_ACK;
4682                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4683         }
4684         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4685                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4686                 tmp |= DC_HPDx_INT_ACK;
4687                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4688         }
4689         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4690                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4691                 tmp |= DC_HPDx_INT_ACK;
4692                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4693         }
4694         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4695                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4696                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4697                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4698         }
4699         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4700                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4701                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4702                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4703         }
4704         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4705                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4706                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4707                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4708         }
4709         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4710                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4711                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4712                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4713         }
4714         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4715                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4716                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4717                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4718         }
4719         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4720                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4721                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4722                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4723         }
4724 }
4725
4726 static void evergreen_irq_disable(struct radeon_device *rdev)
4727 {
4728         r600_disable_interrupts(rdev);
4729         /* Wait and acknowledge irq */
4730         mdelay(1);
4731         evergreen_irq_ack(rdev);
4732         evergreen_disable_interrupt_state(rdev);
4733 }
4734
4735 void evergreen_irq_suspend(struct radeon_device *rdev)
4736 {
4737         evergreen_irq_disable(rdev);
4738         r600_rlc_stop(rdev);
4739 }
4740
4741 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4742 {
4743         u32 wptr, tmp;
4744
4745         if (rdev->wb.enabled)
4746                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4747         else
4748                 wptr = RREG32(IH_RB_WPTR);
4749
4750         if (wptr & RB_OVERFLOW) {
4751                 wptr &= ~RB_OVERFLOW;
4752                 /* When a ring buffer overflow happen start parsing interrupt
4753                  * from the last not overwritten vector (wptr + 16). Hopefully
4754                  * this should allow us to catchup.
4755                  */
4756                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4757                          wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4758                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4759                 tmp = RREG32(IH_RB_CNTL);
4760                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4761                 WREG32(IH_RB_CNTL, tmp);
4762         }
4763         return (wptr & rdev->ih.ptr_mask);
4764 }
4765
4766 int evergreen_irq_process(struct radeon_device *rdev)
4767 {
4768         u32 wptr;
4769         u32 rptr;
4770         u32 src_id, src_data;
4771         u32 ring_index;
4772         bool queue_hotplug = false;
4773         bool queue_hdmi = false;
4774         bool queue_thermal = false;
4775         u32 status, addr;
4776
4777         if (!rdev->ih.enabled || rdev->shutdown)
4778                 return IRQ_NONE;
4779
4780         wptr = evergreen_get_ih_wptr(rdev);
4781
4782 restart_ih:
4783         /* is somebody else already processing irqs? */
4784         if (atomic_xchg(&rdev->ih.lock, 1))
4785                 return IRQ_NONE;
4786
4787         rptr = rdev->ih.rptr;
4788         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4789
4790         /* Order reading of wptr vs. reading of IH ring data */
4791         rmb();
4792
4793         /* display interrupts */
4794         evergreen_irq_ack(rdev);
4795
4796         while (rptr != wptr) {
4797                 /* wptr/rptr are in bytes! */
4798                 ring_index = rptr / 4;
4799                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4800                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4801
4802                 switch (src_id) {
4803                 case 1: /* D1 vblank/vline */
4804                         switch (src_data) {
4805                         case 0: /* D1 vblank */
4806                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4807                                         if (rdev->irq.crtc_vblank_int[0]) {
4808                                                 drm_handle_vblank(rdev->ddev, 0);
4809                                                 rdev->pm.vblank_sync = true;
4810                                                 wake_up(&rdev->irq.vblank_queue);
4811                                         }
4812                                         if (atomic_read(&rdev->irq.pflip[0]))
4813                                                 radeon_crtc_handle_vblank(rdev, 0);
4814                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4815                                         DRM_DEBUG("IH: D1 vblank\n");
4816                                 }
4817                                 break;
4818                         case 1: /* D1 vline */
4819                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4820                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4821                                         DRM_DEBUG("IH: D1 vline\n");
4822                                 }
4823                                 break;
4824                         default:
4825                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4826                                 break;
4827                         }
4828                         break;
4829                 case 2: /* D2 vblank/vline */
4830                         switch (src_data) {
4831                         case 0: /* D2 vblank */
4832                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4833                                         if (rdev->irq.crtc_vblank_int[1]) {
4834                                                 drm_handle_vblank(rdev->ddev, 1);
4835                                                 rdev->pm.vblank_sync = true;
4836                                                 wake_up(&rdev->irq.vblank_queue);
4837                                         }
4838                                         if (atomic_read(&rdev->irq.pflip[1]))
4839                                                 radeon_crtc_handle_vblank(rdev, 1);
4840                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4841                                         DRM_DEBUG("IH: D2 vblank\n");
4842                                 }
4843                                 break;
4844                         case 1: /* D2 vline */
4845                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4846                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4847                                         DRM_DEBUG("IH: D2 vline\n");
4848                                 }
4849                                 break;
4850                         default:
4851                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4852                                 break;
4853                         }
4854                         break;
4855                 case 3: /* D3 vblank/vline */
4856                         switch (src_data) {
4857                         case 0: /* D3 vblank */
4858                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4859                                         if (rdev->irq.crtc_vblank_int[2]) {
4860                                                 drm_handle_vblank(rdev->ddev, 2);
4861                                                 rdev->pm.vblank_sync = true;
4862                                                 wake_up(&rdev->irq.vblank_queue);
4863                                         }
4864                                         if (atomic_read(&rdev->irq.pflip[2]))
4865                                                 radeon_crtc_handle_vblank(rdev, 2);
4866                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4867                                         DRM_DEBUG("IH: D3 vblank\n");
4868                                 }
4869                                 break;
4870                         case 1: /* D3 vline */
4871                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4872                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4873                                         DRM_DEBUG("IH: D3 vline\n");
4874                                 }
4875                                 break;
4876                         default:
4877                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4878                                 break;
4879                         }
4880                         break;
4881                 case 4: /* D4 vblank/vline */
4882                         switch (src_data) {
4883                         case 0: /* D4 vblank */
4884                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4885                                         if (rdev->irq.crtc_vblank_int[3]) {
4886                                                 drm_handle_vblank(rdev->ddev, 3);
4887                                                 rdev->pm.vblank_sync = true;
4888                                                 wake_up(&rdev->irq.vblank_queue);
4889                                         }
4890                                         if (atomic_read(&rdev->irq.pflip[3]))
4891                                                 radeon_crtc_handle_vblank(rdev, 3);
4892                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4893                                         DRM_DEBUG("IH: D4 vblank\n");
4894                                 }
4895                                 break;
4896                         case 1: /* D4 vline */
4897                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4898                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4899                                         DRM_DEBUG("IH: D4 vline\n");
4900                                 }
4901                                 break;
4902                         default:
4903                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4904                                 break;
4905                         }
4906                         break;
4907                 case 5: /* D5 vblank/vline */
4908                         switch (src_data) {
4909                         case 0: /* D5 vblank */
4910                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4911                                         if (rdev->irq.crtc_vblank_int[4]) {
4912                                                 drm_handle_vblank(rdev->ddev, 4);
4913                                                 rdev->pm.vblank_sync = true;
4914                                                 wake_up(&rdev->irq.vblank_queue);
4915                                         }
4916                                         if (atomic_read(&rdev->irq.pflip[4]))
4917                                                 radeon_crtc_handle_vblank(rdev, 4);
4918                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4919                                         DRM_DEBUG("IH: D5 vblank\n");
4920                                 }
4921                                 break;
4922                         case 1: /* D5 vline */
4923                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4924                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4925                                         DRM_DEBUG("IH: D5 vline\n");
4926                                 }
4927                                 break;
4928                         default:
4929                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4930                                 break;
4931                         }
4932                         break;
4933                 case 6: /* D6 vblank/vline */
4934                         switch (src_data) {
4935                         case 0: /* D6 vblank */
4936                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4937                                         if (rdev->irq.crtc_vblank_int[5]) {
4938                                                 drm_handle_vblank(rdev->ddev, 5);
4939                                                 rdev->pm.vblank_sync = true;
4940                                                 wake_up(&rdev->irq.vblank_queue);
4941                                         }
4942                                         if (atomic_read(&rdev->irq.pflip[5]))
4943                                                 radeon_crtc_handle_vblank(rdev, 5);
4944                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4945                                         DRM_DEBUG("IH: D6 vblank\n");
4946                                 }
4947                                 break;
4948                         case 1: /* D6 vline */
4949                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4950                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4951                                         DRM_DEBUG("IH: D6 vline\n");
4952                                 }
4953                                 break;
4954                         default:
4955                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4956                                 break;
4957                         }
4958                         break;
4959                 case 8: /* D1 page flip */
4960                 case 10: /* D2 page flip */
4961                 case 12: /* D3 page flip */
4962                 case 14: /* D4 page flip */
4963                 case 16: /* D5 page flip */
4964                 case 18: /* D6 page flip */
4965                         DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4966                         if (radeon_use_pflipirq > 0)
4967                                 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4968                         break;
4969                 case 42: /* HPD hotplug */
4970                         switch (src_data) {
4971                         case 0:
4972                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4973                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4974                                         queue_hotplug = true;
4975                                         DRM_DEBUG("IH: HPD1\n");
4976                                 }
4977                                 break;
4978                         case 1:
4979                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4980                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4981                                         queue_hotplug = true;
4982                                         DRM_DEBUG("IH: HPD2\n");
4983                                 }
4984                                 break;
4985                         case 2:
4986                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4987                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4988                                         queue_hotplug = true;
4989                                         DRM_DEBUG("IH: HPD3\n");
4990                                 }
4991                                 break;
4992                         case 3:
4993                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4994                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4995                                         queue_hotplug = true;
4996                                         DRM_DEBUG("IH: HPD4\n");
4997                                 }
4998                                 break;
4999                         case 4:
5000                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5001                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5002                                         queue_hotplug = true;
5003                                         DRM_DEBUG("IH: HPD5\n");
5004                                 }
5005                                 break;
5006                         case 5:
5007                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5008                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5009                                         queue_hotplug = true;
5010                                         DRM_DEBUG("IH: HPD6\n");
5011                                 }
5012                                 break;
5013                         default:
5014                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5015                                 break;
5016                         }
5017                         break;
5018                 case 44: /* hdmi */
5019                         switch (src_data) {
5020                         case 0:
5021                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5022                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5023                                         queue_hdmi = true;
5024                                         DRM_DEBUG("IH: HDMI0\n");
5025                                 }
5026                                 break;
5027                         case 1:
5028                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5029                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5030                                         queue_hdmi = true;
5031                                         DRM_DEBUG("IH: HDMI1\n");
5032                                 }
5033                                 break;
5034                         case 2:
5035                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5036                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5037                                         queue_hdmi = true;
5038                                         DRM_DEBUG("IH: HDMI2\n");
5039                                 }
5040                                 break;
5041                         case 3:
5042                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5043                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5044                                         queue_hdmi = true;
5045                                         DRM_DEBUG("IH: HDMI3\n");
5046                                 }
5047                                 break;
5048                         case 4:
5049                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5050                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5051                                         queue_hdmi = true;
5052                                         DRM_DEBUG("IH: HDMI4\n");
5053                                 }
5054                                 break;
5055                         case 5:
5056                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5057                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5058                                         queue_hdmi = true;
5059                                         DRM_DEBUG("IH: HDMI5\n");
5060                                 }
5061                                 break;
5062                         default:
5063                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5064                                 break;
5065                         }
5066                 case 124: /* UVD */
5067                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5068                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5069                         break;
5070                 case 146:
5071                 case 147:
5072                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5073                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5074                         /* reset addr and status */
5075                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5076                         if (addr == 0x0 && status == 0x0)
5077                                 break;
5078                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5079                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
5080                                 addr);
5081                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5082                                 status);
5083                         cayman_vm_decode_fault(rdev, status, addr);
5084                         break;
5085                 case 176: /* CP_INT in ring buffer */
5086                 case 177: /* CP_INT in IB1 */
5087                 case 178: /* CP_INT in IB2 */
5088                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5089                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5090                         break;
5091                 case 181: /* CP EOP event */
5092                         DRM_DEBUG("IH: CP EOP\n");
5093                         if (rdev->family >= CHIP_CAYMAN) {
5094                                 switch (src_data) {
5095                                 case 0:
5096                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5097                                         break;
5098                                 case 1:
5099                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5100                                         break;
5101                                 case 2:
5102                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5103                                         break;
5104                                 }
5105                         } else
5106                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5107                         break;
5108                 case 224: /* DMA trap event */
5109                         DRM_DEBUG("IH: DMA trap\n");
5110                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5111                         break;
5112                 case 230: /* thermal low to high */
5113                         DRM_DEBUG("IH: thermal low to high\n");
5114                         rdev->pm.dpm.thermal.high_to_low = false;
5115                         queue_thermal = true;
5116                         break;
5117                 case 231: /* thermal high to low */
5118                         DRM_DEBUG("IH: thermal high to low\n");
5119                         rdev->pm.dpm.thermal.high_to_low = true;
5120                         queue_thermal = true;
5121                         break;
5122                 case 233: /* GUI IDLE */
5123                         DRM_DEBUG("IH: GUI idle\n");
5124                         break;
5125                 case 244: /* DMA trap event */
5126                         if (rdev->family >= CHIP_CAYMAN) {
5127                                 DRM_DEBUG("IH: DMA1 trap\n");
5128                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5129                         }
5130                         break;
5131                 default:
5132                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5133                         break;
5134                 }
5135
5136                 /* wptr/rptr are in bytes! */
5137                 rptr += 16;
5138                 rptr &= rdev->ih.ptr_mask;
5139                 WREG32(IH_RB_RPTR, rptr);
5140         }
5141         if (queue_hotplug)
5142                 schedule_work(&rdev->hotplug_work);
5143         if (queue_hdmi)
5144                 schedule_work(&rdev->audio_work);
5145         if (queue_thermal && rdev->pm.dpm_enabled)
5146                 schedule_work(&rdev->pm.dpm.thermal.work);
5147         rdev->ih.rptr = rptr;
5148         atomic_set(&rdev->ih.lock, 0);
5149
5150         /* make sure wptr hasn't changed while processing */
5151         wptr = evergreen_get_ih_wptr(rdev);
5152         if (wptr != rptr)
5153                 goto restart_ih;
5154
5155         return IRQ_HANDLED;
5156 }
5157
5158 static int evergreen_startup(struct radeon_device *rdev)
5159 {
5160         struct radeon_ring *ring;
5161         int r;
5162
5163         /* enable pcie gen2 link */
5164         evergreen_pcie_gen2_enable(rdev);
5165         /* enable aspm */
5166         evergreen_program_aspm(rdev);
5167
5168         /* scratch needs to be initialized before MC */
5169         r = r600_vram_scratch_init(rdev);
5170         if (r)
5171                 return r;
5172
5173         evergreen_mc_program(rdev);
5174
5175         if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5176                 r = ni_mc_load_microcode(rdev);
5177                 if (r) {
5178                         DRM_ERROR("Failed to load MC firmware!\n");
5179                         return r;
5180                 }
5181         }
5182
5183         if (rdev->flags & RADEON_IS_AGP) {
5184                 evergreen_agp_enable(rdev);
5185         } else {
5186                 r = evergreen_pcie_gart_enable(rdev);
5187                 if (r)
5188                         return r;
5189         }
5190         evergreen_gpu_init(rdev);
5191
5192         /* allocate rlc buffers */
5193         if (rdev->flags & RADEON_IS_IGP) {
5194                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5195                 rdev->rlc.reg_list_size =
5196                         (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5197                 rdev->rlc.cs_data = evergreen_cs_data;
5198                 r = sumo_rlc_init(rdev);
5199                 if (r) {
5200                         DRM_ERROR("Failed to init rlc BOs!\n");
5201                         return r;
5202                 }
5203         }
5204
5205         /* allocate wb buffer */
5206         r = radeon_wb_init(rdev);
5207         if (r)
5208                 return r;
5209
5210         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5211         if (r) {
5212                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5213                 return r;
5214         }
5215
5216         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5217         if (r) {
5218                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5219                 return r;
5220         }
5221
5222         r = uvd_v2_2_resume(rdev);
5223         if (!r) {
5224                 r = radeon_fence_driver_start_ring(rdev,
5225                                                    R600_RING_TYPE_UVD_INDEX);
5226                 if (r)
5227                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5228         }
5229
5230         if (r)
5231                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5232
5233         /* Enable IRQ */
5234         if (!rdev->irq.installed) {
5235                 r = radeon_irq_kms_init(rdev);
5236                 if (r)
5237                         return r;
5238         }
5239
5240         r = r600_irq_init(rdev);
5241         if (r) {
5242                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5243                 radeon_irq_kms_fini(rdev);
5244                 return r;
5245         }
5246         evergreen_irq_set(rdev);
5247
5248         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5249         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5250                              RADEON_CP_PACKET2);
5251         if (r)
5252                 return r;
5253
5254         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5255         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5256                              DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5257         if (r)
5258                 return r;
5259
5260         r = evergreen_cp_load_microcode(rdev);
5261         if (r)
5262                 return r;
5263         r = evergreen_cp_resume(rdev);
5264         if (r)
5265                 return r;
5266         r = r600_dma_resume(rdev);
5267         if (r)
5268                 return r;
5269
5270         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5271         if (ring->ring_size) {
5272                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5273                                      RADEON_CP_PACKET2);
5274                 if (!r)
5275                         r = uvd_v1_0_init(rdev);
5276
5277                 if (r)
5278                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5279         }
5280
5281         r = radeon_ib_pool_init(rdev);
5282         if (r) {
5283                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5284                 return r;
5285         }
5286
5287         r = r600_audio_init(rdev);
5288         if (r) {
5289                 DRM_ERROR("radeon: audio init failed\n");
5290                 return r;
5291         }
5292
5293         return 0;
5294 }
5295
5296 int evergreen_resume(struct radeon_device *rdev)
5297 {
5298         int r;
5299
5300         /* reset the asic, the gfx blocks are often in a bad state
5301          * after the driver is unloaded or after a resume
5302          */
5303         if (radeon_asic_reset(rdev))
5304                 dev_warn(rdev->dev, "GPU reset failed !\n");
5305         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5306          * posting will perform necessary task to bring back GPU into good
5307          * shape.
5308          */
5309         /* post card */
5310         atom_asic_init(rdev->mode_info.atom_context);
5311
5312         /* init golden registers */
5313         evergreen_init_golden_registers(rdev);
5314
5315         if (rdev->pm.pm_method == PM_METHOD_DPM)
5316                 radeon_pm_resume(rdev);
5317
5318         rdev->accel_working = true;
5319         r = evergreen_startup(rdev);
5320         if (r) {
5321                 DRM_ERROR("evergreen startup failed on resume\n");
5322                 rdev->accel_working = false;
5323                 return r;
5324         }
5325
5326         return r;
5327
5328 }
5329
5330 int evergreen_suspend(struct radeon_device *rdev)
5331 {
5332         radeon_pm_suspend(rdev);
5333         r600_audio_fini(rdev);
5334         uvd_v1_0_fini(rdev);
5335         radeon_uvd_suspend(rdev);
5336         r700_cp_stop(rdev);
5337         r600_dma_stop(rdev);
5338         evergreen_irq_suspend(rdev);
5339         radeon_wb_disable(rdev);
5340         evergreen_pcie_gart_disable(rdev);
5341
5342         return 0;
5343 }
5344
5345 /* Plan is to move initialization in that function and use
5346  * helper function so that radeon_device_init pretty much
5347  * do nothing more than calling asic specific function. This
5348  * should also allow to remove a bunch of callback function
5349  * like vram_info.
5350  */
5351 int evergreen_init(struct radeon_device *rdev)
5352 {
5353         int r;
5354
5355         /* Read BIOS */
5356         if (!radeon_get_bios(rdev)) {
5357                 if (ASIC_IS_AVIVO(rdev))
5358                         return -EINVAL;
5359         }
5360         /* Must be an ATOMBIOS */
5361         if (!rdev->is_atom_bios) {
5362                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5363                 return -EINVAL;
5364         }
5365         r = radeon_atombios_init(rdev);
5366         if (r)
5367                 return r;
5368         /* reset the asic, the gfx blocks are often in a bad state
5369          * after the driver is unloaded or after a resume
5370          */
5371         if (radeon_asic_reset(rdev))
5372                 dev_warn(rdev->dev, "GPU reset failed !\n");
5373         /* Post card if necessary */
5374         if (!radeon_card_posted(rdev)) {
5375                 if (!rdev->bios) {
5376                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5377                         return -EINVAL;
5378                 }
5379                 DRM_INFO("GPU not posted. posting now...\n");
5380                 atom_asic_init(rdev->mode_info.atom_context);
5381         }
5382         /* init golden registers */
5383         evergreen_init_golden_registers(rdev);
5384         /* Initialize scratch registers */
5385         r600_scratch_init(rdev);
5386         /* Initialize surface registers */
5387         radeon_surface_init(rdev);
5388         /* Initialize clocks */
5389         radeon_get_clock_info(rdev->ddev);
5390         /* Fence driver */
5391         r = radeon_fence_driver_init(rdev);
5392         if (r)
5393                 return r;
5394         /* initialize AGP */
5395         if (rdev->flags & RADEON_IS_AGP) {
5396                 r = radeon_agp_init(rdev);
5397                 if (r)
5398                         radeon_agp_disable(rdev);
5399         }
5400         /* initialize memory controller */
5401         r = evergreen_mc_init(rdev);
5402         if (r)
5403                 return r;
5404         /* Memory manager */
5405         r = radeon_bo_init(rdev);
5406         if (r)
5407                 return r;
5408
5409         if (ASIC_IS_DCE5(rdev)) {
5410                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5411                         r = ni_init_microcode(rdev);
5412                         if (r) {
5413                                 DRM_ERROR("Failed to load firmware!\n");
5414                                 return r;
5415                         }
5416                 }
5417         } else {
5418                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5419                         r = r600_init_microcode(rdev);
5420                         if (r) {
5421                                 DRM_ERROR("Failed to load firmware!\n");
5422                                 return r;
5423                         }
5424                 }
5425         }
5426
5427         /* Initialize power management */
5428         radeon_pm_init(rdev);
5429
5430         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5431         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5432
5433         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5434         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5435
5436         r = radeon_uvd_init(rdev);
5437         if (!r) {
5438                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5439                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5440                                4096);
5441         }
5442
5443         rdev->ih.ring_obj = NULL;
5444         r600_ih_ring_init(rdev, 64 * 1024);
5445
5446         r = r600_pcie_gart_init(rdev);
5447         if (r)
5448                 return r;
5449
5450         rdev->accel_working = true;
5451         r = evergreen_startup(rdev);
5452         if (r) {
5453                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5454                 r700_cp_fini(rdev);
5455                 r600_dma_fini(rdev);
5456                 r600_irq_fini(rdev);
5457                 if (rdev->flags & RADEON_IS_IGP)
5458                         sumo_rlc_fini(rdev);
5459                 radeon_wb_fini(rdev);
5460                 radeon_ib_pool_fini(rdev);
5461                 radeon_irq_kms_fini(rdev);
5462                 evergreen_pcie_gart_fini(rdev);
5463                 rdev->accel_working = false;
5464         }
5465
5466         /* Don't start up if the MC ucode is missing on BTC parts.
5467          * The default clocks and voltages before the MC ucode
5468          * is loaded are not suffient for advanced operations.
5469          */
5470         if (ASIC_IS_DCE5(rdev)) {
5471                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5472                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5473                         return -EINVAL;
5474                 }
5475         }
5476
5477         return 0;
5478 }
5479
5480 void evergreen_fini(struct radeon_device *rdev)
5481 {
5482         radeon_pm_fini(rdev);
5483         r600_audio_fini(rdev);
5484         r700_cp_fini(rdev);
5485         r600_dma_fini(rdev);
5486         r600_irq_fini(rdev);
5487         if (rdev->flags & RADEON_IS_IGP)
5488                 sumo_rlc_fini(rdev);
5489         radeon_wb_fini(rdev);
5490         radeon_ib_pool_fini(rdev);
5491         radeon_irq_kms_fini(rdev);
5492         uvd_v1_0_fini(rdev);
5493         radeon_uvd_fini(rdev);
5494         evergreen_pcie_gart_fini(rdev);
5495         r600_vram_scratch_fini(rdev);
5496         radeon_gem_fini(rdev);
5497         radeon_fence_driver_fini(rdev);
5498         radeon_agp_fini(rdev);
5499         radeon_bo_fini(rdev);
5500         radeon_atombios_fini(rdev);
5501         kfree(rdev->bios);
5502         rdev->bios = NULL;
5503 }
5504
5505 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5506 {
5507         u32 link_width_cntl, speed_cntl;
5508
5509         if (radeon_pcie_gen2 == 0)
5510                 return;
5511
5512         if (rdev->flags & RADEON_IS_IGP)
5513                 return;
5514
5515         if (!(rdev->flags & RADEON_IS_PCIE))
5516                 return;
5517
5518         /* x2 cards have a special sequence */
5519         if (ASIC_IS_X2(rdev))
5520                 return;
5521
5522         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5523                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5524                 return;
5525
5526         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5527         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5528                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5529                 return;
5530         }
5531
5532         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5533
5534         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5535             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5536
5537                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5538                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5539                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5540
5541                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5542                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5543                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5544
5545                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5546                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5547                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5548
5549                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5550                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5551                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5552
5553                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5554                 speed_cntl |= LC_GEN2_EN_STRAP;
5555                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5556
5557         } else {
5558                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5559                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5560                 if (1)
5561                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5562                 else
5563                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5564                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5565         }
5566 }
5567
5568 void evergreen_program_aspm(struct radeon_device *rdev)
5569 {
5570         u32 data, orig;
5571         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5572         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5573         /* fusion_platform = true
5574          * if the system is a fusion system
5575          * (APU or DGPU in a fusion system).
5576          * todo: check if the system is a fusion platform.
5577          */
5578         bool fusion_platform = false;
5579
5580         if (radeon_aspm == 0)
5581                 return;
5582
5583         if (!(rdev->flags & RADEON_IS_PCIE))
5584                 return;
5585
5586         switch (rdev->family) {
5587         case CHIP_CYPRESS:
5588         case CHIP_HEMLOCK:
5589         case CHIP_JUNIPER:
5590         case CHIP_REDWOOD:
5591         case CHIP_CEDAR:
5592         case CHIP_SUMO:
5593         case CHIP_SUMO2:
5594         case CHIP_PALM:
5595         case CHIP_ARUBA:
5596                 disable_l0s = true;
5597                 break;
5598         default:
5599                 disable_l0s = false;
5600                 break;
5601         }
5602
5603         if (rdev->flags & RADEON_IS_IGP)
5604                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5605
5606         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5607         if (fusion_platform)
5608                 data &= ~MULTI_PIF;
5609         else
5610                 data |= MULTI_PIF;
5611         if (data != orig)
5612                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5613
5614         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5615         if (fusion_platform)
5616                 data &= ~MULTI_PIF;
5617         else
5618                 data |= MULTI_PIF;
5619         if (data != orig)
5620                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5621
5622         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5623         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5624         if (!disable_l0s) {
5625                 if (rdev->family >= CHIP_BARTS)
5626                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5627                 else
5628                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5629         }
5630
5631         if (!disable_l1) {
5632                 if (rdev->family >= CHIP_BARTS)
5633                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5634                 else
5635                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5636
5637                 if (!disable_plloff_in_l1) {
5638                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5639                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5640                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5641                         if (data != orig)
5642                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5643
5644                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5645                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5646                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5647                         if (data != orig)
5648                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5649
5650                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5651                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5652                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5653                         if (data != orig)
5654                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5655
5656                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5657                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5658                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5659                         if (data != orig)
5660                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5661
5662                         if (rdev->family >= CHIP_BARTS) {
5663                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5664                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5665                                 data |= PLL_RAMP_UP_TIME_0(4);
5666                                 if (data != orig)
5667                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5668
5669                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5670                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5671                                 data |= PLL_RAMP_UP_TIME_1(4);
5672                                 if (data != orig)
5673                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5674
5675                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5676                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5677                                 data |= PLL_RAMP_UP_TIME_0(4);
5678                                 if (data != orig)
5679                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5680
5681                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5682                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5683                                 data |= PLL_RAMP_UP_TIME_1(4);
5684                                 if (data != orig)
5685                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5686                         }
5687
5688                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5689                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5690                         data |= LC_DYN_LANES_PWR_STATE(3);
5691                         if (data != orig)
5692                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5693
5694                         if (rdev->family >= CHIP_BARTS) {
5695                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5696                                 data &= ~LS2_EXIT_TIME_MASK;
5697                                 data |= LS2_EXIT_TIME(1);
5698                                 if (data != orig)
5699                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5700
5701                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5702                                 data &= ~LS2_EXIT_TIME_MASK;
5703                                 data |= LS2_EXIT_TIME(1);
5704                                 if (data != orig)
5705                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5706                         }
5707                 }
5708         }
5709
5710         /* evergreen parts only */
5711         if (rdev->family < CHIP_BARTS)
5712                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5713
5714         if (pcie_lc_cntl != pcie_lc_cntl_old)
5715                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5716 }