drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 static const u32 tn_rlc_save_restore_register_list[] =
  40 {
  41         0x98fc,
  42         0x98f0,
  43         0x9834,
  44         0x9838,
  45         0x9870,
  46         0x9874,
  47         0x8a14,
  48         0x8b24,
  49         0x8bcc,
  50         0x8b10,
  51         0x8c30,
  52         0x8d00,
  53         0x8d04,
  54         0x8c00,
  55         0x8c04,
  56         0x8c10,
  57         0x8c14,
  58         0x8d8c,
  59         0x8cf0,
  60         0x8e38,
  61         0x9508,
  62         0x9688,
  63         0x9608,
  64         0x960c,
  65         0x9610,
  66         0x9614,
  67         0x88c4,
  68         0x8978,
  69         0x88d4,
  70         0x900c,
  71         0x9100,
  72         0x913c,
  73         0x90e8,
  74         0x9354,
  75         0xa008,
  76         0x98f8,
  77         0x9148,
  78         0x914c,
  79         0x3f94,
  80         0x98f4,
  81         0x9b7c,
  82         0x3f8c,
  83         0x8950,
  84         0x8954,
  85         0x8a18,
  86         0x8b28,
  87         0x9144,
  88         0x3f90,
  89         0x915c,
  90         0x9160,
  91         0x9178,
  92         0x917c,
  93         0x9180,
  94         0x918c,
  95         0x9190,
  96         0x9194,
  97         0x9198,
  98         0x919c,
  99         0x91a8,
 100         0x91ac,
 101         0x91b0,
 102         0x91b4,
 103         0x91b8,
 104         0x91c4,
 105         0x91c8,
 106         0x91cc,
 107         0x91d0,
 108         0x91d4,
 109         0x91e0,
 110         0x91e4,
 111         0x91ec,
 112         0x91f0,
 113         0x91f4,
 114         0x9200,
 115         0x9204,
 116         0x929c,
 117         0x8030,
 118         0x9150,
 119         0x9a60,
 120         0x920c,
 121         0x9210,
 122         0x9228,
 123         0x922c,
 124         0x9244,
 125         0x9248,
 126         0x91e8,
 127         0x9294,
 128         0x9208,
 129         0x9224,
 130         0x9240,
 131         0x9220,
 132         0x923c,
 133         0x9258,
 134         0x9744,
 135         0xa200,
 136         0xa204,
 137         0xa208,
 138         0xa20c,
 139         0x8d58,
 140         0x9030,
 141         0x9034,
 142         0x9038,
 143         0x903c,
 144         0x9040,
 145         0x9654,
 146         0x897c,
 147         0xa210,
 148         0xa214,
 149         0x9868,
 150         0xa02c,
 151         0x9664,
 152         0x9698,
 153         0x949c,
 154         0x8e10,
 155         0x8e18,
 156         0x8c50,
 157         0x8c58,
 158         0x8c60,
 159         0x8c68,
 160         0x89b4,
 161         0x9830,
 162         0x802c,
 163 };
 164
 165 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 166 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 167 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 168 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 169 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 170 extern void evergreen_mc_program(struct radeon_device *rdev);
 171 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 172 extern int evergreen_mc_init(struct radeon_device *rdev);
 173 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 174 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 175 extern void evergreen_program_aspm(struct radeon_device *rdev);
 176 extern void sumo_rlc_fini(struct radeon_device *rdev);
 177 extern int sumo_rlc_init(struct radeon_device *rdev);
 178 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 179
 180 /* Firmware Names */
 181 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 182 MODULE_FIRMWARE("radeon/BARTS_me.bin");
 183 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 184 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 185 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 186 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 187 MODULE_FIRMWARE("radeon/TURKS_me.bin");
 188 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 189 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 190 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 191 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 192 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 193 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 194 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 195 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 196 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 197 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 198 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 199 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 200 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 201 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 202
 203
 204 static const u32 cayman_golden_registers2[] =
 205 {
 206         0x3e5c, 0xffffffff, 0x00000000,
 207         0x3e48, 0xffffffff, 0x00000000,
 208         0x3e4c, 0xffffffff, 0x00000000,
 209         0x3e64, 0xffffffff, 0x00000000,
 210         0x3e50, 0xffffffff, 0x00000000,
 211         0x3e60, 0xffffffff, 0x00000000
 212 };
 213
 214 static const u32 cayman_golden_registers[] =
 215 {
 216         0x5eb4, 0xffffffff, 0x00000002,
 217         0x5e78, 0x8f311ff1, 0x001000f0,
 218         0x3f90, 0xffff0000, 0xff000000,
 219         0x9148, 0xffff0000, 0xff000000,
 220         0x3f94, 0xffff0000, 0xff000000,
 221         0x914c, 0xffff0000, 0xff000000,
 222         0xc78, 0x00000080, 0x00000080,
 223         0xbd4, 0x70073777, 0x00011003,
 224         0xd02c, 0xbfffff1f, 0x08421000,
 225         0xd0b8, 0x73773777, 0x02011003,
 226         0x5bc0, 0x00200000, 0x50100000,
 227         0x98f8, 0x33773777, 0x02011003,
 228         0x98fc, 0xffffffff, 0x76541032,
 229         0x7030, 0x31000311, 0x00000011,
 230         0x2f48, 0x33773777, 0x42010001,
 231         0x6b28, 0x00000010, 0x00000012,
 232         0x7728, 0x00000010, 0x00000012,
 233         0x10328, 0x00000010, 0x00000012,
 234         0x10f28, 0x00000010, 0x00000012,
 235         0x11b28, 0x00000010, 0x00000012,
 236         0x12728, 0x00000010, 0x00000012,
 237         0x240c, 0x000007ff, 0x00000000,
 238         0x8a14, 0xf000001f, 0x00000007,
 239         0x8b24, 0x3fff3fff, 0x00ff0fff,
 240         0x8b10, 0x0000ff0f, 0x00000000,
 241         0x28a4c, 0x07ffffff, 0x06000000,
 242         0x10c, 0x00000001, 0x00010003,
 243         0xa02c, 0xffffffff, 0x0000009b,
 244         0x913c, 0x0000010f, 0x01000100,
 245         0x8c04, 0xf8ff00ff, 0x40600060,
 246         0x28350, 0x00000f01, 0x00000000,
 247         0x9508, 0x3700001f, 0x00000002,
 248         0x960c, 0xffffffff, 0x54763210,
 249         0x88c4, 0x001f3ae3, 0x00000082,
 250         0x88d0, 0xffffffff, 0x0f40df40,
 251         0x88d4, 0x0000001f, 0x00000010,
 252         0x8974, 0xffffffff, 0x00000000
 253 };
 254
 255 static const u32 dvst_golden_registers2[] =
 256 {
 257         0x8f8, 0xffffffff, 0,
 258         0x8fc, 0x00380000, 0,
 259         0x8f8, 0xffffffff, 1,
 260         0x8fc, 0x0e000000, 0
 261 };
 262
 263 static const u32 dvst_golden_registers[] =
 264 {
 265         0x690, 0x3fff3fff, 0x20c00033,
 266         0x918c, 0x0fff0fff, 0x00010006,
 267         0x91a8, 0x0fff0fff, 0x00010006,
 268         0x9150, 0xffffdfff, 0x6e944040,
 269         0x917c, 0x0fff0fff, 0x00030002,
 270         0x9198, 0x0fff0fff, 0x00030002,
 271         0x915c, 0x0fff0fff, 0x00010000,
 272         0x3f90, 0xffff0001, 0xff000000,
 273         0x9178, 0x0fff0fff, 0x00070000,
 274         0x9194, 0x0fff0fff, 0x00070000,
 275         0x9148, 0xffff0001, 0xff000000,
 276         0x9190, 0x0fff0fff, 0x00090008,
 277         0x91ac, 0x0fff0fff, 0x00090008,
 278         0x3f94, 0xffff0000, 0xff000000,
 279         0x914c, 0xffff0000, 0xff000000,
 280         0x929c, 0x00000fff, 0x00000001,
 281         0x55e4, 0xff607fff, 0xfc000100,
 282         0x8a18, 0xff000fff, 0x00000100,
 283         0x8b28, 0xff000fff, 0x00000100,
 284         0x9144, 0xfffc0fff, 0x00000100,
 285         0x6ed8, 0x00010101, 0x00010000,
 286         0x9830, 0xffffffff, 0x00000000,
 287         0x9834, 0xf00fffff, 0x00000400,
 288         0x9838, 0xfffffffe, 0x00000000,
 289         0xd0c0, 0xff000fff, 0x00000100,
 290         0xd02c, 0xbfffff1f, 0x08421000,
 291         0xd0b8, 0x73773777, 0x12010001,
 292         0x5bb0, 0x000000f0, 0x00000070,
 293         0x98f8, 0x73773777, 0x12010001,
 294         0x98fc, 0xffffffff, 0x00000010,
 295         0x9b7c, 0x00ff0000, 0x00fc0000,
 296         0x8030, 0x00001f0f, 0x0000100a,
 297         0x2f48, 0x73773777, 0x12010001,
 298         0x2408, 0x00030000, 0x000c007f,
 299         0x8a14, 0xf000003f, 0x00000007,
 300         0x8b24, 0x3fff3fff, 0x00ff0fff,
 301         0x8b10, 0x0000ff0f, 0x00000000,
 302         0x28a4c, 0x07ffffff, 0x06000000,
 303         0x4d8, 0x00000fff, 0x00000100,
 304         0xa008, 0xffffffff, 0x00010000,
 305         0x913c, 0xffff03ff, 0x01000100,
 306         0x8c00, 0x000000ff, 0x00000003,
 307         0x8c04, 0xf8ff00ff, 0x40600060,
 308         0x8cf0, 0x1fff1fff, 0x08e00410,
 309         0x28350, 0x00000f01, 0x00000000,
 310         0x9508, 0xf700071f, 0x00000002,
 311         0x960c, 0xffffffff, 0x54763210,
 312         0x20ef8, 0x01ff01ff, 0x00000002,
 313         0x20e98, 0xfffffbff, 0x00200000,
 314         0x2015c, 0xffffffff, 0x00000f40,
 315         0x88c4, 0x001f3ae3, 0x00000082,
 316         0x8978, 0x3fffffff, 0x04050140,
 317         0x88d4, 0x0000001f, 0x00000010,
 318         0x8974, 0xffffffff, 0x00000000
 319 };
 320
 321 static const u32 scrapper_golden_registers[] =
 322 {
 323         0x690, 0x3fff3fff, 0x20c00033,
 324         0x918c, 0x0fff0fff, 0x00010006,
 325         0x918c, 0x0fff0fff, 0x00010006,
 326         0x91a8, 0x0fff0fff, 0x00010006,
 327         0x91a8, 0x0fff0fff, 0x00010006,
 328         0x9150, 0xffffdfff, 0x6e944040,
 329         0x9150, 0xffffdfff, 0x6e944040,
 330         0x917c, 0x0fff0fff, 0x00030002,
 331         0x917c, 0x0fff0fff, 0x00030002,
 332         0x9198, 0x0fff0fff, 0x00030002,
 333         0x9198, 0x0fff0fff, 0x00030002,
 334         0x915c, 0x0fff0fff, 0x00010000,
 335         0x915c, 0x0fff0fff, 0x00010000,
 336         0x3f90, 0xffff0001, 0xff000000,
 337         0x3f90, 0xffff0001, 0xff000000,
 338         0x9178, 0x0fff0fff, 0x00070000,
 339         0x9178, 0x0fff0fff, 0x00070000,
 340         0x9194, 0x0fff0fff, 0x00070000,
 341         0x9194, 0x0fff0fff, 0x00070000,
 342         0x9148, 0xffff0001, 0xff000000,
 343         0x9148, 0xffff0001, 0xff000000,
 344         0x9190, 0x0fff0fff, 0x00090008,
 345         0x9190, 0x0fff0fff, 0x00090008,
 346         0x91ac, 0x0fff0fff, 0x00090008,
 347         0x91ac, 0x0fff0fff, 0x00090008,
 348         0x3f94, 0xffff0000, 0xff000000,
 349         0x3f94, 0xffff0000, 0xff000000,
 350         0x914c, 0xffff0000, 0xff000000,
 351         0x914c, 0xffff0000, 0xff000000,
 352         0x929c, 0x00000fff, 0x00000001,
 353         0x929c, 0x00000fff, 0x00000001,
 354         0x55e4, 0xff607fff, 0xfc000100,
 355         0x8a18, 0xff000fff, 0x00000100,
 356         0x8a18, 0xff000fff, 0x00000100,
 357         0x8b28, 0xff000fff, 0x00000100,
 358         0x8b28, 0xff000fff, 0x00000100,
 359         0x9144, 0xfffc0fff, 0x00000100,
 360         0x9144, 0xfffc0fff, 0x00000100,
 361         0x6ed8, 0x00010101, 0x00010000,
 362         0x9830, 0xffffffff, 0x00000000,
 363         0x9830, 0xffffffff, 0x00000000,
 364         0x9834, 0xf00fffff, 0x00000400,
 365         0x9834, 0xf00fffff, 0x00000400,
 366         0x9838, 0xfffffffe, 0x00000000,
 367         0x9838, 0xfffffffe, 0x00000000,
 368         0xd0c0, 0xff000fff, 0x00000100,
 369         0xd02c, 0xbfffff1f, 0x08421000,
 370         0xd02c, 0xbfffff1f, 0x08421000,
 371         0xd0b8, 0x73773777, 0x12010001,
 372         0xd0b8, 0x73773777, 0x12010001,
 373         0x5bb0, 0x000000f0, 0x00000070,
 374         0x98f8, 0x73773777, 0x12010001,
 375         0x98f8, 0x73773777, 0x12010001,
 376         0x98fc, 0xffffffff, 0x00000010,
 377         0x98fc, 0xffffffff, 0x00000010,
 378         0x9b7c, 0x00ff0000, 0x00fc0000,
 379         0x9b7c, 0x00ff0000, 0x00fc0000,
 380         0x8030, 0x00001f0f, 0x0000100a,
 381         0x8030, 0x00001f0f, 0x0000100a,
 382         0x2f48, 0x73773777, 0x12010001,
 383         0x2f48, 0x73773777, 0x12010001,
 384         0x2408, 0x00030000, 0x000c007f,
 385         0x8a14, 0xf000003f, 0x00000007,
 386         0x8a14, 0xf000003f, 0x00000007,
 387         0x8b24, 0x3fff3fff, 0x00ff0fff,
 388         0x8b24, 0x3fff3fff, 0x00ff0fff,
 389         0x8b10, 0x0000ff0f, 0x00000000,
 390         0x8b10, 0x0000ff0f, 0x00000000,
 391         0x28a4c, 0x07ffffff, 0x06000000,
 392         0x28a4c, 0x07ffffff, 0x06000000,
 393         0x4d8, 0x00000fff, 0x00000100,
 394         0x4d8, 0x00000fff, 0x00000100,
 395         0xa008, 0xffffffff, 0x00010000,
 396         0xa008, 0xffffffff, 0x00010000,
 397         0x913c, 0xffff03ff, 0x01000100,
 398         0x913c, 0xffff03ff, 0x01000100,
 399         0x90e8, 0x001fffff, 0x010400c0,
 400         0x8c00, 0x000000ff, 0x00000003,
 401         0x8c00, 0x000000ff, 0x00000003,
 402         0x8c04, 0xf8ff00ff, 0x40600060,
 403         0x8c04, 0xf8ff00ff, 0x40600060,
 404         0x8c30, 0x0000000f, 0x00040005,
 405         0x8cf0, 0x1fff1fff, 0x08e00410,
 406         0x8cf0, 0x1fff1fff, 0x08e00410,
 407         0x900c, 0x00ffffff, 0x0017071f,
 408         0x28350, 0x00000f01, 0x00000000,
 409         0x28350, 0x00000f01, 0x00000000,
 410         0x9508, 0xf700071f, 0x00000002,
 411         0x9508, 0xf700071f, 0x00000002,
 412         0x9688, 0x00300000, 0x0017000f,
 413         0x960c, 0xffffffff, 0x54763210,
 414         0x960c, 0xffffffff, 0x54763210,
 415         0x20ef8, 0x01ff01ff, 0x00000002,
 416         0x20e98, 0xfffffbff, 0x00200000,
 417         0x2015c, 0xffffffff, 0x00000f40,
 418         0x88c4, 0x001f3ae3, 0x00000082,
 419         0x88c4, 0x001f3ae3, 0x00000082,
 420         0x8978, 0x3fffffff, 0x04050140,
 421         0x8978, 0x3fffffff, 0x04050140,
 422         0x88d4, 0x0000001f, 0x00000010,
 423         0x88d4, 0x0000001f, 0x00000010,
 424         0x8974, 0xffffffff, 0x00000000,
 425         0x8974, 0xffffffff, 0x00000000
 426 };
 427
 428 static void ni_init_golden_registers(struct radeon_device *rdev)
 429 {
 430         switch (rdev->family) {
 431         case CHIP_CAYMAN:
 432                 radeon_program_register_sequence(rdev,
 433                                                  cayman_golden_registers,
 434                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 435                 radeon_program_register_sequence(rdev,
 436                                                  cayman_golden_registers2,
 437                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 438                 break;
 439         case CHIP_ARUBA:
 440                 if ((rdev->pdev->device == 0x9900) ||
 441                     (rdev->pdev->device == 0x9901) ||
 442                     (rdev->pdev->device == 0x9903) ||
 443                     (rdev->pdev->device == 0x9904) ||
 444                     (rdev->pdev->device == 0x9905) ||
 445                     (rdev->pdev->device == 0x9906) ||
 446                     (rdev->pdev->device == 0x9907) ||
 447                     (rdev->pdev->device == 0x9908) ||
 448                     (rdev->pdev->device == 0x9909) ||
 449                     (rdev->pdev->device == 0x990A) ||
 450                     (rdev->pdev->device == 0x990B) ||
 451                     (rdev->pdev->device == 0x990C) ||
 452                     (rdev->pdev->device == 0x990D) ||
 453                     (rdev->pdev->device == 0x990E) ||
 454                     (rdev->pdev->device == 0x990F) ||
 455                     (rdev->pdev->device == 0x9910) ||
 456                     (rdev->pdev->device == 0x9913) ||
 457                     (rdev->pdev->device == 0x9917) ||
 458                     (rdev->pdev->device == 0x9918)) {
 459                         radeon_program_register_sequence(rdev,
 460                                                          dvst_golden_registers,
 461                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 462                         radeon_program_register_sequence(rdev,
 463                                                          dvst_golden_registers2,
 464                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 465                 } else {
 466                         radeon_program_register_sequence(rdev,
 467                                                          scrapper_golden_registers,
 468                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 469                         radeon_program_register_sequence(rdev,
 470                                                          dvst_golden_registers2,
 471                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 472                 }
 473                 break;
 474         default:
 475                 break;
 476         }
 477 }
 478
 479 #define BTC_IO_MC_REGS_SIZE 29
 480
 481 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 482         {0x00000077, 0xff010100},
 483         {0x00000078, 0x00000000},
 484         {0x00000079, 0x00001434},
 485         {0x0000007a, 0xcc08ec08},
 486         {0x0000007b, 0x00040000},
 487         {0x0000007c, 0x000080c0},
 488         {0x0000007d, 0x09000000},
 489         {0x0000007e, 0x00210404},
 490         {0x00000081, 0x08a8e800},
 491         {0x00000082, 0x00030444},
 492         {0x00000083, 0x00000000},
 493         {0x00000085, 0x00000001},
 494         {0x00000086, 0x00000002},
 495         {0x00000087, 0x48490000},
 496         {0x00000088, 0x20244647},
 497         {0x00000089, 0x00000005},
 498         {0x0000008b, 0x66030000},
 499         {0x0000008c, 0x00006603},
 500         {0x0000008d, 0x00000100},
 501         {0x0000008f, 0x00001c0a},
 502         {0x00000090, 0xff000001},
 503         {0x00000094, 0x00101101},
 504         {0x00000095, 0x00000fff},
 505         {0x00000096, 0x00116fff},
 506         {0x00000097, 0x60010000},
 507         {0x00000098, 0x10010000},
 508         {0x00000099, 0x00006000},
 509         {0x0000009a, 0x00001000},
 510         {0x0000009f, 0x00946a00}
 511 };
 512
 513 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 514         {0x00000077, 0xff010100},
 515         {0x00000078, 0x00000000},
 516         {0x00000079, 0x00001434},
 517         {0x0000007a, 0xcc08ec08},
 518         {0x0000007b, 0x00040000},
 519         {0x0000007c, 0x000080c0},
 520         {0x0000007d, 0x09000000},
 521         {0x0000007e, 0x00210404},
 522         {0x00000081, 0x08a8e800},
 523         {0x00000082, 0x00030444},
 524         {0x00000083, 0x00000000},
 525         {0x00000085, 0x00000001},
 526         {0x00000086, 0x00000002},
 527         {0x00000087, 0x48490000},
 528         {0x00000088, 0x20244647},
 529         {0x00000089, 0x00000005},
 530         {0x0000008b, 0x66030000},
 531         {0x0000008c, 0x00006603},
 532         {0x0000008d, 0x00000100},
 533         {0x0000008f, 0x00001c0a},
 534         {0x00000090, 0xff000001},
 535         {0x00000094, 0x00101101},
 536         {0x00000095, 0x00000fff},
 537         {0x00000096, 0x00116fff},
 538         {0x00000097, 0x60010000},
 539         {0x00000098, 0x10010000},
 540         {0x00000099, 0x00006000},
 541         {0x0000009a, 0x00001000},
 542         {0x0000009f, 0x00936a00}
 543 };
 544
 545 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 546         {0x00000077, 0xff010100},
 547         {0x00000078, 0x00000000},
 548         {0x00000079, 0x00001434},
 549         {0x0000007a, 0xcc08ec08},
 550         {0x0000007b, 0x00040000},
 551         {0x0000007c, 0x000080c0},
 552         {0x0000007d, 0x09000000},
 553         {0x0000007e, 0x00210404},
 554         {0x00000081, 0x08a8e800},
 555         {0x00000082, 0x00030444},
 556         {0x00000083, 0x00000000},
 557         {0x00000085, 0x00000001},
 558         {0x00000086, 0x00000002},
 559         {0x00000087, 0x48490000},
 560         {0x00000088, 0x20244647},
 561         {0x00000089, 0x00000005},
 562         {0x0000008b, 0x66030000},
 563         {0x0000008c, 0x00006603},
 564         {0x0000008d, 0x00000100},
 565         {0x0000008f, 0x00001c0a},
 566         {0x00000090, 0xff000001},
 567         {0x00000094, 0x00101101},
 568         {0x00000095, 0x00000fff},
 569         {0x00000096, 0x00116fff},
 570         {0x00000097, 0x60010000},
 571         {0x00000098, 0x10010000},
 572         {0x00000099, 0x00006000},
 573         {0x0000009a, 0x00001000},
 574         {0x0000009f, 0x00916a00}
 575 };
 576
 577 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 578         {0x00000077, 0xff010100},
 579         {0x00000078, 0x00000000},
 580         {0x00000079, 0x00001434},
 581         {0x0000007a, 0xcc08ec08},
 582         {0x0000007b, 0x00040000},
 583         {0x0000007c, 0x000080c0},
 584         {0x0000007d, 0x09000000},
 585         {0x0000007e, 0x00210404},
 586         {0x00000081, 0x08a8e800},
 587         {0x00000082, 0x00030444},
 588         {0x00000083, 0x00000000},
 589         {0x00000085, 0x00000001},
 590         {0x00000086, 0x00000002},
 591         {0x00000087, 0x48490000},
 592         {0x00000088, 0x20244647},
 593         {0x00000089, 0x00000005},
 594         {0x0000008b, 0x66030000},
 595         {0x0000008c, 0x00006603},
 596         {0x0000008d, 0x00000100},
 597         {0x0000008f, 0x00001c0a},
 598         {0x00000090, 0xff000001},
 599         {0x00000094, 0x00101101},
 600         {0x00000095, 0x00000fff},
 601         {0x00000096, 0x00116fff},
 602         {0x00000097, 0x60010000},
 603         {0x00000098, 0x10010000},
 604         {0x00000099, 0x00006000},
 605         {0x0000009a, 0x00001000},
 606         {0x0000009f, 0x00976b00}
 607 };
 608
 609 int ni_mc_load_microcode(struct radeon_device *rdev)
 610 {
 611         const __be32 *fw_data;
 612         u32 mem_type, running, blackout = 0;
 613         u32 *io_mc_regs;
 614         int i, ucode_size, regs_size;
 615
 616         if (!rdev->mc_fw)
 617                 return -EINVAL;
 618
 619         switch (rdev->family) {
 620         case CHIP_BARTS:
 621                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 622                 ucode_size = BTC_MC_UCODE_SIZE;
 623                 regs_size = BTC_IO_MC_REGS_SIZE;
 624                 break;
 625         case CHIP_TURKS:
 626                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_CAICOS:
 631         default:
 632                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 633                 ucode_size = BTC_MC_UCODE_SIZE;
 634                 regs_size = BTC_IO_MC_REGS_SIZE;
 635                 break;
 636         case CHIP_CAYMAN:
 637                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 638                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         }
 642
 643         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 644         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 645
 646         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 647                 if (running) {
 648                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 649                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 650                 }
 651
 652                 /* reset the engine and set to writable */
 653                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 654                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 655
 656                 /* load mc io regs */
 657                 for (i = 0; i < regs_size; i++) {
 658                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 659                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 660                 }
 661                 /* load the MC ucode */
 662                 fw_data = (const __be32 *)rdev->mc_fw->data;
 663                 for (i = 0; i < ucode_size; i++)
 664                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 665
 666                 /* put the engine back into the active state */
 667                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 668                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 669                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 670
 671                 /* wait for training to complete */
 672                 for (i = 0; i < rdev->usec_timeout; i++) {
 673                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 674                                 break;
 675                         udelay(1);
 676                 }
 677
 678                 if (running)
 679                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 680         }
 681
 682         return 0;
 683 }
 684
 685 int ni_init_microcode(struct radeon_device *rdev)
 686 {
 687         const char *chip_name;
 688         const char *rlc_chip_name;
 689         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 690         size_t smc_req_size = 0;
 691         char fw_name[30];
 692         int err;
 693
 694         DRM_DEBUG("\n");
 695
 696         switch (rdev->family) {
 697         case CHIP_BARTS:
 698                 chip_name = "BARTS";
 699                 rlc_chip_name = "BTC";
 700                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 701                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 702                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 703                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 704                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 705                 break;
 706         case CHIP_TURKS:
 707                 chip_name = "TURKS";
 708                 rlc_chip_name = "BTC";
 709                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 710                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 711                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 712                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 713                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 714                 break;
 715         case CHIP_CAICOS:
 716                 chip_name = "CAICOS";
 717                 rlc_chip_name = "BTC";
 718                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 719                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 720                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 721                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 722                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 723                 break;
 724         case CHIP_CAYMAN:
 725                 chip_name = "CAYMAN";
 726                 rlc_chip_name = "CAYMAN";
 727                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 728                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 729                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 730                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 731                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 732                 break;
 733         case CHIP_ARUBA:
 734                 chip_name = "ARUBA";
 735                 rlc_chip_name = "ARUBA";
 736                 /* pfp/me same size as CAYMAN */
 737                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 738                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 739                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 740                 mc_req_size = 0;
 741                 break;
 742         default: BUG();
 743         }
 744
 745         DRM_INFO("Loading %s Microcode\n", chip_name);
 746
 747         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 748         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 749         if (err)
 750                 goto out;
 751         if (rdev->pfp_fw->size != pfp_req_size) {
 752                 printk(KERN_ERR
 753                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 754                        rdev->pfp_fw->size, fw_name);
 755                 err = -EINVAL;
 756                 goto out;
 757         }
 758
 759         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 760         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 761         if (err)
 762                 goto out;
 763         if (rdev->me_fw->size != me_req_size) {
 764                 printk(KERN_ERR
 765                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 766                        rdev->me_fw->size, fw_name);
 767                 err = -EINVAL;
 768         }
 769
 770         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 771         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 772         if (err)
 773                 goto out;
 774         if (rdev->rlc_fw->size != rlc_req_size) {
 775                 printk(KERN_ERR
 776                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 777                        rdev->rlc_fw->size, fw_name);
 778                 err = -EINVAL;
 779         }
 780
 781         /* no MC ucode on TN */
 782         if (!(rdev->flags & RADEON_IS_IGP)) {
 783                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 784                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 785                 if (err)
 786                         goto out;
 787                 if (rdev->mc_fw->size != mc_req_size) {
 788                         printk(KERN_ERR
 789                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 790                                rdev->mc_fw->size, fw_name);
 791                         err = -EINVAL;
 792                 }
 793         }
 794
 795         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 796                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 797                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 798                 if (err) {
 799                         printk(KERN_ERR
 800                                "smc: error loading firmware \"%s\"\n",
 801                                fw_name);
 802                         release_firmware(rdev->smc_fw);
 803                         rdev->smc_fw = NULL;
 804                         err = 0;
 805                 } else if (rdev->smc_fw->size != smc_req_size) {
 806                         printk(KERN_ERR
 807                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 808                                rdev->mc_fw->size, fw_name);
 809                         err = -EINVAL;
 810                 }
 811         }
 812
 813 out:
 814         if (err) {
 815                 if (err != -EINVAL)
 816                         printk(KERN_ERR
 817                                "ni_cp: Failed to load firmware \"%s\"\n",
 818                                fw_name);
 819                 release_firmware(rdev->pfp_fw);
 820                 rdev->pfp_fw = NULL;
 821                 release_firmware(rdev->me_fw);
 822                 rdev->me_fw = NULL;
 823                 release_firmware(rdev->rlc_fw);
 824                 rdev->rlc_fw = NULL;
 825                 release_firmware(rdev->mc_fw);
 826                 rdev->mc_fw = NULL;
 827         }
 828         return err;
 829 }
 830
 831 /**
 832  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 833  *
 834  * @rdev: radeon_device pointer
 835  * @reg: register offset in bytes
 836  * @val: register value
 837  *
 838  * Returns 0 for success or -EINVAL for an invalid register
 839  *
 840  */
 841 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 842                                      u32 reg, u32 *val)
 843 {
 844         switch (reg) {
 845         case GRBM_STATUS:
 846         case GRBM_STATUS_SE0:
 847         case GRBM_STATUS_SE1:
 848         case SRBM_STATUS:
 849         case SRBM_STATUS2:
 850         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 851         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 852         case UVD_STATUS:
 853                 *val = RREG32(reg);
 854                 return 0;
 855         default:
 856                 return -EINVAL;
 857         }
 858 }
 859
 860 int tn_get_temp(struct radeon_device *rdev)
 861 {
 862         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 863         int actual_temp = (temp / 8) - 49;
 864
 865         return actual_temp * 1000;
 866 }
 867
 868 /*
 869  * Core functions
 870  */
 871 static void cayman_gpu_init(struct radeon_device *rdev)
 872 {
 873         u32 gb_addr_config = 0;
 874         u32 mc_shared_chmap, mc_arb_ramcfg;
 875         u32 cgts_tcc_disable;
 876         u32 sx_debug_1;
 877         u32 smx_dc_ctl0;
 878         u32 cgts_sm_ctrl_reg;
 879         u32 hdp_host_path_cntl;
 880         u32 tmp;
 881         u32 disabled_rb_mask;
 882         int i, j;
 883
 884         switch (rdev->family) {
 885         case CHIP_CAYMAN:
 886                 rdev->config.cayman.max_shader_engines = 2;
 887                 rdev->config.cayman.max_pipes_per_simd = 4;
 888                 rdev->config.cayman.max_tile_pipes = 8;
 889                 rdev->config.cayman.max_simds_per_se = 12;
 890                 rdev->config.cayman.max_backends_per_se = 4;
 891                 rdev->config.cayman.max_texture_channel_caches = 8;
 892                 rdev->config.cayman.max_gprs = 256;
 893                 rdev->config.cayman.max_threads = 256;
 894                 rdev->config.cayman.max_gs_threads = 32;
 895                 rdev->config.cayman.max_stack_entries = 512;
 896                 rdev->config.cayman.sx_num_of_sets = 8;
 897                 rdev->config.cayman.sx_max_export_size = 256;
 898                 rdev->config.cayman.sx_max_export_pos_size = 64;
 899                 rdev->config.cayman.sx_max_export_smx_size = 192;
 900                 rdev->config.cayman.max_hw_contexts = 8;
 901                 rdev->config.cayman.sq_num_cf_insts = 2;
 902
 903                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 904                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 905                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 906                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 907                 break;
 908         case CHIP_ARUBA:
 909         default:
 910                 rdev->config.cayman.max_shader_engines = 1;
 911                 rdev->config.cayman.max_pipes_per_simd = 4;
 912                 rdev->config.cayman.max_tile_pipes = 2;
 913                 if ((rdev->pdev->device == 0x9900) ||
 914                     (rdev->pdev->device == 0x9901) ||
 915                     (rdev->pdev->device == 0x9905) ||
 916                     (rdev->pdev->device == 0x9906) ||
 917                     (rdev->pdev->device == 0x9907) ||
 918                     (rdev->pdev->device == 0x9908) ||
 919                     (rdev->pdev->device == 0x9909) ||
 920                     (rdev->pdev->device == 0x990B) ||
 921                     (rdev->pdev->device == 0x990C) ||
 922                     (rdev->pdev->device == 0x990F) ||
 923                     (rdev->pdev->device == 0x9910) ||
 924                     (rdev->pdev->device == 0x9917) ||
 925                     (rdev->pdev->device == 0x9999) ||
 926                     (rdev->pdev->device == 0x999C)) {
 927                         rdev->config.cayman.max_simds_per_se = 6;
 928                         rdev->config.cayman.max_backends_per_se = 2;
 929                         rdev->config.cayman.max_hw_contexts = 8;
 930                         rdev->config.cayman.sx_max_export_size = 256;
 931                         rdev->config.cayman.sx_max_export_pos_size = 64;
 932                         rdev->config.cayman.sx_max_export_smx_size = 192;
 933                 } else if ((rdev->pdev->device == 0x9903) ||
 934                            (rdev->pdev->device == 0x9904) ||
 935                            (rdev->pdev->device == 0x990A) ||
 936                            (rdev->pdev->device == 0x990D) ||
 937                            (rdev->pdev->device == 0x990E) ||
 938                            (rdev->pdev->device == 0x9913) ||
 939                            (rdev->pdev->device == 0x9918) ||
 940                            (rdev->pdev->device == 0x999D)) {
 941                         rdev->config.cayman.max_simds_per_se = 4;
 942                         rdev->config.cayman.max_backends_per_se = 2;
 943                         rdev->config.cayman.max_hw_contexts = 8;
 944                         rdev->config.cayman.sx_max_export_size = 256;
 945                         rdev->config.cayman.sx_max_export_pos_size = 64;
 946                         rdev->config.cayman.sx_max_export_smx_size = 192;
 947                 } else if ((rdev->pdev->device == 0x9919) ||
 948                            (rdev->pdev->device == 0x9990) ||
 949                            (rdev->pdev->device == 0x9991) ||
 950                            (rdev->pdev->device == 0x9994) ||
 951                            (rdev->pdev->device == 0x9995) ||
 952                            (rdev->pdev->device == 0x9996) ||
 953                            (rdev->pdev->device == 0x999A) ||
 954                            (rdev->pdev->device == 0x99A0)) {
 955                         rdev->config.cayman.max_simds_per_se = 3;
 956                         rdev->config.cayman.max_backends_per_se = 1;
 957                         rdev->config.cayman.max_hw_contexts = 4;
 958                         rdev->config.cayman.sx_max_export_size = 128;
 959                         rdev->config.cayman.sx_max_export_pos_size = 32;
 960                         rdev->config.cayman.sx_max_export_smx_size = 96;
 961                 } else {
 962                         rdev->config.cayman.max_simds_per_se = 2;
 963                         rdev->config.cayman.max_backends_per_se = 1;
 964                         rdev->config.cayman.max_hw_contexts = 4;
 965                         rdev->config.cayman.sx_max_export_size = 128;
 966                         rdev->config.cayman.sx_max_export_pos_size = 32;
 967                         rdev->config.cayman.sx_max_export_smx_size = 96;
 968                 }
 969                 rdev->config.cayman.max_texture_channel_caches = 2;
 970                 rdev->config.cayman.max_gprs = 256;
 971                 rdev->config.cayman.max_threads = 256;
 972                 rdev->config.cayman.max_gs_threads = 32;
 973                 rdev->config.cayman.max_stack_entries = 512;
 974                 rdev->config.cayman.sx_num_of_sets = 8;
 975                 rdev->config.cayman.sq_num_cf_insts = 2;
 976
 977                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 978                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 979                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 980                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 981                 break;
 982         }
 983
 984         /* Initialize HDP */
 985         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 986                 WREG32((0x2c14 + j), 0x00000000);
 987                 WREG32((0x2c18 + j), 0x00000000);
 988                 WREG32((0x2c1c + j), 0x00000000);
 989                 WREG32((0x2c20 + j), 0x00000000);
 990                 WREG32((0x2c24 + j), 0x00000000);
 991         }
 992
 993         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 994         WREG32(SRBM_INT_CNTL, 0x1);
 995         WREG32(SRBM_INT_ACK, 0x1);
 996
 997         evergreen_fix_pci_max_read_req_size(rdev);
 998
 999         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1000         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1001
1002         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1003         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1004         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1005                 rdev->config.cayman.mem_row_size_in_kb = 4;
1006         /* XXX use MC settings? */
1007         rdev->config.cayman.shader_engine_tile_size = 32;
1008         rdev->config.cayman.num_gpus = 1;
1009         rdev->config.cayman.multi_gpu_tile_size = 64;
1010
1011         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1012         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1013         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1014         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1015         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1016         rdev->config.cayman.num_shader_engines = tmp + 1;
1017         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1018         rdev->config.cayman.num_gpus = tmp + 1;
1019         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1020         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1021         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1022         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1023
1024
1025         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1026          * not have bank info, so create a custom tiling dword.
1027          * bits 3:0   num_pipes
1028          * bits 7:4   num_banks
1029          * bits 11:8  group_size
1030          * bits 15:12 row_size
1031          */
1032         rdev->config.cayman.tile_config = 0;
1033         switch (rdev->config.cayman.num_tile_pipes) {
1034         case 1:
1035         default:
1036                 rdev->config.cayman.tile_config |= (0 << 0);
1037                 break;
1038         case 2:
1039                 rdev->config.cayman.tile_config |= (1 << 0);
1040                 break;
1041         case 4:
1042                 rdev->config.cayman.tile_config |= (2 << 0);
1043                 break;
1044         case 8:
1045                 rdev->config.cayman.tile_config |= (3 << 0);
1046                 break;
1047         }
1048
1049         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1050         if (rdev->flags & RADEON_IS_IGP)
1051                 rdev->config.cayman.tile_config |= 1 << 4;
1052         else {
1053                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1054                 case 0: /* four banks */
1055                         rdev->config.cayman.tile_config |= 0 << 4;
1056                         break;
1057                 case 1: /* eight banks */
1058                         rdev->config.cayman.tile_config |= 1 << 4;
1059                         break;
1060                 case 2: /* sixteen banks */
1061                 default:
1062                         rdev->config.cayman.tile_config |= 2 << 4;
1063                         break;
1064                 }
1065         }
1066         rdev->config.cayman.tile_config |=
1067                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1068         rdev->config.cayman.tile_config |=
1069                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1070
1071         tmp = 0;
1072         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1073                 u32 rb_disable_bitmap;
1074
1075                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1076                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1077                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1078                 tmp <<= 4;
1079                 tmp |= rb_disable_bitmap;
1080         }
1081         /* enabled rb are just the one not disabled :) */
1082         disabled_rb_mask = tmp;
1083         tmp = 0;
1084         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1085                 tmp |= (1 << i);
1086         /* if all the backends are disabled, fix it up here */
1087         if ((disabled_rb_mask & tmp) == tmp) {
1088                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1089                         disabled_rb_mask &= ~(1 << i);
1090         }
1091
1092         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1093                 u32 simd_disable_bitmap;
1094
1095                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1096                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1097                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1098                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1099                 tmp <<= 16;
1100                 tmp |= simd_disable_bitmap;
1101         }
1102         rdev->config.cayman.active_simds = hweight32(~tmp);
1103
1104         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1105         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1106
1107         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1108         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1109         if (ASIC_IS_DCE6(rdev))
1110                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1111         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1112         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1113         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1114         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1115         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1116         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1117
1118         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1119             (rdev->flags & RADEON_IS_IGP)) {
1120                 if ((disabled_rb_mask & 3) == 2) {
1121                         /* RB1 disabled, RB0 enabled */
1122                         tmp = 0x00000000;
1123                 } else {
1124                         /* RB0 disabled, RB1 enabled */
1125                         tmp = 0x11111111;
1126                 }
1127         } else {
1128                 tmp = gb_addr_config & NUM_PIPES_MASK;
1129                 tmp = r6xx_remap_render_backend(rdev, tmp,
1130                                                 rdev->config.cayman.max_backends_per_se *
1131                                                 rdev->config.cayman.max_shader_engines,
1132                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1133         }
1134         WREG32(GB_BACKEND_MAP, tmp);
1135
1136         cgts_tcc_disable = 0xffff0000;
1137         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1138                 cgts_tcc_disable &= ~(1 << (16 + i));
1139         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1140         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1141         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1142         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1143
1144         /* reprogram the shader complex */
1145         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1146         for (i = 0; i < 16; i++)
1147                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1148         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1149
1150         /* set HW defaults for 3D engine */
1151         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1152
1153         sx_debug_1 = RREG32(SX_DEBUG_1);
1154         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1155         WREG32(SX_DEBUG_1, sx_debug_1);
1156
1157         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1158         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1159         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1160         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1161
1162         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1163
1164         /* need to be explicitly zero-ed */
1165         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1166         WREG32(SQ_LSTMP_RING_BASE, 0);
1167         WREG32(SQ_HSTMP_RING_BASE, 0);
1168         WREG32(SQ_ESTMP_RING_BASE, 0);
1169         WREG32(SQ_GSTMP_RING_BASE, 0);
1170         WREG32(SQ_VSTMP_RING_BASE, 0);
1171         WREG32(SQ_PSTMP_RING_BASE, 0);
1172
1173         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1174
1175         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1176                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1177                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1178
1179         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1180                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1181                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1182
1183
1184         WREG32(VGT_NUM_INSTANCES, 1);
1185
1186         WREG32(CP_PERFMON_CNTL, 0);
1187
1188         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1189                                   FETCH_FIFO_HIWATER(0x4) |
1190                                   DONE_FIFO_HIWATER(0xe0) |
1191                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1192
1193         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1194         WREG32(SQ_CONFIG, (VC_ENABLE |
1195                            EXPORT_SRC_C |
1196                            GFX_PRIO(0) |
1197                            CS1_PRIO(0) |
1198                            CS2_PRIO(1)));
1199         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1200
1201         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1202                                           FORCE_EOV_MAX_REZ_CNT(255)));
1203
1204         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1205                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1206
1207         WREG32(VGT_GS_VERTEX_REUSE, 16);
1208         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1209
1210         WREG32(CB_PERF_CTR0_SEL_0, 0);
1211         WREG32(CB_PERF_CTR0_SEL_1, 0);
1212         WREG32(CB_PERF_CTR1_SEL_0, 0);
1213         WREG32(CB_PERF_CTR1_SEL_1, 0);
1214         WREG32(CB_PERF_CTR2_SEL_0, 0);
1215         WREG32(CB_PERF_CTR2_SEL_1, 0);
1216         WREG32(CB_PERF_CTR3_SEL_0, 0);
1217         WREG32(CB_PERF_CTR3_SEL_1, 0);
1218
1219         tmp = RREG32(HDP_MISC_CNTL);
1220         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1221         WREG32(HDP_MISC_CNTL, tmp);
1222
1223         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1224         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1225
1226         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1227
1228         udelay(50);
1229
1230         /* set clockgating golden values on TN */
1231         if (rdev->family == CHIP_ARUBA) {
1232                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1233                 tmp &= ~0x00380000;
1234                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1235                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1236                 tmp &= ~0x0e000000;
1237                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1238         }
1239 }
1240
1241 /*
1242  * GART
1243  */
1244 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1245 {
1246         /* flush hdp cache */
1247         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1248
1249         /* bits 0-7 are the VM contexts0-7 */
1250         WREG32(VM_INVALIDATE_REQUEST, 1);
1251 }
1252
1253 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1254 {
1255         int i, r;
1256
1257         if (rdev->gart.robj == NULL) {
1258                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1259                 return -EINVAL;
1260         }
1261         r = radeon_gart_table_vram_pin(rdev);
1262         if (r)
1263                 return r;
1264         /* Setup TLB control */
1265         WREG32(MC_VM_MX_L1_TLB_CNTL,
1266                (0xA << 7) |
1267                ENABLE_L1_TLB |
1268                ENABLE_L1_FRAGMENT_PROCESSING |
1269                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1270                ENABLE_ADVANCED_DRIVER_MODEL |
1271                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1272         /* Setup L2 cache */
1273         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1274                ENABLE_L2_FRAGMENT_PROCESSING |
1275                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1276                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1277                EFFECTIVE_L2_QUEUE_SIZE(7) |
1278                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1279         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1280         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1281                BANK_SELECT(6) |
1282                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1283         /* setup context0 */
1284         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1285         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, (rdev->mc.gtt_end >> 12) - 1);
1286         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1287         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1288                         (u32)(rdev->dummy_page.addr >> 12));
1289         WREG32(VM_CONTEXT0_CNTL2, 0);
1290         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1291                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1292
1293         WREG32(0x15D4, 0);
1294         WREG32(0x15D8, 0);
1295         WREG32(0x15DC, 0);
1296
1297         /* empty context1-7 */
1298         /* Assign the pt base to something valid for now; the pts used for
1299          * the VMs are determined by the application and setup and assigned
1300          * on the fly in the vm part of radeon_gart.c
1301          */
1302         for (i = 1; i < 8; i++) {
1303                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1304                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1305                         rdev->vm_manager.max_pfn - 1);
1306                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1307                        rdev->vm_manager.saved_table_addr[i]);
1308         }
1309
1310         /* enable context1-7 */
1311         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1312                (u32)(rdev->dummy_page.addr >> 12));
1313         WREG32(VM_CONTEXT1_CNTL2, 4);
1314         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1315                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1316                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1317                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1318                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1319                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1320                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1321                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1322                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1323                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1324                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1325                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1326                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1327                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1328
1329         cayman_pcie_gart_tlb_flush(rdev);
1330         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1331                  (unsigned)(rdev->mc.gtt_size >> 20),
1332                  (unsigned long long)rdev->gart.table_addr);
1333         rdev->gart.ready = true;
1334         return 0;
1335 }
1336
1337 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1338 {
1339         unsigned i;
1340
1341         for (i = 1; i < 8; ++i) {
1342                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1343                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1344         }
1345
1346         /* Disable all tables */
1347         WREG32(VM_CONTEXT0_CNTL, 0);
1348         WREG32(VM_CONTEXT1_CNTL, 0);
1349         /* Setup TLB control */
1350         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1351                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1352                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1353         /* Setup L2 cache */
1354         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1355                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1356                EFFECTIVE_L2_QUEUE_SIZE(7) |
1357                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1358         WREG32(VM_L2_CNTL2, 0);
1359         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1360                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1361         radeon_gart_table_vram_unpin(rdev);
1362 }
1363
1364 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1365 {
1366         cayman_pcie_gart_disable(rdev);
1367         radeon_gart_table_vram_free(rdev);
1368         radeon_gart_fini(rdev);
1369 }
1370
1371 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1372                               int ring, u32 cp_int_cntl)
1373 {
1374         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1375
1376         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1377         WREG32(CP_INT_CNTL, cp_int_cntl);
1378 }
1379
1380 /*
1381  * CP.
1382  */
1383 void cayman_fence_ring_emit(struct radeon_device *rdev,
1384                             struct radeon_fence *fence)
1385 {
1386         struct radeon_ring *ring = &rdev->ring[fence->ring];
1387         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1388         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1389                 PACKET3_SH_ACTION_ENA;
1390
1391         /* flush read cache over gart for this vmid */
1392         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1393         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1394         radeon_ring_write(ring, 0xFFFFFFFF);
1395         radeon_ring_write(ring, 0);
1396         radeon_ring_write(ring, 10); /* poll interval */
1397         /* EVENT_WRITE_EOP - flush caches, send int */
1398         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1399         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1400         radeon_ring_write(ring, lower_32_bits(addr));
1401         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1402         radeon_ring_write(ring, fence->seq);
1403         radeon_ring_write(ring, 0);
1404 }
1405
1406 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1407 {
1408         struct radeon_ring *ring = &rdev->ring[ib->ring];
1409         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1410         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1411                 PACKET3_SH_ACTION_ENA;
1412
1413         /* set to DX10/11 mode */
1414         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1415         radeon_ring_write(ring, 1);
1416
1417         if (ring->rptr_save_reg) {
1418                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1419                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1420                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1421                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1422                 radeon_ring_write(ring, next_rptr);
1423         }
1424
1425         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1426         radeon_ring_write(ring,
1427 #ifdef __BIG_ENDIAN
1428                           (2 << 0) |
1429 #endif
1430                           (ib->gpu_addr & 0xFFFFFFFC));
1431         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1432         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1433
1434         /* flush read cache over gart for this vmid */
1435         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1436         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1437         radeon_ring_write(ring, 0xFFFFFFFF);
1438         radeon_ring_write(ring, 0);
1439         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1440 }
1441
1442 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1443 {
1444         if (enable)
1445                 WREG32(CP_ME_CNTL, 0);
1446         else {
1447                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1448                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1449                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1450                 WREG32(SCRATCH_UMSK, 0);
1451                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1452         }
1453 }
1454
1455 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1456                         struct radeon_ring *ring)
1457 {
1458         u32 rptr;
1459
1460         if (rdev->wb.enabled)
1461                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1462         else {
1463                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1464                         rptr = RREG32(CP_RB0_RPTR);
1465                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1466                         rptr = RREG32(CP_RB1_RPTR);
1467                 else
1468                         rptr = RREG32(CP_RB2_RPTR);
1469         }
1470
1471         return rptr;
1472 }
1473
1474 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1475                         struct radeon_ring *ring)
1476 {
1477         u32 wptr;
1478
1479         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1480                 wptr = RREG32(CP_RB0_WPTR);
1481         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1482                 wptr = RREG32(CP_RB1_WPTR);
1483         else
1484                 wptr = RREG32(CP_RB2_WPTR);
1485
1486         return wptr;
1487 }
1488
1489 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1490                          struct radeon_ring *ring)
1491 {
1492         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1493                 WREG32(CP_RB0_WPTR, ring->wptr);
1494                 (void)RREG32(CP_RB0_WPTR);
1495         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1496                 WREG32(CP_RB1_WPTR, ring->wptr);
1497                 (void)RREG32(CP_RB1_WPTR);
1498         } else {
1499                 WREG32(CP_RB2_WPTR, ring->wptr);
1500                 (void)RREG32(CP_RB2_WPTR);
1501         }
1502 }
1503
1504 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1505 {
1506         const __be32 *fw_data;
1507         int i;
1508
1509         if (!rdev->me_fw || !rdev->pfp_fw)
1510                 return -EINVAL;
1511
1512         cayman_cp_enable(rdev, false);
1513
1514         fw_data = (const __be32 *)rdev->pfp_fw->data;
1515         WREG32(CP_PFP_UCODE_ADDR, 0);
1516         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1517                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1518         WREG32(CP_PFP_UCODE_ADDR, 0);
1519
1520         fw_data = (const __be32 *)rdev->me_fw->data;
1521         WREG32(CP_ME_RAM_WADDR, 0);
1522         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1523                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1524
1525         WREG32(CP_PFP_UCODE_ADDR, 0);
1526         WREG32(CP_ME_RAM_WADDR, 0);
1527         WREG32(CP_ME_RAM_RADDR, 0);
1528         return 0;
1529 }
1530
1531 static int cayman_cp_start(struct radeon_device *rdev)
1532 {
1533         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1534         int r, i;
1535
1536         r = radeon_ring_lock(rdev, ring, 7);
1537         if (r) {
1538                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1539                 return r;
1540         }
1541         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1542         radeon_ring_write(ring, 0x1);
1543         radeon_ring_write(ring, 0x0);
1544         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1545         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1546         radeon_ring_write(ring, 0);
1547         radeon_ring_write(ring, 0);
1548         radeon_ring_unlock_commit(rdev, ring, false);
1549
1550         cayman_cp_enable(rdev, true);
1551
1552         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1553         if (r) {
1554                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1555                 return r;
1556         }
1557
1558         /* setup clear context state */
1559         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1560         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1561
1562         for (i = 0; i < cayman_default_size; i++)
1563                 radeon_ring_write(ring, cayman_default_state[i]);
1564
1565         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1566         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1567
1568         /* set clear context state */
1569         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1570         radeon_ring_write(ring, 0);
1571
1572         /* SQ_VTX_BASE_VTX_LOC */
1573         radeon_ring_write(ring, 0xc0026f00);
1574         radeon_ring_write(ring, 0x00000000);
1575         radeon_ring_write(ring, 0x00000000);
1576         radeon_ring_write(ring, 0x00000000);
1577
1578         /* Clear consts */
1579         radeon_ring_write(ring, 0xc0036f00);
1580         radeon_ring_write(ring, 0x00000bc4);
1581         radeon_ring_write(ring, 0xffffffff);
1582         radeon_ring_write(ring, 0xffffffff);
1583         radeon_ring_write(ring, 0xffffffff);
1584
1585         radeon_ring_write(ring, 0xc0026900);
1586         radeon_ring_write(ring, 0x00000316);
1587         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1588         radeon_ring_write(ring, 0x00000010); /*  */
1589
1590         radeon_ring_unlock_commit(rdev, ring, false);
1591
1592         /* XXX init other rings */
1593
1594         return 0;
1595 }
1596
1597 static void cayman_cp_fini(struct radeon_device *rdev)
1598 {
1599         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1600         cayman_cp_enable(rdev, false);
1601         radeon_ring_fini(rdev, ring);
1602         radeon_scratch_free(rdev, ring->rptr_save_reg);
1603 }
1604
1605 static int cayman_cp_resume(struct radeon_device *rdev)
1606 {
1607         static const int ridx[] = {
1608                 RADEON_RING_TYPE_GFX_INDEX,
1609                 CAYMAN_RING_TYPE_CP1_INDEX,
1610                 CAYMAN_RING_TYPE_CP2_INDEX
1611         };
1612         static const unsigned cp_rb_cntl[] = {
1613                 CP_RB0_CNTL,
1614                 CP_RB1_CNTL,
1615                 CP_RB2_CNTL,
1616         };
1617         static const unsigned cp_rb_rptr_addr[] = {
1618                 CP_RB0_RPTR_ADDR,
1619                 CP_RB1_RPTR_ADDR,
1620                 CP_RB2_RPTR_ADDR
1621         };
1622         static const unsigned cp_rb_rptr_addr_hi[] = {
1623                 CP_RB0_RPTR_ADDR_HI,
1624                 CP_RB1_RPTR_ADDR_HI,
1625                 CP_RB2_RPTR_ADDR_HI
1626         };
1627         static const unsigned cp_rb_base[] = {
1628                 CP_RB0_BASE,
1629                 CP_RB1_BASE,
1630                 CP_RB2_BASE
1631         };
1632         static const unsigned cp_rb_rptr[] = {
1633                 CP_RB0_RPTR,
1634                 CP_RB1_RPTR,
1635                 CP_RB2_RPTR
1636         };
1637         static const unsigned cp_rb_wptr[] = {
1638                 CP_RB0_WPTR,
1639                 CP_RB1_WPTR,
1640                 CP_RB2_WPTR
1641         };
1642         struct radeon_ring *ring;
1643         int i, r;
1644
1645         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1646         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1647                                  SOFT_RESET_PA |
1648                                  SOFT_RESET_SH |
1649                                  SOFT_RESET_VGT |
1650                                  SOFT_RESET_SPI |
1651                                  SOFT_RESET_SX));
1652         RREG32(GRBM_SOFT_RESET);
1653         mdelay(15);
1654         WREG32(GRBM_SOFT_RESET, 0);
1655         RREG32(GRBM_SOFT_RESET);
1656
1657         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1658         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1659
1660         /* Set the write pointer delay */
1661         WREG32(CP_RB_WPTR_DELAY, 0);
1662
1663         WREG32(CP_DEBUG, (1 << 27));
1664
1665         /* set the wb address whether it's enabled or not */
1666         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1667         WREG32(SCRATCH_UMSK, 0xff);
1668
1669         for (i = 0; i < 3; ++i) {
1670                 uint32_t rb_cntl;
1671                 uint64_t addr;
1672
1673                 /* Set ring buffer size */
1674                 ring = &rdev->ring[ridx[i]];
1675                 rb_cntl = order_base_2(ring->ring_size / 8);
1676                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1677 #ifdef __BIG_ENDIAN
1678                 rb_cntl |= BUF_SWAP_32BIT;
1679 #endif
1680                 WREG32(cp_rb_cntl[i], rb_cntl);
1681
1682                 /* set the wb address whether it's enabled or not */
1683                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1684                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1685                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1686         }
1687
1688         /* set the rb base addr, this causes an internal reset of ALL rings */
1689         for (i = 0; i < 3; ++i) {
1690                 ring = &rdev->ring[ridx[i]];
1691                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1692         }
1693
1694         for (i = 0; i < 3; ++i) {
1695                 /* Initialize the ring buffer's read and write pointers */
1696                 ring = &rdev->ring[ridx[i]];
1697                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1698
1699                 ring->wptr = 0;
1700                 WREG32(cp_rb_rptr[i], 0);
1701                 WREG32(cp_rb_wptr[i], ring->wptr);
1702
1703                 mdelay(1);
1704                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1705         }
1706
1707         /* start the rings */
1708         cayman_cp_start(rdev);
1709         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1710         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1711         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1712         /* this only test cp0 */
1713         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1714         if (r) {
1715                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1716                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1717                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1718                 return r;
1719         }
1720
1721         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1722                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1723
1724         return 0;
1725 }
1726
1727 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1728 {
1729         u32 reset_mask = 0;
1730         u32 tmp;
1731
1732         /* GRBM_STATUS */
1733         tmp = RREG32(GRBM_STATUS);
1734         if (tmp & (PA_BUSY | SC_BUSY |
1735                    SH_BUSY | SX_BUSY |
1736                    TA_BUSY | VGT_BUSY |
1737                    DB_BUSY | CB_BUSY |
1738                    GDS_BUSY | SPI_BUSY |
1739                    IA_BUSY | IA_BUSY_NO_DMA))
1740                 reset_mask |= RADEON_RESET_GFX;
1741
1742         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1743                    CP_BUSY | CP_COHERENCY_BUSY))
1744                 reset_mask |= RADEON_RESET_CP;
1745
1746         if (tmp & GRBM_EE_BUSY)
1747                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1748
1749         /* DMA_STATUS_REG 0 */
1750         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1751         if (!(tmp & DMA_IDLE))
1752                 reset_mask |= RADEON_RESET_DMA;
1753
1754         /* DMA_STATUS_REG 1 */
1755         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1756         if (!(tmp & DMA_IDLE))
1757                 reset_mask |= RADEON_RESET_DMA1;
1758
1759         /* SRBM_STATUS2 */
1760         tmp = RREG32(SRBM_STATUS2);
1761         if (tmp & DMA_BUSY)
1762                 reset_mask |= RADEON_RESET_DMA;
1763
1764         if (tmp & DMA1_BUSY)
1765                 reset_mask |= RADEON_RESET_DMA1;
1766
1767         /* SRBM_STATUS */
1768         tmp = RREG32(SRBM_STATUS);
1769         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1770                 reset_mask |= RADEON_RESET_RLC;
1771
1772         if (tmp & IH_BUSY)
1773                 reset_mask |= RADEON_RESET_IH;
1774
1775         if (tmp & SEM_BUSY)
1776                 reset_mask |= RADEON_RESET_SEM;
1777
1778         if (tmp & GRBM_RQ_PENDING)
1779                 reset_mask |= RADEON_RESET_GRBM;
1780
1781         if (tmp & VMC_BUSY)
1782                 reset_mask |= RADEON_RESET_VMC;
1783
1784         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1785                    MCC_BUSY | MCD_BUSY))
1786                 reset_mask |= RADEON_RESET_MC;
1787
1788         if (evergreen_is_display_hung(rdev))
1789                 reset_mask |= RADEON_RESET_DISPLAY;
1790
1791         /* VM_L2_STATUS */
1792         tmp = RREG32(VM_L2_STATUS);
1793         if (tmp & L2_BUSY)
1794                 reset_mask |= RADEON_RESET_VMC;
1795
1796         /* Skip MC reset as it's mostly likely not hung, just busy */
1797         if (reset_mask & RADEON_RESET_MC) {
1798                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1799                 reset_mask &= ~RADEON_RESET_MC;
1800         }
1801
1802         return reset_mask;
1803 }
1804
1805 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1806 {
1807         struct evergreen_mc_save save;
1808         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1809         u32 tmp;
1810
1811         if (reset_mask == 0)
1812                 return;
1813
1814         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1815
1816         evergreen_print_gpu_status_regs(rdev);
1817         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1818                  RREG32(0x14F8));
1819         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1820                  RREG32(0x14D8));
1821         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1822                  RREG32(0x14FC));
1823         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1824                  RREG32(0x14DC));
1825
1826         /* Disable CP parsing/prefetching */
1827         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1828
1829         if (reset_mask & RADEON_RESET_DMA) {
1830                 /* dma0 */
1831                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1832                 tmp &= ~DMA_RB_ENABLE;
1833                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1834         }
1835
1836         if (reset_mask & RADEON_RESET_DMA1) {
1837                 /* dma1 */
1838                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1839                 tmp &= ~DMA_RB_ENABLE;
1840                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1841         }
1842
1843         udelay(50);
1844
1845         evergreen_mc_stop(rdev, &save);
1846         if (evergreen_mc_wait_for_idle(rdev)) {
1847                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1848         }
1849
1850         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1851                 grbm_soft_reset = SOFT_RESET_CB |
1852                         SOFT_RESET_DB |
1853                         SOFT_RESET_GDS |
1854                         SOFT_RESET_PA |
1855                         SOFT_RESET_SC |
1856                         SOFT_RESET_SPI |
1857                         SOFT_RESET_SH |
1858                         SOFT_RESET_SX |
1859                         SOFT_RESET_TC |
1860                         SOFT_RESET_TA |
1861                         SOFT_RESET_VGT |
1862                         SOFT_RESET_IA;
1863         }
1864
1865         if (reset_mask & RADEON_RESET_CP) {
1866                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1867
1868                 srbm_soft_reset |= SOFT_RESET_GRBM;
1869         }
1870
1871         if (reset_mask & RADEON_RESET_DMA)
1872                 srbm_soft_reset |= SOFT_RESET_DMA;
1873
1874         if (reset_mask & RADEON_RESET_DMA1)
1875                 srbm_soft_reset |= SOFT_RESET_DMA1;
1876
1877         if (reset_mask & RADEON_RESET_DISPLAY)
1878                 srbm_soft_reset |= SOFT_RESET_DC;
1879
1880         if (reset_mask & RADEON_RESET_RLC)
1881                 srbm_soft_reset |= SOFT_RESET_RLC;
1882
1883         if (reset_mask & RADEON_RESET_SEM)
1884                 srbm_soft_reset |= SOFT_RESET_SEM;
1885
1886         if (reset_mask & RADEON_RESET_IH)
1887                 srbm_soft_reset |= SOFT_RESET_IH;
1888
1889         if (reset_mask & RADEON_RESET_GRBM)
1890                 srbm_soft_reset |= SOFT_RESET_GRBM;
1891
1892         if (reset_mask & RADEON_RESET_VMC)
1893                 srbm_soft_reset |= SOFT_RESET_VMC;
1894
1895         if (!(rdev->flags & RADEON_IS_IGP)) {
1896                 if (reset_mask & RADEON_RESET_MC)
1897                         srbm_soft_reset |= SOFT_RESET_MC;
1898         }
1899
1900         if (grbm_soft_reset) {
1901                 tmp = RREG32(GRBM_SOFT_RESET);
1902                 tmp |= grbm_soft_reset;
1903                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1904                 WREG32(GRBM_SOFT_RESET, tmp);
1905                 tmp = RREG32(GRBM_SOFT_RESET);
1906
1907                 udelay(50);
1908
1909                 tmp &= ~grbm_soft_reset;
1910                 WREG32(GRBM_SOFT_RESET, tmp);
1911                 tmp = RREG32(GRBM_SOFT_RESET);
1912         }
1913
1914         if (srbm_soft_reset) {
1915                 tmp = RREG32(SRBM_SOFT_RESET);
1916                 tmp |= srbm_soft_reset;
1917                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1918                 WREG32(SRBM_SOFT_RESET, tmp);
1919                 tmp = RREG32(SRBM_SOFT_RESET);
1920
1921                 udelay(50);
1922
1923                 tmp &= ~srbm_soft_reset;
1924                 WREG32(SRBM_SOFT_RESET, tmp);
1925                 tmp = RREG32(SRBM_SOFT_RESET);
1926         }
1927
1928         /* Wait a little for things to settle down */
1929         udelay(50);
1930
1931         evergreen_mc_resume(rdev, &save);
1932         udelay(50);
1933
1934         evergreen_print_gpu_status_regs(rdev);
1935 }
1936
1937 int cayman_asic_reset(struct radeon_device *rdev)
1938 {
1939         u32 reset_mask;
1940
1941         reset_mask = cayman_gpu_check_soft_reset(rdev);
1942
1943         if (reset_mask)
1944                 r600_set_bios_scratch_engine_hung(rdev, true);
1945
1946         cayman_gpu_soft_reset(rdev, reset_mask);
1947
1948         reset_mask = cayman_gpu_check_soft_reset(rdev);
1949
1950         if (reset_mask)
1951                 evergreen_gpu_pci_config_reset(rdev);
1952
1953         r600_set_bios_scratch_engine_hung(rdev, false);
1954
1955         return 0;
1956 }
1957
1958 /**
1959  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1960  *
1961  * @rdev: radeon_device pointer
1962  * @ring: radeon_ring structure holding ring information
1963  *
1964  * Check if the GFX engine is locked up.
1965  * Returns true if the engine appears to be locked up, false if not.
1966  */
1967 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1968 {
1969         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1970
1971         if (!(reset_mask & (RADEON_RESET_GFX |
1972                             RADEON_RESET_COMPUTE |
1973                             RADEON_RESET_CP))) {
1974                 radeon_ring_lockup_update(rdev, ring);
1975                 return false;
1976         }
1977         return radeon_ring_test_lockup(rdev, ring);
1978 }
1979
1980 static int cayman_startup(struct radeon_device *rdev)
1981 {
1982         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1983         int r;
1984
1985         /* enable pcie gen2 link */
1986         evergreen_pcie_gen2_enable(rdev);
1987         /* enable aspm */
1988         evergreen_program_aspm(rdev);
1989
1990         /* scratch needs to be initialized before MC */
1991         r = r600_vram_scratch_init(rdev);
1992         if (r)
1993                 return r;
1994
1995         evergreen_mc_program(rdev);
1996
1997         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1998                 r = ni_mc_load_microcode(rdev);
1999                 if (r) {
2000                         DRM_ERROR("Failed to load MC firmware!\n");
2001                         return r;
2002                 }
2003         }
2004
2005         r = cayman_pcie_gart_enable(rdev);
2006         if (r)
2007                 return r;
2008         cayman_gpu_init(rdev);
2009
2010         /* allocate rlc buffers */
2011         if (rdev->flags & RADEON_IS_IGP) {
2012                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2013                 rdev->rlc.reg_list_size =
2014                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2015                 rdev->rlc.cs_data = cayman_cs_data;
2016                 r = sumo_rlc_init(rdev);
2017                 if (r) {
2018                         DRM_ERROR("Failed to init rlc BOs!\n");
2019                         return r;
2020                 }
2021         }
2022
2023         /* allocate wb buffer */
2024         r = radeon_wb_init(rdev);
2025         if (r)
2026                 return r;
2027
2028         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2029         if (r) {
2030                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2031                 return r;
2032         }
2033
2034         r = uvd_v2_2_resume(rdev);
2035         if (!r) {
2036                 r = radeon_fence_driver_start_ring(rdev,
2037                                                    R600_RING_TYPE_UVD_INDEX);
2038                 if (r)
2039                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2040         }
2041         if (r)
2042                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2043
2044         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2045         if (r) {
2046                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2047                 return r;
2048         }
2049
2050         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2051         if (r) {
2052                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2053                 return r;
2054         }
2055
2056         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2057         if (r) {
2058                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2059                 return r;
2060         }
2061
2062         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2063         if (r) {
2064                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2065                 return r;
2066         }
2067
2068         /* Enable IRQ */
2069         if (!rdev->irq.installed) {
2070                 r = radeon_irq_kms_init(rdev);
2071                 if (r)
2072                         return r;
2073         }
2074
2075         r = r600_irq_init(rdev);
2076         if (r) {
2077                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2078                 radeon_irq_kms_fini(rdev);
2079                 return r;
2080         }
2081         evergreen_irq_set(rdev);
2082
2083         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2084                              RADEON_CP_PACKET2);
2085         if (r)
2086                 return r;
2087
2088         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2089         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2090                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2091         if (r)
2092                 return r;
2093
2094         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2095         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2096                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2097         if (r)
2098                 return r;
2099
2100         r = cayman_cp_load_microcode(rdev);
2101         if (r)
2102                 return r;
2103         r = cayman_cp_resume(rdev);
2104         if (r)
2105                 return r;
2106
2107         r = cayman_dma_resume(rdev);
2108         if (r)
2109                 return r;
2110
2111         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2112         if (ring->ring_size) {
2113                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2114                                      RADEON_CP_PACKET2);
2115                 if (!r)
2116                         r = uvd_v1_0_init(rdev);
2117                 if (r)
2118                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2119         }
2120
2121         r = radeon_ib_pool_init(rdev);
2122         if (r) {
2123                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2124                 return r;
2125         }
2126
2127         r = radeon_vm_manager_init(rdev);
2128         if (r) {
2129                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2130                 return r;
2131         }
2132
2133         r = radeon_audio_init(rdev);
2134         if (r)
2135                 return r;
2136
2137         return 0;
2138 }
2139
2140 int cayman_resume(struct radeon_device *rdev)
2141 {
2142         int r;
2143
2144         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2145          * posting will perform necessary task to bring back GPU into good
2146          * shape.
2147          */
2148         /* post card */
2149         atom_asic_init(rdev->mode_info.atom_context);
2150
2151         /* init golden registers */
2152         ni_init_golden_registers(rdev);
2153
2154         if (rdev->pm.pm_method == PM_METHOD_DPM)
2155                 radeon_pm_resume(rdev);
2156
2157         rdev->accel_working = true;
2158         r = cayman_startup(rdev);
2159         if (r) {
2160                 DRM_ERROR("cayman startup failed on resume\n");
2161                 rdev->accel_working = false;
2162                 return r;
2163         }
2164         return r;
2165 }
2166
2167 int cayman_suspend(struct radeon_device *rdev)
2168 {
2169         radeon_pm_suspend(rdev);
2170         radeon_audio_fini(rdev);
2171         radeon_vm_manager_fini(rdev);
2172         cayman_cp_enable(rdev, false);
2173         cayman_dma_stop(rdev);
2174         uvd_v1_0_fini(rdev);
2175         radeon_uvd_suspend(rdev);
2176         evergreen_irq_suspend(rdev);
2177         radeon_wb_disable(rdev);
2178         cayman_pcie_gart_disable(rdev);
2179         return 0;
2180 }
2181
2182 /* Plan is to move initialization in that function and use
2183  * helper function so that radeon_device_init pretty much
2184  * do nothing more than calling asic specific function. This
2185  * should also allow to remove a bunch of callback function
2186  * like vram_info.
2187  */
2188 int cayman_init(struct radeon_device *rdev)
2189 {
2190         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2191         int r;
2192
2193         /* Read BIOS */
2194         if (!radeon_get_bios(rdev)) {
2195                 if (ASIC_IS_AVIVO(rdev))
2196                         return -EINVAL;
2197         }
2198         /* Must be an ATOMBIOS */
2199         if (!rdev->is_atom_bios) {
2200                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2201                 return -EINVAL;
2202         }
2203         r = radeon_atombios_init(rdev);
2204         if (r)
2205                 return r;
2206
2207         /* Post card if necessary */
2208         if (!radeon_card_posted(rdev)) {
2209                 if (!rdev->bios) {
2210                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2211                         return -EINVAL;
2212                 }
2213                 DRM_INFO("GPU not posted. posting now...\n");
2214                 atom_asic_init(rdev->mode_info.atom_context);
2215         }
2216         /* init golden registers */
2217         ni_init_golden_registers(rdev);
2218         /* Initialize scratch registers */
2219         r600_scratch_init(rdev);
2220         /* Initialize surface registers */
2221         radeon_surface_init(rdev);
2222         /* Initialize clocks */
2223         radeon_get_clock_info(rdev->ddev);
2224         /* Fence driver */
2225         r = radeon_fence_driver_init(rdev);
2226         if (r)
2227                 return r;
2228         /* initialize memory controller */
2229         r = evergreen_mc_init(rdev);
2230         if (r)
2231                 return r;
2232         /* Memory manager */
2233         r = radeon_bo_init(rdev);
2234         if (r)
2235                 return r;
2236
2237         if (rdev->flags & RADEON_IS_IGP) {
2238                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2239                         r = ni_init_microcode(rdev);
2240                         if (r) {
2241                                 DRM_ERROR("Failed to load firmware!\n");
2242                                 return r;
2243                         }
2244                 }
2245         } else {
2246                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2247                         r = ni_init_microcode(rdev);
2248                         if (r) {
2249                                 DRM_ERROR("Failed to load firmware!\n");
2250                                 return r;
2251                         }
2252                 }
2253         }
2254
2255         /* Initialize power management */
2256         radeon_pm_init(rdev);
2257
2258         ring->ring_obj = NULL;
2259         r600_ring_init(rdev, ring, 1024 * 1024);
2260
2261         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2262         ring->ring_obj = NULL;
2263         r600_ring_init(rdev, ring, 64 * 1024);
2264
2265         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2266         ring->ring_obj = NULL;
2267         r600_ring_init(rdev, ring, 64 * 1024);
2268
2269         r = radeon_uvd_init(rdev);
2270         if (!r) {
2271                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2272                 ring->ring_obj = NULL;
2273                 r600_ring_init(rdev, ring, 4096);
2274         }
2275
2276         rdev->ih.ring_obj = NULL;
2277         r600_ih_ring_init(rdev, 64 * 1024);
2278
2279         r = r600_pcie_gart_init(rdev);
2280         if (r)
2281                 return r;
2282
2283         rdev->accel_working = true;
2284         r = cayman_startup(rdev);
2285         if (r) {
2286                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2287                 cayman_cp_fini(rdev);
2288                 cayman_dma_fini(rdev);
2289                 r600_irq_fini(rdev);
2290                 if (rdev->flags & RADEON_IS_IGP)
2291                         sumo_rlc_fini(rdev);
2292                 radeon_wb_fini(rdev);
2293                 radeon_ib_pool_fini(rdev);
2294                 radeon_vm_manager_fini(rdev);
2295                 radeon_irq_kms_fini(rdev);
2296                 cayman_pcie_gart_fini(rdev);
2297                 rdev->accel_working = false;
2298         }
2299
2300         /* Don't start up if the MC ucode is missing.
2301          * The default clocks and voltages before the MC ucode
2302          * is loaded are not suffient for advanced operations.
2303          *
2304          * We can skip this check for TN, because there is no MC
2305          * ucode.
2306          */
2307         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2308                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2309                 return -EINVAL;
2310         }
2311
2312         return 0;
2313 }
2314
2315 void cayman_fini(struct radeon_device *rdev)
2316 {
2317         radeon_pm_fini(rdev);
2318         cayman_cp_fini(rdev);
2319         cayman_dma_fini(rdev);
2320         r600_irq_fini(rdev);
2321         if (rdev->flags & RADEON_IS_IGP)
2322                 sumo_rlc_fini(rdev);
2323         radeon_wb_fini(rdev);
2324         radeon_vm_manager_fini(rdev);
2325         radeon_ib_pool_fini(rdev);
2326         radeon_irq_kms_fini(rdev);
2327         uvd_v1_0_fini(rdev);
2328         radeon_uvd_fini(rdev);
2329         cayman_pcie_gart_fini(rdev);
2330         r600_vram_scratch_fini(rdev);
2331         radeon_gem_fini(rdev);
2332         radeon_fence_driver_fini(rdev);
2333         radeon_bo_fini(rdev);
2334         radeon_atombios_fini(rdev);
2335         kfree(rdev->bios);
2336         rdev->bios = NULL;
2337 }
2338
2339 /*
2340  * vm
2341  */
2342 int cayman_vm_init(struct radeon_device *rdev)
2343 {
2344         /* number of VMs */
2345         rdev->vm_manager.nvm = 8;
2346         /* base offset of vram pages */
2347         if (rdev->flags & RADEON_IS_IGP) {
2348                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2349                 tmp <<= 22;
2350                 rdev->vm_manager.vram_base_offset = tmp;
2351         } else
2352                 rdev->vm_manager.vram_base_offset = 0;
2353         return 0;
2354 }
2355
2356 void cayman_vm_fini(struct radeon_device *rdev)
2357 {
2358 }
2359
2360 /**
2361  * cayman_vm_decode_fault - print human readable fault info
2362  *
2363  * @rdev: radeon_device pointer
2364  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2365  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2366  *
2367  * Print human readable fault information (cayman/TN).
2368  */
2369 void cayman_vm_decode_fault(struct radeon_device *rdev,
2370                             u32 status, u32 addr)
2371 {
2372         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2373         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2374         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2375         char *block;
2376
2377         switch (mc_id) {
2378         case 32:
2379         case 16:
2380         case 96:
2381         case 80:
2382         case 160:
2383         case 144:
2384         case 224:
2385         case 208:
2386                 block = "CB";
2387                 break;
2388         case 33:
2389         case 17:
2390         case 97:
2391         case 81:
2392         case 161:
2393         case 145:
2394         case 225:
2395         case 209:
2396                 block = "CB_FMASK";
2397                 break;
2398         case 34:
2399         case 18:
2400         case 98:
2401         case 82:
2402         case 162:
2403         case 146:
2404         case 226:
2405         case 210:
2406                 block = "CB_CMASK";
2407                 break;
2408         case 35:
2409         case 19:
2410         case 99:
2411         case 83:
2412         case 163:
2413         case 147:
2414         case 227:
2415         case 211:
2416                 block = "CB_IMMED";
2417                 break;
2418         case 36:
2419         case 20:
2420         case 100:
2421         case 84:
2422         case 164:
2423         case 148:
2424         case 228:
2425         case 212:
2426                 block = "DB";
2427                 break;
2428         case 37:
2429         case 21:
2430         case 101:
2431         case 85:
2432         case 165:
2433         case 149:
2434         case 229:
2435         case 213:
2436                 block = "DB_HTILE";
2437                 break;
2438         case 38:
2439         case 22:
2440         case 102:
2441         case 86:
2442         case 166:
2443         case 150:
2444         case 230:
2445         case 214:
2446                 block = "SX";
2447                 break;
2448         case 39:
2449         case 23:
2450         case 103:
2451         case 87:
2452         case 167:
2453         case 151:
2454         case 231:
2455         case 215:
2456                 block = "DB_STEN";
2457                 break;
2458         case 40:
2459         case 24:
2460         case 104:
2461         case 88:
2462         case 232:
2463         case 216:
2464         case 168:
2465         case 152:
2466                 block = "TC_TFETCH";
2467                 break;
2468         case 41:
2469         case 25:
2470         case 105:
2471         case 89:
2472         case 233:
2473         case 217:
2474         case 169:
2475         case 153:
2476                 block = "TC_VFETCH";
2477                 break;
2478         case 42:
2479         case 26:
2480         case 106:
2481         case 90:
2482         case 234:
2483         case 218:
2484         case 170:
2485         case 154:
2486                 block = "VC";
2487                 break;
2488         case 112:
2489                 block = "CP";
2490                 break;
2491         case 113:
2492         case 114:
2493                 block = "SH";
2494                 break;
2495         case 115:
2496                 block = "VGT";
2497                 break;
2498         case 178:
2499                 block = "IH";
2500                 break;
2501         case 51:
2502                 block = "RLC";
2503                 break;
2504         case 55:
2505                 block = "DMA";
2506                 break;
2507         case 56:
2508                 block = "HDP";
2509                 break;
2510         default:
2511                 block = "unknown";
2512                 break;
2513         }
2514
2515         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2516                protections, vmid, addr,
2517                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2518                block, mc_id);
2519 }
2520
2521 /**
2522  * cayman_vm_flush - vm flush using the CP
2523  *
2524  * @rdev: radeon_device pointer
2525  *
2526  * Update the page table base and flush the VM TLB
2527  * using the CP (cayman-si).
2528  */
2529 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2530                      unsigned vm_id, uint64_t pd_addr)
2531 {
2532         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2533         radeon_ring_write(ring, pd_addr >> 12);
2534
2535         /* flush hdp cache */
2536         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2537         radeon_ring_write(ring, 0x1);
2538
2539         /* bits 0-7 are the VM contexts0-7 */
2540         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2541         radeon_ring_write(ring, 1 << vm_id);
2542
2543         /* wait for the invalidate to complete */
2544         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2545         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2546                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2547         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2548         radeon_ring_write(ring, 0);
2549         radeon_ring_write(ring, 0); /* ref */
2550         radeon_ring_write(ring, 0); /* mask */
2551         radeon_ring_write(ring, 0x20); /* poll interval */
2552
2553         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2554         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2555         radeon_ring_write(ring, 0x0);
2556 }