Merge remote-tracking branch 'airlied/drm-next' into drm-misc-next
[sfrench/cifs-2.6.git] / drivers / gpu / drm / amd / amdgpu / gmc_v6_0.c
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <drm/drmP.h>
25 #include "amdgpu.h"
26 #include "gmc_v6_0.h"
27 #include "amdgpu_ucode.h"
28
29 #include "bif/bif_3_0_d.h"
30 #include "bif/bif_3_0_sh_mask.h"
31 #include "oss/oss_1_0_d.h"
32 #include "oss/oss_1_0_sh_mask.h"
33 #include "gmc/gmc_6_0_d.h"
34 #include "gmc/gmc_6_0_sh_mask.h"
35 #include "dce/dce_6_0_d.h"
36 #include "dce/dce_6_0_sh_mask.h"
37 #include "si_enums.h"
38
39 static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev);
40 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
41 static int gmc_v6_0_wait_for_idle(void *handle);
42
43 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
44 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
45 MODULE_FIRMWARE("radeon/verde_mc.bin");
46 MODULE_FIRMWARE("radeon/oland_mc.bin");
47 MODULE_FIRMWARE("radeon/si58_mc.bin");
48
49 #define MC_SEQ_MISC0__MT__MASK   0xf0000000
50 #define MC_SEQ_MISC0__MT__GDDR1  0x10000000
51 #define MC_SEQ_MISC0__MT__DDR2   0x20000000
52 #define MC_SEQ_MISC0__MT__GDDR3  0x30000000
53 #define MC_SEQ_MISC0__MT__GDDR4  0x40000000
54 #define MC_SEQ_MISC0__MT__GDDR5  0x50000000
55 #define MC_SEQ_MISC0__MT__HBM    0x60000000
56 #define MC_SEQ_MISC0__MT__DDR3   0xB0000000
57
58
59 static const u32 crtc_offsets[6] =
60 {
61         SI_CRTC0_REGISTER_OFFSET,
62         SI_CRTC1_REGISTER_OFFSET,
63         SI_CRTC2_REGISTER_OFFSET,
64         SI_CRTC3_REGISTER_OFFSET,
65         SI_CRTC4_REGISTER_OFFSET,
66         SI_CRTC5_REGISTER_OFFSET
67 };
68
69 static void gmc_v6_0_mc_stop(struct amdgpu_device *adev,
70                              struct amdgpu_mode_mc_save *save)
71 {
72         u32 blackout;
73
74         if (adev->mode_info.num_crtc)
75                 amdgpu_display_stop_mc_access(adev, save);
76
77         gmc_v6_0_wait_for_idle((void *)adev);
78
79         blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
80         if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
81                 /* Block CPU access */
82                 WREG32(mmBIF_FB_EN, 0);
83                 /* blackout the MC */
84                 blackout = REG_SET_FIELD(blackout,
85                                          MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
86                 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
87         }
88         /* wait for the MC to settle */
89         udelay(100);
90
91 }
92
93 static void gmc_v6_0_mc_resume(struct amdgpu_device *adev,
94                                struct amdgpu_mode_mc_save *save)
95 {
96         u32 tmp;
97
98         /* unblackout the MC */
99         tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
100         tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
101         WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
102         /* allow CPU access */
103         tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
104         tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
105         WREG32(mmBIF_FB_EN, tmp);
106
107         if (adev->mode_info.num_crtc)
108                 amdgpu_display_resume_mc_access(adev, save);
109
110 }
111
112 static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
113 {
114         const char *chip_name;
115         char fw_name[30];
116         int err;
117         bool is_58_fw = false;
118
119         DRM_DEBUG("\n");
120
121         switch (adev->asic_type) {
122         case CHIP_TAHITI:
123                 chip_name = "tahiti";
124                 break;
125         case CHIP_PITCAIRN:
126                 chip_name = "pitcairn";
127                 break;
128         case CHIP_VERDE:
129                 chip_name = "verde";
130                 break;
131         case CHIP_OLAND:
132                 chip_name = "oland";
133                 break;
134         case CHIP_HAINAN:
135                 chip_name = "hainan";
136                 break;
137         default: BUG();
138         }
139
140         /* this memory configuration requires special firmware */
141         if (((RREG32(mmMC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
142                 is_58_fw = true;
143
144         if (is_58_fw)
145                 snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
146         else
147                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
148         err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
149         if (err)
150                 goto out;
151
152         err = amdgpu_ucode_validate(adev->mc.fw);
153
154 out:
155         if (err) {
156                 dev_err(adev->dev,
157                        "si_mc: Failed to load firmware \"%s\"\n",
158                        fw_name);
159                 release_firmware(adev->mc.fw);
160                 adev->mc.fw = NULL;
161         }
162         return err;
163 }
164
165 static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
166 {
167         const __le32 *new_fw_data = NULL;
168         u32 running;
169         const __le32 *new_io_mc_regs = NULL;
170         int i, regs_size, ucode_size;
171         const struct mc_firmware_header_v1_0 *hdr;
172
173         if (!adev->mc.fw)
174                 return -EINVAL;
175
176         hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
177
178         amdgpu_ucode_print_mc_hdr(&hdr->header);
179
180         adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
181         regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
182         new_io_mc_regs = (const __le32 *)
183                 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
184         ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
185         new_fw_data = (const __le32 *)
186                 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
187
188         running = RREG32(mmMC_SEQ_SUP_CNTL) & MC_SEQ_SUP_CNTL__RUN_MASK;
189
190         if (running == 0) {
191
192                 /* reset the engine and set to writable */
193                 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
194                 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
195
196                 /* load mc io regs */
197                 for (i = 0; i < regs_size; i++) {
198                         WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
199                         WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
200                 }
201                 /* load the MC ucode */
202                 for (i = 0; i < ucode_size; i++) {
203                         WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
204                 }
205
206                 /* put the engine back into the active state */
207                 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
208                 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
209                 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
210
211                 /* wait for training to complete */
212                 for (i = 0; i < adev->usec_timeout; i++) {
213                         if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK)
214                                 break;
215                         udelay(1);
216                 }
217                 for (i = 0; i < adev->usec_timeout; i++) {
218                         if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK)
219                                 break;
220                         udelay(1);
221                 }
222
223         }
224
225         return 0;
226 }
227
228 static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
229                                        struct amdgpu_mc *mc)
230 {
231         if (mc->mc_vram_size > 0xFFC0000000ULL) {
232                 dev_warn(adev->dev, "limiting VRAM\n");
233                 mc->real_vram_size = 0xFFC0000000ULL;
234                 mc->mc_vram_size = 0xFFC0000000ULL;
235         }
236         amdgpu_vram_location(adev, &adev->mc, 0);
237         adev->mc.gtt_base_align = 0;
238         amdgpu_gtt_location(adev, mc);
239 }
240
241 static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
242 {
243         struct amdgpu_mode_mc_save save;
244         u32 tmp;
245         int i, j;
246
247         /* Initialize HDP */
248         for (i = 0, j = 0; i < 32; i++, j += 0x6) {
249                 WREG32((0xb05 + j), 0x00000000);
250                 WREG32((0xb06 + j), 0x00000000);
251                 WREG32((0xb07 + j), 0x00000000);
252                 WREG32((0xb08 + j), 0x00000000);
253                 WREG32((0xb09 + j), 0x00000000);
254         }
255         WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
256
257         if (adev->mode_info.num_crtc)
258                 amdgpu_display_set_vga_render_state(adev, false);
259
260         gmc_v6_0_mc_stop(adev, &save);
261
262         if (gmc_v6_0_wait_for_idle((void *)adev)) {
263                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
264         }
265
266         WREG32(mmVGA_HDP_CONTROL, VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK);
267         /* Update configuration */
268         WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
269                adev->mc.vram_start >> 12);
270         WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
271                adev->mc.vram_end >> 12);
272         WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
273                adev->vram_scratch.gpu_addr >> 12);
274         tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
275         tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
276         WREG32(mmMC_VM_FB_LOCATION, tmp);
277         /* XXX double check these! */
278         WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
279         WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
280         WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
281         WREG32(mmMC_VM_AGP_BASE, 0);
282         WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
283         WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
284
285         if (gmc_v6_0_wait_for_idle((void *)adev)) {
286                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
287         }
288         gmc_v6_0_mc_resume(adev, &save);
289 }
290
291 static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
292 {
293
294         u32 tmp;
295         int chansize, numchan;
296
297         tmp = RREG32(mmMC_ARB_RAMCFG);
298         if (tmp & (1 << 11)) {
299                 chansize = 16;
300         } else if (tmp & MC_ARB_RAMCFG__CHANSIZE_MASK) {
301                 chansize = 64;
302         } else {
303                 chansize = 32;
304         }
305         tmp = RREG32(mmMC_SHARED_CHMAP);
306         switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
307         case 0:
308         default:
309                 numchan = 1;
310                 break;
311         case 1:
312                 numchan = 2;
313                 break;
314         case 2:
315                 numchan = 4;
316                 break;
317         case 3:
318                 numchan = 8;
319                 break;
320         case 4:
321                 numchan = 3;
322                 break;
323         case 5:
324                 numchan = 6;
325                 break;
326         case 6:
327                 numchan = 10;
328                 break;
329         case 7:
330                 numchan = 12;
331                 break;
332         case 8:
333                 numchan = 16;
334                 break;
335         }
336         adev->mc.vram_width = numchan * chansize;
337         /* Could aper size report 0 ? */
338         adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
339         adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
340         /* size in MB on si */
341         adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
342         adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
343         adev->mc.visible_vram_size = adev->mc.aper_size;
344
345         /* unless the user had overridden it, set the gart
346          * size equal to the 1024 or vram, whichever is larger.
347          */
348         if (amdgpu_gart_size == -1)
349                 adev->mc.gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
350                                         adev->mc.mc_vram_size);
351         else
352                 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
353
354         gmc_v6_0_vram_gtt_location(adev, &adev->mc);
355
356         return 0;
357 }
358
359 static void gmc_v6_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
360                                         uint32_t vmid)
361 {
362         WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
363
364         WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
365 }
366
367 static int gmc_v6_0_gart_set_pte_pde(struct amdgpu_device *adev,
368                                      void *cpu_pt_addr,
369                                      uint32_t gpu_page_idx,
370                                      uint64_t addr,
371                                      uint64_t flags)
372 {
373         void __iomem *ptr = (void *)cpu_pt_addr;
374         uint64_t value;
375
376         value = addr & 0xFFFFFFFFFFFFF000ULL;
377         value |= flags;
378         writeq(value, ptr + (gpu_page_idx * 8));
379
380         return 0;
381 }
382
383 static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,
384                                           uint32_t flags)
385 {
386         uint64_t pte_flag = 0;
387
388         if (flags & AMDGPU_VM_PAGE_READABLE)
389                 pte_flag |= AMDGPU_PTE_READABLE;
390         if (flags & AMDGPU_VM_PAGE_WRITEABLE)
391                 pte_flag |= AMDGPU_PTE_WRITEABLE;
392         if (flags & AMDGPU_VM_PAGE_PRT)
393                 pte_flag |= AMDGPU_PTE_PRT;
394
395         return pte_flag;
396 }
397
398 static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
399                                               bool value)
400 {
401         u32 tmp;
402
403         tmp = RREG32(mmVM_CONTEXT1_CNTL);
404         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
405                             RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
406         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
407                             DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
408         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
409                             PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
410         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
411                             VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
412         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
413                             READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
414         tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
415                             WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
416         WREG32(mmVM_CONTEXT1_CNTL, tmp);
417 }
418
419  /**
420    + * gmc_v8_0_set_prt - set PRT VM fault
421    + *
422    + * @adev: amdgpu_device pointer
423    + * @enable: enable/disable VM fault handling for PRT
424    +*/
425 static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
426 {
427         u32 tmp;
428
429         if (enable && !adev->mc.prt_warning) {
430                 dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
431                 adev->mc.prt_warning = true;
432         }
433
434         tmp = RREG32(mmVM_PRT_CNTL);
435         tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
436                             CB_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
437                             enable);
438         tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
439                             TC_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
440                             enable);
441         tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
442                             L2_CACHE_STORE_INVALID_ENTRIES,
443                             enable);
444         tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
445                             L1_TLB_STORE_INVALID_ENTRIES,
446                             enable);
447         WREG32(mmVM_PRT_CNTL, tmp);
448
449         if (enable) {
450                 uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
451                 uint32_t high = adev->vm_manager.max_pfn;
452
453                 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
454                 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
455                 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
456                 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
457                 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
458                 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
459                 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
460                 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
461         } else {
462                 WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
463                 WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
464                 WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
465                 WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
466                 WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
467                 WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
468                 WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
469                 WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
470         }
471 }
472
473 static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
474 {
475         int r, i;
476
477         if (adev->gart.robj == NULL) {
478                 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
479                 return -EINVAL;
480         }
481         r = amdgpu_gart_table_vram_pin(adev);
482         if (r)
483                 return r;
484         /* Setup TLB control */
485         WREG32(mmMC_VM_MX_L1_TLB_CNTL,
486                (0xA << 7) |
487                MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK |
488                MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK |
489                MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
490                MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK |
491                (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
492         /* Setup L2 cache */
493         WREG32(mmVM_L2_CNTL,
494                VM_L2_CNTL__ENABLE_L2_CACHE_MASK |
495                VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK |
496                VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
497                VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
498                (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
499                (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
500         WREG32(mmVM_L2_CNTL2,
501                VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
502                VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
503         WREG32(mmVM_L2_CNTL3,
504                VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
505                (4UL << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
506                (4UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
507         /* setup context0 */
508         WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
509         WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
510         WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
511         WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
512                         (u32)(adev->dummy_page.addr >> 12));
513         WREG32(mmVM_CONTEXT0_CNTL2, 0);
514         WREG32(mmVM_CONTEXT0_CNTL,
515                VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK |
516                (0UL << VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
517                VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
518
519         WREG32(0x575, 0);
520         WREG32(0x576, 0);
521         WREG32(0x577, 0);
522
523         /* empty context1-15 */
524         /* set vm size, must be a multiple of 4 */
525         WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
526         WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
527         /* Assign the pt base to something valid for now; the pts used for
528          * the VMs are determined by the application and setup and assigned
529          * on the fly in the vm part of radeon_gart.c
530          */
531         for (i = 1; i < 16; i++) {
532                 if (i < 8)
533                         WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
534                                adev->gart.table_addr >> 12);
535                 else
536                         WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
537                                adev->gart.table_addr >> 12);
538         }
539
540         /* enable context1-15 */
541         WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
542                (u32)(adev->dummy_page.addr >> 12));
543         WREG32(mmVM_CONTEXT1_CNTL2, 4);
544         WREG32(mmVM_CONTEXT1_CNTL,
545                VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
546                (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
547                ((adev->vm_manager.block_size - 9)
548                << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT));
549         if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
550                 gmc_v6_0_set_fault_enable_default(adev, false);
551         else
552                 gmc_v6_0_set_fault_enable_default(adev, true);
553
554         gmc_v6_0_gart_flush_gpu_tlb(adev, 0);
555         dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
556                  (unsigned)(adev->mc.gtt_size >> 20),
557                  (unsigned long long)adev->gart.table_addr);
558         adev->gart.ready = true;
559         return 0;
560 }
561
562 static int gmc_v6_0_gart_init(struct amdgpu_device *adev)
563 {
564         int r;
565
566         if (adev->gart.robj) {
567                 dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");
568                 return 0;
569         }
570         r = amdgpu_gart_init(adev);
571         if (r)
572                 return r;
573         adev->gart.table_size = adev->gart.num_gpu_pages * 8;
574         adev->gart.gart_pte_flags = 0;
575         return amdgpu_gart_table_vram_alloc(adev);
576 }
577
578 static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
579 {
580         /*unsigned i;
581
582         for (i = 1; i < 16; ++i) {
583                 uint32_t reg;
584                 if (i < 8)
585                         reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i ;
586                 else
587                         reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (i - 8);
588                 adev->vm_manager.saved_table_addr[i] = RREG32(reg);
589         }*/
590
591         /* Disable all tables */
592         WREG32(mmVM_CONTEXT0_CNTL, 0);
593         WREG32(mmVM_CONTEXT1_CNTL, 0);
594         /* Setup TLB control */
595         WREG32(mmMC_VM_MX_L1_TLB_CNTL,
596                MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
597                (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
598         /* Setup L2 cache */
599         WREG32(mmVM_L2_CNTL,
600                VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
601                VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
602                (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
603                (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
604         WREG32(mmVM_L2_CNTL2, 0);
605         WREG32(mmVM_L2_CNTL3,
606                VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
607                (0UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
608         amdgpu_gart_table_vram_unpin(adev);
609 }
610
611 static void gmc_v6_0_gart_fini(struct amdgpu_device *adev)
612 {
613         amdgpu_gart_table_vram_free(adev);
614         amdgpu_gart_fini(adev);
615 }
616
617 static int gmc_v6_0_vm_init(struct amdgpu_device *adev)
618 {
619         /*
620          * number of VMs
621          * VMID 0 is reserved for System
622          * amdgpu graphics/compute will use VMIDs 1-7
623          * amdkfd will use VMIDs 8-15
624          */
625         adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
626         adev->vm_manager.num_level = 1;
627         amdgpu_vm_manager_init(adev);
628
629         /* base offset of vram pages */
630         if (adev->flags & AMD_IS_APU) {
631                 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
632                 tmp <<= 22;
633                 adev->vm_manager.vram_base_offset = tmp;
634         } else
635                 adev->vm_manager.vram_base_offset = 0;
636
637         return 0;
638 }
639
640 static void gmc_v6_0_vm_fini(struct amdgpu_device *adev)
641 {
642 }
643
644 static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
645                                      u32 status, u32 addr, u32 mc_client)
646 {
647         u32 mc_id;
648         u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
649         u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
650                                         PROTECTIONS);
651         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
652                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
653
654         mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
655                               MEMORY_CLIENT_ID);
656
657         dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
658                protections, vmid, addr,
659                REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
660                              MEMORY_CLIENT_RW) ?
661                "write" : "read", block, mc_client, mc_id);
662 }
663
664 /*
665 static const u32 mc_cg_registers[] = {
666         MC_HUB_MISC_HUB_CG,
667         MC_HUB_MISC_SIP_CG,
668         MC_HUB_MISC_VM_CG,
669         MC_XPB_CLK_GAT,
670         ATC_MISC_CG,
671         MC_CITF_MISC_WR_CG,
672         MC_CITF_MISC_RD_CG,
673         MC_CITF_MISC_VM_CG,
674         VM_L2_CG,
675 };
676
677 static const u32 mc_cg_ls_en[] = {
678         MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
679         MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
680         MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
681         MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
682         ATC_MISC_CG__MEM_LS_ENABLE_MASK,
683         MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
684         MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
685         MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
686         VM_L2_CG__MEM_LS_ENABLE_MASK,
687 };
688
689 static const u32 mc_cg_en[] = {
690         MC_HUB_MISC_HUB_CG__ENABLE_MASK,
691         MC_HUB_MISC_SIP_CG__ENABLE_MASK,
692         MC_HUB_MISC_VM_CG__ENABLE_MASK,
693         MC_XPB_CLK_GAT__ENABLE_MASK,
694         ATC_MISC_CG__ENABLE_MASK,
695         MC_CITF_MISC_WR_CG__ENABLE_MASK,
696         MC_CITF_MISC_RD_CG__ENABLE_MASK,
697         MC_CITF_MISC_VM_CG__ENABLE_MASK,
698         VM_L2_CG__ENABLE_MASK,
699 };
700
701 static void gmc_v6_0_enable_mc_ls(struct amdgpu_device *adev,
702                                   bool enable)
703 {
704         int i;
705         u32 orig, data;
706
707         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
708                 orig = data = RREG32(mc_cg_registers[i]);
709                 if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_LS))
710                         data |= mc_cg_ls_en[i];
711                 else
712                         data &= ~mc_cg_ls_en[i];
713                 if (data != orig)
714                         WREG32(mc_cg_registers[i], data);
715         }
716 }
717
718 static void gmc_v6_0_enable_mc_mgcg(struct amdgpu_device *adev,
719                                     bool enable)
720 {
721         int i;
722         u32 orig, data;
723
724         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
725                 orig = data = RREG32(mc_cg_registers[i]);
726                 if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_MGCG))
727                         data |= mc_cg_en[i];
728                 else
729                         data &= ~mc_cg_en[i];
730                 if (data != orig)
731                         WREG32(mc_cg_registers[i], data);
732         }
733 }
734
735 static void gmc_v6_0_enable_bif_mgls(struct amdgpu_device *adev,
736                                      bool enable)
737 {
738         u32 orig, data;
739
740         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
741
742         if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_BIF_LS)) {
743                 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
744                 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
745                 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
746                 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
747         } else {
748                 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
749                 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
750                 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
751                 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
752         }
753
754         if (orig != data)
755                 WREG32_PCIE(ixPCIE_CNTL2, data);
756 }
757
758 static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
759                                      bool enable)
760 {
761         u32 orig, data;
762
763         orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
764
765         if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_MGCG))
766                 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
767         else
768                 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
769
770         if (orig != data)
771                 WREG32(mmHDP_HOST_PATH_CNTL, data);
772 }
773
774 static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
775                                    bool enable)
776 {
777         u32 orig, data;
778
779         orig = data = RREG32(mmHDP_MEM_POWER_LS);
780
781         if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_LS))
782                 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
783         else
784                 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
785
786         if (orig != data)
787                 WREG32(mmHDP_MEM_POWER_LS, data);
788 }
789 */
790
791 static int gmc_v6_0_convert_vram_type(int mc_seq_vram_type)
792 {
793         switch (mc_seq_vram_type) {
794         case MC_SEQ_MISC0__MT__GDDR1:
795                 return AMDGPU_VRAM_TYPE_GDDR1;
796         case MC_SEQ_MISC0__MT__DDR2:
797                 return AMDGPU_VRAM_TYPE_DDR2;
798         case MC_SEQ_MISC0__MT__GDDR3:
799                 return AMDGPU_VRAM_TYPE_GDDR3;
800         case MC_SEQ_MISC0__MT__GDDR4:
801                 return AMDGPU_VRAM_TYPE_GDDR4;
802         case MC_SEQ_MISC0__MT__GDDR5:
803                 return AMDGPU_VRAM_TYPE_GDDR5;
804         case MC_SEQ_MISC0__MT__DDR3:
805                 return AMDGPU_VRAM_TYPE_DDR3;
806         default:
807                 return AMDGPU_VRAM_TYPE_UNKNOWN;
808         }
809 }
810
811 static int gmc_v6_0_early_init(void *handle)
812 {
813         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
814
815         gmc_v6_0_set_gart_funcs(adev);
816         gmc_v6_0_set_irq_funcs(adev);
817
818         if (adev->flags & AMD_IS_APU) {
819                 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
820         } else {
821                 u32 tmp = RREG32(mmMC_SEQ_MISC0);
822                 tmp &= MC_SEQ_MISC0__MT__MASK;
823                 adev->mc.vram_type = gmc_v6_0_convert_vram_type(tmp);
824         }
825
826         return 0;
827 }
828
829 static int gmc_v6_0_late_init(void *handle)
830 {
831         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
832
833         if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
834                 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
835         else
836                 return 0;
837 }
838
839 static int gmc_v6_0_sw_init(void *handle)
840 {
841         int r;
842         int dma_bits;
843         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
844
845         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
846         if (r)
847                 return r;
848
849         r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
850         if (r)
851                 return r;
852
853         amdgpu_vm_adjust_size(adev, 64);
854         adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
855
856         adev->mc.mc_mask = 0xffffffffffULL;
857
858         adev->need_dma32 = false;
859         dma_bits = adev->need_dma32 ? 32 : 40;
860         r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
861         if (r) {
862                 adev->need_dma32 = true;
863                 dma_bits = 32;
864                 dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
865         }
866         r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
867         if (r) {
868                 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
869                 dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n");
870         }
871
872         r = gmc_v6_0_init_microcode(adev);
873         if (r) {
874                 dev_err(adev->dev, "Failed to load mc firmware!\n");
875                 return r;
876         }
877
878         r = gmc_v6_0_mc_init(adev);
879         if (r)
880                 return r;
881
882         r = amdgpu_bo_init(adev);
883         if (r)
884                 return r;
885
886         r = gmc_v6_0_gart_init(adev);
887         if (r)
888                 return r;
889
890         if (!adev->vm_manager.enabled) {
891                 r = gmc_v6_0_vm_init(adev);
892                 if (r) {
893                         dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
894                         return r;
895                 }
896                 adev->vm_manager.enabled = true;
897         }
898
899         return r;
900 }
901
902 static int gmc_v6_0_sw_fini(void *handle)
903 {
904         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
905
906         if (adev->vm_manager.enabled) {
907                 gmc_v6_0_vm_fini(adev);
908                 adev->vm_manager.enabled = false;
909         }
910         gmc_v6_0_gart_fini(adev);
911         amdgpu_gem_force_release(adev);
912         amdgpu_bo_fini(adev);
913
914         return 0;
915 }
916
917 static int gmc_v6_0_hw_init(void *handle)
918 {
919         int r;
920         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
921
922         gmc_v6_0_mc_program(adev);
923
924         if (!(adev->flags & AMD_IS_APU)) {
925                 r = gmc_v6_0_mc_load_microcode(adev);
926                 if (r) {
927                         dev_err(adev->dev, "Failed to load MC firmware!\n");
928                         return r;
929                 }
930         }
931
932         r = gmc_v6_0_gart_enable(adev);
933         if (r)
934                 return r;
935
936         return r;
937 }
938
939 static int gmc_v6_0_hw_fini(void *handle)
940 {
941         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
942
943         amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
944         gmc_v6_0_gart_disable(adev);
945
946         return 0;
947 }
948
949 static int gmc_v6_0_suspend(void *handle)
950 {
951         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
952
953         if (adev->vm_manager.enabled) {
954                 gmc_v6_0_vm_fini(adev);
955                 adev->vm_manager.enabled = false;
956         }
957         gmc_v6_0_hw_fini(adev);
958
959         return 0;
960 }
961
962 static int gmc_v6_0_resume(void *handle)
963 {
964         int r;
965         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
966
967         r = gmc_v6_0_hw_init(adev);
968         if (r)
969                 return r;
970
971         if (!adev->vm_manager.enabled) {
972                 r = gmc_v6_0_vm_init(adev);
973                 if (r) {
974                         dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
975                         return r;
976                 }
977                 adev->vm_manager.enabled = true;
978         }
979
980         return r;
981 }
982
983 static bool gmc_v6_0_is_idle(void *handle)
984 {
985         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
986         u32 tmp = RREG32(mmSRBM_STATUS);
987
988         if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
989                    SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
990                 return false;
991
992         return true;
993 }
994
995 static int gmc_v6_0_wait_for_idle(void *handle)
996 {
997         unsigned i;
998         u32 tmp;
999         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1000
1001         for (i = 0; i < adev->usec_timeout; i++) {
1002                 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1003                                                SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1004                                                SRBM_STATUS__MCC_BUSY_MASK |
1005                                                SRBM_STATUS__MCD_BUSY_MASK |
1006                                                SRBM_STATUS__VMC_BUSY_MASK);
1007                 if (!tmp)
1008                         return 0;
1009                 udelay(1);
1010         }
1011         return -ETIMEDOUT;
1012
1013 }
1014
1015 static int gmc_v6_0_soft_reset(void *handle)
1016 {
1017         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1018         struct amdgpu_mode_mc_save save;
1019         u32 srbm_soft_reset = 0;
1020         u32 tmp = RREG32(mmSRBM_STATUS);
1021
1022         if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1023                 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1024                                                 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1025
1026         if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1027                    SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1028                 if (!(adev->flags & AMD_IS_APU))
1029                         srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1030                                                         SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1031         }
1032
1033         if (srbm_soft_reset) {
1034                 gmc_v6_0_mc_stop(adev, &save);
1035                 if (gmc_v6_0_wait_for_idle(adev)) {
1036                         dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1037                 }
1038
1039
1040                 tmp = RREG32(mmSRBM_SOFT_RESET);
1041                 tmp |= srbm_soft_reset;
1042                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1043                 WREG32(mmSRBM_SOFT_RESET, tmp);
1044                 tmp = RREG32(mmSRBM_SOFT_RESET);
1045
1046                 udelay(50);
1047
1048                 tmp &= ~srbm_soft_reset;
1049                 WREG32(mmSRBM_SOFT_RESET, tmp);
1050                 tmp = RREG32(mmSRBM_SOFT_RESET);
1051
1052                 udelay(50);
1053
1054                 gmc_v6_0_mc_resume(adev, &save);
1055                 udelay(50);
1056         }
1057
1058         return 0;
1059 }
1060
1061 static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1062                                              struct amdgpu_irq_src *src,
1063                                              unsigned type,
1064                                              enum amdgpu_interrupt_state state)
1065 {
1066         u32 tmp;
1067         u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1068                     VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1069                     VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1070                     VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1071                     VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1072                     VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1073
1074         switch (state) {
1075         case AMDGPU_IRQ_STATE_DISABLE:
1076                 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1077                 tmp &= ~bits;
1078                 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1079                 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1080                 tmp &= ~bits;
1081                 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1082                 break;
1083         case AMDGPU_IRQ_STATE_ENABLE:
1084                 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1085                 tmp |= bits;
1086                 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1087                 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1088                 tmp |= bits;
1089                 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1090                 break;
1091         default:
1092                 break;
1093         }
1094
1095         return 0;
1096 }
1097
1098 static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
1099                                       struct amdgpu_irq_src *source,
1100                                       struct amdgpu_iv_entry *entry)
1101 {
1102         u32 addr, status;
1103
1104         addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1105         status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1106         WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1107
1108         if (!addr && !status)
1109                 return 0;
1110
1111         if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1112                 gmc_v6_0_set_fault_enable_default(adev, false);
1113
1114         if (printk_ratelimit()) {
1115                 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1116                         entry->src_id, entry->src_data[0]);
1117                 dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1118                         addr);
1119                 dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1120                         status);
1121                 gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
1122         }
1123
1124         return 0;
1125 }
1126
1127 static int gmc_v6_0_set_clockgating_state(void *handle,
1128                                           enum amd_clockgating_state state)
1129 {
1130         return 0;
1131 }
1132
1133 static int gmc_v6_0_set_powergating_state(void *handle,
1134                                           enum amd_powergating_state state)
1135 {
1136         return 0;
1137 }
1138
1139 static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
1140         .name = "gmc_v6_0",
1141         .early_init = gmc_v6_0_early_init,
1142         .late_init = gmc_v6_0_late_init,
1143         .sw_init = gmc_v6_0_sw_init,
1144         .sw_fini = gmc_v6_0_sw_fini,
1145         .hw_init = gmc_v6_0_hw_init,
1146         .hw_fini = gmc_v6_0_hw_fini,
1147         .suspend = gmc_v6_0_suspend,
1148         .resume = gmc_v6_0_resume,
1149         .is_idle = gmc_v6_0_is_idle,
1150         .wait_for_idle = gmc_v6_0_wait_for_idle,
1151         .soft_reset = gmc_v6_0_soft_reset,
1152         .set_clockgating_state = gmc_v6_0_set_clockgating_state,
1153         .set_powergating_state = gmc_v6_0_set_powergating_state,
1154 };
1155
1156 static const struct amdgpu_gart_funcs gmc_v6_0_gart_funcs = {
1157         .flush_gpu_tlb = gmc_v6_0_gart_flush_gpu_tlb,
1158         .set_pte_pde = gmc_v6_0_gart_set_pte_pde,
1159         .set_prt = gmc_v6_0_set_prt,
1160         .get_vm_pte_flags = gmc_v6_0_get_vm_pte_flags
1161 };
1162
1163 static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = {
1164         .set = gmc_v6_0_vm_fault_interrupt_state,
1165         .process = gmc_v6_0_process_interrupt,
1166 };
1167
1168 static void gmc_v6_0_set_gart_funcs(struct amdgpu_device *adev)
1169 {
1170         if (adev->gart.gart_funcs == NULL)
1171                 adev->gart.gart_funcs = &gmc_v6_0_gart_funcs;
1172 }
1173
1174 static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1175 {
1176         adev->mc.vm_fault.num_types = 1;
1177         adev->mc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
1178 }
1179
1180 const struct amdgpu_ip_block_version gmc_v6_0_ip_block =
1181 {
1182         .type = AMD_IP_BLOCK_TYPE_GMC,
1183         .major = 6,
1184         .minor = 0,
1185         .rev = 0,
1186         .funcs = &gmc_v6_0_ip_funcs,
1187 };