Merge remote-tracking branch 'airlied/drm-next' into drm-misc-next
[sfrench/cifs-2.6.git] / drivers / gpu / drm / amd / amdgpu / gfx_v7_0.c
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <drm/drmP.h>
25 #include "amdgpu.h"
26 #include "amdgpu_ih.h"
27 #include "amdgpu_gfx.h"
28 #include "cikd.h"
29 #include "cik.h"
30 #include "atom.h"
31 #include "amdgpu_ucode.h"
32 #include "clearstate_ci.h"
33
34 #include "dce/dce_8_0_d.h"
35 #include "dce/dce_8_0_sh_mask.h"
36
37 #include "bif/bif_4_1_d.h"
38 #include "bif/bif_4_1_sh_mask.h"
39
40 #include "gca/gfx_7_0_d.h"
41 #include "gca/gfx_7_2_enum.h"
42 #include "gca/gfx_7_2_sh_mask.h"
43
44 #include "gmc/gmc_7_0_d.h"
45 #include "gmc/gmc_7_0_sh_mask.h"
46
47 #include "oss/oss_2_0_d.h"
48 #include "oss/oss_2_0_sh_mask.h"
49
50 #define GFX7_NUM_GFX_RINGS     1
51 #define GFX7_NUM_COMPUTE_RINGS 8
52
53 static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev);
54 static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev);
55 static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev);
56
57 MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
58 MODULE_FIRMWARE("radeon/bonaire_me.bin");
59 MODULE_FIRMWARE("radeon/bonaire_ce.bin");
60 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
61 MODULE_FIRMWARE("radeon/bonaire_mec.bin");
62
63 MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
64 MODULE_FIRMWARE("radeon/hawaii_me.bin");
65 MODULE_FIRMWARE("radeon/hawaii_ce.bin");
66 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
67 MODULE_FIRMWARE("radeon/hawaii_mec.bin");
68
69 MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
70 MODULE_FIRMWARE("radeon/kaveri_me.bin");
71 MODULE_FIRMWARE("radeon/kaveri_ce.bin");
72 MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
73 MODULE_FIRMWARE("radeon/kaveri_mec.bin");
74 MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
75
76 MODULE_FIRMWARE("radeon/kabini_pfp.bin");
77 MODULE_FIRMWARE("radeon/kabini_me.bin");
78 MODULE_FIRMWARE("radeon/kabini_ce.bin");
79 MODULE_FIRMWARE("radeon/kabini_rlc.bin");
80 MODULE_FIRMWARE("radeon/kabini_mec.bin");
81
82 MODULE_FIRMWARE("radeon/mullins_pfp.bin");
83 MODULE_FIRMWARE("radeon/mullins_me.bin");
84 MODULE_FIRMWARE("radeon/mullins_ce.bin");
85 MODULE_FIRMWARE("radeon/mullins_rlc.bin");
86 MODULE_FIRMWARE("radeon/mullins_mec.bin");
87
88 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
89 {
90         {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
91         {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
92         {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
93         {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
94         {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
95         {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
96         {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
97         {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
98         {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
99         {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
100         {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
101         {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
102         {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
103         {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
104         {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
105         {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
106 };
107
108 static const u32 spectre_rlc_save_restore_register_list[] =
109 {
110         (0x0e00 << 16) | (0xc12c >> 2),
111         0x00000000,
112         (0x0e00 << 16) | (0xc140 >> 2),
113         0x00000000,
114         (0x0e00 << 16) | (0xc150 >> 2),
115         0x00000000,
116         (0x0e00 << 16) | (0xc15c >> 2),
117         0x00000000,
118         (0x0e00 << 16) | (0xc168 >> 2),
119         0x00000000,
120         (0x0e00 << 16) | (0xc170 >> 2),
121         0x00000000,
122         (0x0e00 << 16) | (0xc178 >> 2),
123         0x00000000,
124         (0x0e00 << 16) | (0xc204 >> 2),
125         0x00000000,
126         (0x0e00 << 16) | (0xc2b4 >> 2),
127         0x00000000,
128         (0x0e00 << 16) | (0xc2b8 >> 2),
129         0x00000000,
130         (0x0e00 << 16) | (0xc2bc >> 2),
131         0x00000000,
132         (0x0e00 << 16) | (0xc2c0 >> 2),
133         0x00000000,
134         (0x0e00 << 16) | (0x8228 >> 2),
135         0x00000000,
136         (0x0e00 << 16) | (0x829c >> 2),
137         0x00000000,
138         (0x0e00 << 16) | (0x869c >> 2),
139         0x00000000,
140         (0x0600 << 16) | (0x98f4 >> 2),
141         0x00000000,
142         (0x0e00 << 16) | (0x98f8 >> 2),
143         0x00000000,
144         (0x0e00 << 16) | (0x9900 >> 2),
145         0x00000000,
146         (0x0e00 << 16) | (0xc260 >> 2),
147         0x00000000,
148         (0x0e00 << 16) | (0x90e8 >> 2),
149         0x00000000,
150         (0x0e00 << 16) | (0x3c000 >> 2),
151         0x00000000,
152         (0x0e00 << 16) | (0x3c00c >> 2),
153         0x00000000,
154         (0x0e00 << 16) | (0x8c1c >> 2),
155         0x00000000,
156         (0x0e00 << 16) | (0x9700 >> 2),
157         0x00000000,
158         (0x0e00 << 16) | (0xcd20 >> 2),
159         0x00000000,
160         (0x4e00 << 16) | (0xcd20 >> 2),
161         0x00000000,
162         (0x5e00 << 16) | (0xcd20 >> 2),
163         0x00000000,
164         (0x6e00 << 16) | (0xcd20 >> 2),
165         0x00000000,
166         (0x7e00 << 16) | (0xcd20 >> 2),
167         0x00000000,
168         (0x8e00 << 16) | (0xcd20 >> 2),
169         0x00000000,
170         (0x9e00 << 16) | (0xcd20 >> 2),
171         0x00000000,
172         (0xae00 << 16) | (0xcd20 >> 2),
173         0x00000000,
174         (0xbe00 << 16) | (0xcd20 >> 2),
175         0x00000000,
176         (0x0e00 << 16) | (0x89bc >> 2),
177         0x00000000,
178         (0x0e00 << 16) | (0x8900 >> 2),
179         0x00000000,
180         0x3,
181         (0x0e00 << 16) | (0xc130 >> 2),
182         0x00000000,
183         (0x0e00 << 16) | (0xc134 >> 2),
184         0x00000000,
185         (0x0e00 << 16) | (0xc1fc >> 2),
186         0x00000000,
187         (0x0e00 << 16) | (0xc208 >> 2),
188         0x00000000,
189         (0x0e00 << 16) | (0xc264 >> 2),
190         0x00000000,
191         (0x0e00 << 16) | (0xc268 >> 2),
192         0x00000000,
193         (0x0e00 << 16) | (0xc26c >> 2),
194         0x00000000,
195         (0x0e00 << 16) | (0xc270 >> 2),
196         0x00000000,
197         (0x0e00 << 16) | (0xc274 >> 2),
198         0x00000000,
199         (0x0e00 << 16) | (0xc278 >> 2),
200         0x00000000,
201         (0x0e00 << 16) | (0xc27c >> 2),
202         0x00000000,
203         (0x0e00 << 16) | (0xc280 >> 2),
204         0x00000000,
205         (0x0e00 << 16) | (0xc284 >> 2),
206         0x00000000,
207         (0x0e00 << 16) | (0xc288 >> 2),
208         0x00000000,
209         (0x0e00 << 16) | (0xc28c >> 2),
210         0x00000000,
211         (0x0e00 << 16) | (0xc290 >> 2),
212         0x00000000,
213         (0x0e00 << 16) | (0xc294 >> 2),
214         0x00000000,
215         (0x0e00 << 16) | (0xc298 >> 2),
216         0x00000000,
217         (0x0e00 << 16) | (0xc29c >> 2),
218         0x00000000,
219         (0x0e00 << 16) | (0xc2a0 >> 2),
220         0x00000000,
221         (0x0e00 << 16) | (0xc2a4 >> 2),
222         0x00000000,
223         (0x0e00 << 16) | (0xc2a8 >> 2),
224         0x00000000,
225         (0x0e00 << 16) | (0xc2ac  >> 2),
226         0x00000000,
227         (0x0e00 << 16) | (0xc2b0 >> 2),
228         0x00000000,
229         (0x0e00 << 16) | (0x301d0 >> 2),
230         0x00000000,
231         (0x0e00 << 16) | (0x30238 >> 2),
232         0x00000000,
233         (0x0e00 << 16) | (0x30250 >> 2),
234         0x00000000,
235         (0x0e00 << 16) | (0x30254 >> 2),
236         0x00000000,
237         (0x0e00 << 16) | (0x30258 >> 2),
238         0x00000000,
239         (0x0e00 << 16) | (0x3025c >> 2),
240         0x00000000,
241         (0x4e00 << 16) | (0xc900 >> 2),
242         0x00000000,
243         (0x5e00 << 16) | (0xc900 >> 2),
244         0x00000000,
245         (0x6e00 << 16) | (0xc900 >> 2),
246         0x00000000,
247         (0x7e00 << 16) | (0xc900 >> 2),
248         0x00000000,
249         (0x8e00 << 16) | (0xc900 >> 2),
250         0x00000000,
251         (0x9e00 << 16) | (0xc900 >> 2),
252         0x00000000,
253         (0xae00 << 16) | (0xc900 >> 2),
254         0x00000000,
255         (0xbe00 << 16) | (0xc900 >> 2),
256         0x00000000,
257         (0x4e00 << 16) | (0xc904 >> 2),
258         0x00000000,
259         (0x5e00 << 16) | (0xc904 >> 2),
260         0x00000000,
261         (0x6e00 << 16) | (0xc904 >> 2),
262         0x00000000,
263         (0x7e00 << 16) | (0xc904 >> 2),
264         0x00000000,
265         (0x8e00 << 16) | (0xc904 >> 2),
266         0x00000000,
267         (0x9e00 << 16) | (0xc904 >> 2),
268         0x00000000,
269         (0xae00 << 16) | (0xc904 >> 2),
270         0x00000000,
271         (0xbe00 << 16) | (0xc904 >> 2),
272         0x00000000,
273         (0x4e00 << 16) | (0xc908 >> 2),
274         0x00000000,
275         (0x5e00 << 16) | (0xc908 >> 2),
276         0x00000000,
277         (0x6e00 << 16) | (0xc908 >> 2),
278         0x00000000,
279         (0x7e00 << 16) | (0xc908 >> 2),
280         0x00000000,
281         (0x8e00 << 16) | (0xc908 >> 2),
282         0x00000000,
283         (0x9e00 << 16) | (0xc908 >> 2),
284         0x00000000,
285         (0xae00 << 16) | (0xc908 >> 2),
286         0x00000000,
287         (0xbe00 << 16) | (0xc908 >> 2),
288         0x00000000,
289         (0x4e00 << 16) | (0xc90c >> 2),
290         0x00000000,
291         (0x5e00 << 16) | (0xc90c >> 2),
292         0x00000000,
293         (0x6e00 << 16) | (0xc90c >> 2),
294         0x00000000,
295         (0x7e00 << 16) | (0xc90c >> 2),
296         0x00000000,
297         (0x8e00 << 16) | (0xc90c >> 2),
298         0x00000000,
299         (0x9e00 << 16) | (0xc90c >> 2),
300         0x00000000,
301         (0xae00 << 16) | (0xc90c >> 2),
302         0x00000000,
303         (0xbe00 << 16) | (0xc90c >> 2),
304         0x00000000,
305         (0x4e00 << 16) | (0xc910 >> 2),
306         0x00000000,
307         (0x5e00 << 16) | (0xc910 >> 2),
308         0x00000000,
309         (0x6e00 << 16) | (0xc910 >> 2),
310         0x00000000,
311         (0x7e00 << 16) | (0xc910 >> 2),
312         0x00000000,
313         (0x8e00 << 16) | (0xc910 >> 2),
314         0x00000000,
315         (0x9e00 << 16) | (0xc910 >> 2),
316         0x00000000,
317         (0xae00 << 16) | (0xc910 >> 2),
318         0x00000000,
319         (0xbe00 << 16) | (0xc910 >> 2),
320         0x00000000,
321         (0x0e00 << 16) | (0xc99c >> 2),
322         0x00000000,
323         (0x0e00 << 16) | (0x9834 >> 2),
324         0x00000000,
325         (0x0000 << 16) | (0x30f00 >> 2),
326         0x00000000,
327         (0x0001 << 16) | (0x30f00 >> 2),
328         0x00000000,
329         (0x0000 << 16) | (0x30f04 >> 2),
330         0x00000000,
331         (0x0001 << 16) | (0x30f04 >> 2),
332         0x00000000,
333         (0x0000 << 16) | (0x30f08 >> 2),
334         0x00000000,
335         (0x0001 << 16) | (0x30f08 >> 2),
336         0x00000000,
337         (0x0000 << 16) | (0x30f0c >> 2),
338         0x00000000,
339         (0x0001 << 16) | (0x30f0c >> 2),
340         0x00000000,
341         (0x0600 << 16) | (0x9b7c >> 2),
342         0x00000000,
343         (0x0e00 << 16) | (0x8a14 >> 2),
344         0x00000000,
345         (0x0e00 << 16) | (0x8a18 >> 2),
346         0x00000000,
347         (0x0600 << 16) | (0x30a00 >> 2),
348         0x00000000,
349         (0x0e00 << 16) | (0x8bf0 >> 2),
350         0x00000000,
351         (0x0e00 << 16) | (0x8bcc >> 2),
352         0x00000000,
353         (0x0e00 << 16) | (0x8b24 >> 2),
354         0x00000000,
355         (0x0e00 << 16) | (0x30a04 >> 2),
356         0x00000000,
357         (0x0600 << 16) | (0x30a10 >> 2),
358         0x00000000,
359         (0x0600 << 16) | (0x30a14 >> 2),
360         0x00000000,
361         (0x0600 << 16) | (0x30a18 >> 2),
362         0x00000000,
363         (0x0600 << 16) | (0x30a2c >> 2),
364         0x00000000,
365         (0x0e00 << 16) | (0xc700 >> 2),
366         0x00000000,
367         (0x0e00 << 16) | (0xc704 >> 2),
368         0x00000000,
369         (0x0e00 << 16) | (0xc708 >> 2),
370         0x00000000,
371         (0x0e00 << 16) | (0xc768 >> 2),
372         0x00000000,
373         (0x0400 << 16) | (0xc770 >> 2),
374         0x00000000,
375         (0x0400 << 16) | (0xc774 >> 2),
376         0x00000000,
377         (0x0400 << 16) | (0xc778 >> 2),
378         0x00000000,
379         (0x0400 << 16) | (0xc77c >> 2),
380         0x00000000,
381         (0x0400 << 16) | (0xc780 >> 2),
382         0x00000000,
383         (0x0400 << 16) | (0xc784 >> 2),
384         0x00000000,
385         (0x0400 << 16) | (0xc788 >> 2),
386         0x00000000,
387         (0x0400 << 16) | (0xc78c >> 2),
388         0x00000000,
389         (0x0400 << 16) | (0xc798 >> 2),
390         0x00000000,
391         (0x0400 << 16) | (0xc79c >> 2),
392         0x00000000,
393         (0x0400 << 16) | (0xc7a0 >> 2),
394         0x00000000,
395         (0x0400 << 16) | (0xc7a4 >> 2),
396         0x00000000,
397         (0x0400 << 16) | (0xc7a8 >> 2),
398         0x00000000,
399         (0x0400 << 16) | (0xc7ac >> 2),
400         0x00000000,
401         (0x0400 << 16) | (0xc7b0 >> 2),
402         0x00000000,
403         (0x0400 << 16) | (0xc7b4 >> 2),
404         0x00000000,
405         (0x0e00 << 16) | (0x9100 >> 2),
406         0x00000000,
407         (0x0e00 << 16) | (0x3c010 >> 2),
408         0x00000000,
409         (0x0e00 << 16) | (0x92a8 >> 2),
410         0x00000000,
411         (0x0e00 << 16) | (0x92ac >> 2),
412         0x00000000,
413         (0x0e00 << 16) | (0x92b4 >> 2),
414         0x00000000,
415         (0x0e00 << 16) | (0x92b8 >> 2),
416         0x00000000,
417         (0x0e00 << 16) | (0x92bc >> 2),
418         0x00000000,
419         (0x0e00 << 16) | (0x92c0 >> 2),
420         0x00000000,
421         (0x0e00 << 16) | (0x92c4 >> 2),
422         0x00000000,
423         (0x0e00 << 16) | (0x92c8 >> 2),
424         0x00000000,
425         (0x0e00 << 16) | (0x92cc >> 2),
426         0x00000000,
427         (0x0e00 << 16) | (0x92d0 >> 2),
428         0x00000000,
429         (0x0e00 << 16) | (0x8c00 >> 2),
430         0x00000000,
431         (0x0e00 << 16) | (0x8c04 >> 2),
432         0x00000000,
433         (0x0e00 << 16) | (0x8c20 >> 2),
434         0x00000000,
435         (0x0e00 << 16) | (0x8c38 >> 2),
436         0x00000000,
437         (0x0e00 << 16) | (0x8c3c >> 2),
438         0x00000000,
439         (0x0e00 << 16) | (0xae00 >> 2),
440         0x00000000,
441         (0x0e00 << 16) | (0x9604 >> 2),
442         0x00000000,
443         (0x0e00 << 16) | (0xac08 >> 2),
444         0x00000000,
445         (0x0e00 << 16) | (0xac0c >> 2),
446         0x00000000,
447         (0x0e00 << 16) | (0xac10 >> 2),
448         0x00000000,
449         (0x0e00 << 16) | (0xac14 >> 2),
450         0x00000000,
451         (0x0e00 << 16) | (0xac58 >> 2),
452         0x00000000,
453         (0x0e00 << 16) | (0xac68 >> 2),
454         0x00000000,
455         (0x0e00 << 16) | (0xac6c >> 2),
456         0x00000000,
457         (0x0e00 << 16) | (0xac70 >> 2),
458         0x00000000,
459         (0x0e00 << 16) | (0xac74 >> 2),
460         0x00000000,
461         (0x0e00 << 16) | (0xac78 >> 2),
462         0x00000000,
463         (0x0e00 << 16) | (0xac7c >> 2),
464         0x00000000,
465         (0x0e00 << 16) | (0xac80 >> 2),
466         0x00000000,
467         (0x0e00 << 16) | (0xac84 >> 2),
468         0x00000000,
469         (0x0e00 << 16) | (0xac88 >> 2),
470         0x00000000,
471         (0x0e00 << 16) | (0xac8c >> 2),
472         0x00000000,
473         (0x0e00 << 16) | (0x970c >> 2),
474         0x00000000,
475         (0x0e00 << 16) | (0x9714 >> 2),
476         0x00000000,
477         (0x0e00 << 16) | (0x9718 >> 2),
478         0x00000000,
479         (0x0e00 << 16) | (0x971c >> 2),
480         0x00000000,
481         (0x0e00 << 16) | (0x31068 >> 2),
482         0x00000000,
483         (0x4e00 << 16) | (0x31068 >> 2),
484         0x00000000,
485         (0x5e00 << 16) | (0x31068 >> 2),
486         0x00000000,
487         (0x6e00 << 16) | (0x31068 >> 2),
488         0x00000000,
489         (0x7e00 << 16) | (0x31068 >> 2),
490         0x00000000,
491         (0x8e00 << 16) | (0x31068 >> 2),
492         0x00000000,
493         (0x9e00 << 16) | (0x31068 >> 2),
494         0x00000000,
495         (0xae00 << 16) | (0x31068 >> 2),
496         0x00000000,
497         (0xbe00 << 16) | (0x31068 >> 2),
498         0x00000000,
499         (0x0e00 << 16) | (0xcd10 >> 2),
500         0x00000000,
501         (0x0e00 << 16) | (0xcd14 >> 2),
502         0x00000000,
503         (0x0e00 << 16) | (0x88b0 >> 2),
504         0x00000000,
505         (0x0e00 << 16) | (0x88b4 >> 2),
506         0x00000000,
507         (0x0e00 << 16) | (0x88b8 >> 2),
508         0x00000000,
509         (0x0e00 << 16) | (0x88bc >> 2),
510         0x00000000,
511         (0x0400 << 16) | (0x89c0 >> 2),
512         0x00000000,
513         (0x0e00 << 16) | (0x88c4 >> 2),
514         0x00000000,
515         (0x0e00 << 16) | (0x88c8 >> 2),
516         0x00000000,
517         (0x0e00 << 16) | (0x88d0 >> 2),
518         0x00000000,
519         (0x0e00 << 16) | (0x88d4 >> 2),
520         0x00000000,
521         (0x0e00 << 16) | (0x88d8 >> 2),
522         0x00000000,
523         (0x0e00 << 16) | (0x8980 >> 2),
524         0x00000000,
525         (0x0e00 << 16) | (0x30938 >> 2),
526         0x00000000,
527         (0x0e00 << 16) | (0x3093c >> 2),
528         0x00000000,
529         (0x0e00 << 16) | (0x30940 >> 2),
530         0x00000000,
531         (0x0e00 << 16) | (0x89a0 >> 2),
532         0x00000000,
533         (0x0e00 << 16) | (0x30900 >> 2),
534         0x00000000,
535         (0x0e00 << 16) | (0x30904 >> 2),
536         0x00000000,
537         (0x0e00 << 16) | (0x89b4 >> 2),
538         0x00000000,
539         (0x0e00 << 16) | (0x3c210 >> 2),
540         0x00000000,
541         (0x0e00 << 16) | (0x3c214 >> 2),
542         0x00000000,
543         (0x0e00 << 16) | (0x3c218 >> 2),
544         0x00000000,
545         (0x0e00 << 16) | (0x8904 >> 2),
546         0x00000000,
547         0x5,
548         (0x0e00 << 16) | (0x8c28 >> 2),
549         (0x0e00 << 16) | (0x8c2c >> 2),
550         (0x0e00 << 16) | (0x8c30 >> 2),
551         (0x0e00 << 16) | (0x8c34 >> 2),
552         (0x0e00 << 16) | (0x9600 >> 2),
553 };
554
555 static const u32 kalindi_rlc_save_restore_register_list[] =
556 {
557         (0x0e00 << 16) | (0xc12c >> 2),
558         0x00000000,
559         (0x0e00 << 16) | (0xc140 >> 2),
560         0x00000000,
561         (0x0e00 << 16) | (0xc150 >> 2),
562         0x00000000,
563         (0x0e00 << 16) | (0xc15c >> 2),
564         0x00000000,
565         (0x0e00 << 16) | (0xc168 >> 2),
566         0x00000000,
567         (0x0e00 << 16) | (0xc170 >> 2),
568         0x00000000,
569         (0x0e00 << 16) | (0xc204 >> 2),
570         0x00000000,
571         (0x0e00 << 16) | (0xc2b4 >> 2),
572         0x00000000,
573         (0x0e00 << 16) | (0xc2b8 >> 2),
574         0x00000000,
575         (0x0e00 << 16) | (0xc2bc >> 2),
576         0x00000000,
577         (0x0e00 << 16) | (0xc2c0 >> 2),
578         0x00000000,
579         (0x0e00 << 16) | (0x8228 >> 2),
580         0x00000000,
581         (0x0e00 << 16) | (0x829c >> 2),
582         0x00000000,
583         (0x0e00 << 16) | (0x869c >> 2),
584         0x00000000,
585         (0x0600 << 16) | (0x98f4 >> 2),
586         0x00000000,
587         (0x0e00 << 16) | (0x98f8 >> 2),
588         0x00000000,
589         (0x0e00 << 16) | (0x9900 >> 2),
590         0x00000000,
591         (0x0e00 << 16) | (0xc260 >> 2),
592         0x00000000,
593         (0x0e00 << 16) | (0x90e8 >> 2),
594         0x00000000,
595         (0x0e00 << 16) | (0x3c000 >> 2),
596         0x00000000,
597         (0x0e00 << 16) | (0x3c00c >> 2),
598         0x00000000,
599         (0x0e00 << 16) | (0x8c1c >> 2),
600         0x00000000,
601         (0x0e00 << 16) | (0x9700 >> 2),
602         0x00000000,
603         (0x0e00 << 16) | (0xcd20 >> 2),
604         0x00000000,
605         (0x4e00 << 16) | (0xcd20 >> 2),
606         0x00000000,
607         (0x5e00 << 16) | (0xcd20 >> 2),
608         0x00000000,
609         (0x6e00 << 16) | (0xcd20 >> 2),
610         0x00000000,
611         (0x7e00 << 16) | (0xcd20 >> 2),
612         0x00000000,
613         (0x0e00 << 16) | (0x89bc >> 2),
614         0x00000000,
615         (0x0e00 << 16) | (0x8900 >> 2),
616         0x00000000,
617         0x3,
618         (0x0e00 << 16) | (0xc130 >> 2),
619         0x00000000,
620         (0x0e00 << 16) | (0xc134 >> 2),
621         0x00000000,
622         (0x0e00 << 16) | (0xc1fc >> 2),
623         0x00000000,
624         (0x0e00 << 16) | (0xc208 >> 2),
625         0x00000000,
626         (0x0e00 << 16) | (0xc264 >> 2),
627         0x00000000,
628         (0x0e00 << 16) | (0xc268 >> 2),
629         0x00000000,
630         (0x0e00 << 16) | (0xc26c >> 2),
631         0x00000000,
632         (0x0e00 << 16) | (0xc270 >> 2),
633         0x00000000,
634         (0x0e00 << 16) | (0xc274 >> 2),
635         0x00000000,
636         (0x0e00 << 16) | (0xc28c >> 2),
637         0x00000000,
638         (0x0e00 << 16) | (0xc290 >> 2),
639         0x00000000,
640         (0x0e00 << 16) | (0xc294 >> 2),
641         0x00000000,
642         (0x0e00 << 16) | (0xc298 >> 2),
643         0x00000000,
644         (0x0e00 << 16) | (0xc2a0 >> 2),
645         0x00000000,
646         (0x0e00 << 16) | (0xc2a4 >> 2),
647         0x00000000,
648         (0x0e00 << 16) | (0xc2a8 >> 2),
649         0x00000000,
650         (0x0e00 << 16) | (0xc2ac >> 2),
651         0x00000000,
652         (0x0e00 << 16) | (0x301d0 >> 2),
653         0x00000000,
654         (0x0e00 << 16) | (0x30238 >> 2),
655         0x00000000,
656         (0x0e00 << 16) | (0x30250 >> 2),
657         0x00000000,
658         (0x0e00 << 16) | (0x30254 >> 2),
659         0x00000000,
660         (0x0e00 << 16) | (0x30258 >> 2),
661         0x00000000,
662         (0x0e00 << 16) | (0x3025c >> 2),
663         0x00000000,
664         (0x4e00 << 16) | (0xc900 >> 2),
665         0x00000000,
666         (0x5e00 << 16) | (0xc900 >> 2),
667         0x00000000,
668         (0x6e00 << 16) | (0xc900 >> 2),
669         0x00000000,
670         (0x7e00 << 16) | (0xc900 >> 2),
671         0x00000000,
672         (0x4e00 << 16) | (0xc904 >> 2),
673         0x00000000,
674         (0x5e00 << 16) | (0xc904 >> 2),
675         0x00000000,
676         (0x6e00 << 16) | (0xc904 >> 2),
677         0x00000000,
678         (0x7e00 << 16) | (0xc904 >> 2),
679         0x00000000,
680         (0x4e00 << 16) | (0xc908 >> 2),
681         0x00000000,
682         (0x5e00 << 16) | (0xc908 >> 2),
683         0x00000000,
684         (0x6e00 << 16) | (0xc908 >> 2),
685         0x00000000,
686         (0x7e00 << 16) | (0xc908 >> 2),
687         0x00000000,
688         (0x4e00 << 16) | (0xc90c >> 2),
689         0x00000000,
690         (0x5e00 << 16) | (0xc90c >> 2),
691         0x00000000,
692         (0x6e00 << 16) | (0xc90c >> 2),
693         0x00000000,
694         (0x7e00 << 16) | (0xc90c >> 2),
695         0x00000000,
696         (0x4e00 << 16) | (0xc910 >> 2),
697         0x00000000,
698         (0x5e00 << 16) | (0xc910 >> 2),
699         0x00000000,
700         (0x6e00 << 16) | (0xc910 >> 2),
701         0x00000000,
702         (0x7e00 << 16) | (0xc910 >> 2),
703         0x00000000,
704         (0x0e00 << 16) | (0xc99c >> 2),
705         0x00000000,
706         (0x0e00 << 16) | (0x9834 >> 2),
707         0x00000000,
708         (0x0000 << 16) | (0x30f00 >> 2),
709         0x00000000,
710         (0x0000 << 16) | (0x30f04 >> 2),
711         0x00000000,
712         (0x0000 << 16) | (0x30f08 >> 2),
713         0x00000000,
714         (0x0000 << 16) | (0x30f0c >> 2),
715         0x00000000,
716         (0x0600 << 16) | (0x9b7c >> 2),
717         0x00000000,
718         (0x0e00 << 16) | (0x8a14 >> 2),
719         0x00000000,
720         (0x0e00 << 16) | (0x8a18 >> 2),
721         0x00000000,
722         (0x0600 << 16) | (0x30a00 >> 2),
723         0x00000000,
724         (0x0e00 << 16) | (0x8bf0 >> 2),
725         0x00000000,
726         (0x0e00 << 16) | (0x8bcc >> 2),
727         0x00000000,
728         (0x0e00 << 16) | (0x8b24 >> 2),
729         0x00000000,
730         (0x0e00 << 16) | (0x30a04 >> 2),
731         0x00000000,
732         (0x0600 << 16) | (0x30a10 >> 2),
733         0x00000000,
734         (0x0600 << 16) | (0x30a14 >> 2),
735         0x00000000,
736         (0x0600 << 16) | (0x30a18 >> 2),
737         0x00000000,
738         (0x0600 << 16) | (0x30a2c >> 2),
739         0x00000000,
740         (0x0e00 << 16) | (0xc700 >> 2),
741         0x00000000,
742         (0x0e00 << 16) | (0xc704 >> 2),
743         0x00000000,
744         (0x0e00 << 16) | (0xc708 >> 2),
745         0x00000000,
746         (0x0e00 << 16) | (0xc768 >> 2),
747         0x00000000,
748         (0x0400 << 16) | (0xc770 >> 2),
749         0x00000000,
750         (0x0400 << 16) | (0xc774 >> 2),
751         0x00000000,
752         (0x0400 << 16) | (0xc798 >> 2),
753         0x00000000,
754         (0x0400 << 16) | (0xc79c >> 2),
755         0x00000000,
756         (0x0e00 << 16) | (0x9100 >> 2),
757         0x00000000,
758         (0x0e00 << 16) | (0x3c010 >> 2),
759         0x00000000,
760         (0x0e00 << 16) | (0x8c00 >> 2),
761         0x00000000,
762         (0x0e00 << 16) | (0x8c04 >> 2),
763         0x00000000,
764         (0x0e00 << 16) | (0x8c20 >> 2),
765         0x00000000,
766         (0x0e00 << 16) | (0x8c38 >> 2),
767         0x00000000,
768         (0x0e00 << 16) | (0x8c3c >> 2),
769         0x00000000,
770         (0x0e00 << 16) | (0xae00 >> 2),
771         0x00000000,
772         (0x0e00 << 16) | (0x9604 >> 2),
773         0x00000000,
774         (0x0e00 << 16) | (0xac08 >> 2),
775         0x00000000,
776         (0x0e00 << 16) | (0xac0c >> 2),
777         0x00000000,
778         (0x0e00 << 16) | (0xac10 >> 2),
779         0x00000000,
780         (0x0e00 << 16) | (0xac14 >> 2),
781         0x00000000,
782         (0x0e00 << 16) | (0xac58 >> 2),
783         0x00000000,
784         (0x0e00 << 16) | (0xac68 >> 2),
785         0x00000000,
786         (0x0e00 << 16) | (0xac6c >> 2),
787         0x00000000,
788         (0x0e00 << 16) | (0xac70 >> 2),
789         0x00000000,
790         (0x0e00 << 16) | (0xac74 >> 2),
791         0x00000000,
792         (0x0e00 << 16) | (0xac78 >> 2),
793         0x00000000,
794         (0x0e00 << 16) | (0xac7c >> 2),
795         0x00000000,
796         (0x0e00 << 16) | (0xac80 >> 2),
797         0x00000000,
798         (0x0e00 << 16) | (0xac84 >> 2),
799         0x00000000,
800         (0x0e00 << 16) | (0xac88 >> 2),
801         0x00000000,
802         (0x0e00 << 16) | (0xac8c >> 2),
803         0x00000000,
804         (0x0e00 << 16) | (0x970c >> 2),
805         0x00000000,
806         (0x0e00 << 16) | (0x9714 >> 2),
807         0x00000000,
808         (0x0e00 << 16) | (0x9718 >> 2),
809         0x00000000,
810         (0x0e00 << 16) | (0x971c >> 2),
811         0x00000000,
812         (0x0e00 << 16) | (0x31068 >> 2),
813         0x00000000,
814         (0x4e00 << 16) | (0x31068 >> 2),
815         0x00000000,
816         (0x5e00 << 16) | (0x31068 >> 2),
817         0x00000000,
818         (0x6e00 << 16) | (0x31068 >> 2),
819         0x00000000,
820         (0x7e00 << 16) | (0x31068 >> 2),
821         0x00000000,
822         (0x0e00 << 16) | (0xcd10 >> 2),
823         0x00000000,
824         (0x0e00 << 16) | (0xcd14 >> 2),
825         0x00000000,
826         (0x0e00 << 16) | (0x88b0 >> 2),
827         0x00000000,
828         (0x0e00 << 16) | (0x88b4 >> 2),
829         0x00000000,
830         (0x0e00 << 16) | (0x88b8 >> 2),
831         0x00000000,
832         (0x0e00 << 16) | (0x88bc >> 2),
833         0x00000000,
834         (0x0400 << 16) | (0x89c0 >> 2),
835         0x00000000,
836         (0x0e00 << 16) | (0x88c4 >> 2),
837         0x00000000,
838         (0x0e00 << 16) | (0x88c8 >> 2),
839         0x00000000,
840         (0x0e00 << 16) | (0x88d0 >> 2),
841         0x00000000,
842         (0x0e00 << 16) | (0x88d4 >> 2),
843         0x00000000,
844         (0x0e00 << 16) | (0x88d8 >> 2),
845         0x00000000,
846         (0x0e00 << 16) | (0x8980 >> 2),
847         0x00000000,
848         (0x0e00 << 16) | (0x30938 >> 2),
849         0x00000000,
850         (0x0e00 << 16) | (0x3093c >> 2),
851         0x00000000,
852         (0x0e00 << 16) | (0x30940 >> 2),
853         0x00000000,
854         (0x0e00 << 16) | (0x89a0 >> 2),
855         0x00000000,
856         (0x0e00 << 16) | (0x30900 >> 2),
857         0x00000000,
858         (0x0e00 << 16) | (0x30904 >> 2),
859         0x00000000,
860         (0x0e00 << 16) | (0x89b4 >> 2),
861         0x00000000,
862         (0x0e00 << 16) | (0x3e1fc >> 2),
863         0x00000000,
864         (0x0e00 << 16) | (0x3c210 >> 2),
865         0x00000000,
866         (0x0e00 << 16) | (0x3c214 >> 2),
867         0x00000000,
868         (0x0e00 << 16) | (0x3c218 >> 2),
869         0x00000000,
870         (0x0e00 << 16) | (0x8904 >> 2),
871         0x00000000,
872         0x5,
873         (0x0e00 << 16) | (0x8c28 >> 2),
874         (0x0e00 << 16) | (0x8c2c >> 2),
875         (0x0e00 << 16) | (0x8c30 >> 2),
876         (0x0e00 << 16) | (0x8c34 >> 2),
877         (0x0e00 << 16) | (0x9600 >> 2),
878 };
879
880 static u32 gfx_v7_0_get_csb_size(struct amdgpu_device *adev);
881 static void gfx_v7_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer);
882 static void gfx_v7_0_init_cp_pg_table(struct amdgpu_device *adev);
883 static void gfx_v7_0_init_pg(struct amdgpu_device *adev);
884 static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev);
885
886 /*
887  * Core functions
888  */
889 /**
890  * gfx_v7_0_init_microcode - load ucode images from disk
891  *
892  * @adev: amdgpu_device pointer
893  *
894  * Use the firmware interface to load the ucode images into
895  * the driver (not loaded into hw).
896  * Returns 0 on success, error on failure.
897  */
898 static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
899 {
900         const char *chip_name;
901         char fw_name[30];
902         int err;
903
904         DRM_DEBUG("\n");
905
906         switch (adev->asic_type) {
907         case CHIP_BONAIRE:
908                 chip_name = "bonaire";
909                 break;
910         case CHIP_HAWAII:
911                 chip_name = "hawaii";
912                 break;
913         case CHIP_KAVERI:
914                 chip_name = "kaveri";
915                 break;
916         case CHIP_KABINI:
917                 chip_name = "kabini";
918                 break;
919         case CHIP_MULLINS:
920                 chip_name = "mullins";
921                 break;
922         default: BUG();
923         }
924
925         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
926         err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
927         if (err)
928                 goto out;
929         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
930         if (err)
931                 goto out;
932
933         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
934         err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
935         if (err)
936                 goto out;
937         err = amdgpu_ucode_validate(adev->gfx.me_fw);
938         if (err)
939                 goto out;
940
941         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
942         err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
943         if (err)
944                 goto out;
945         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
946         if (err)
947                 goto out;
948
949         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
950         err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
951         if (err)
952                 goto out;
953         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
954         if (err)
955                 goto out;
956
957         if (adev->asic_type == CHIP_KAVERI) {
958                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", chip_name);
959                 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
960                 if (err)
961                         goto out;
962                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
963                 if (err)
964                         goto out;
965         }
966
967         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
968         err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
969         if (err)
970                 goto out;
971         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
972
973 out:
974         if (err) {
975                 pr_err("gfx7: Failed to load firmware \"%s\"\n", fw_name);
976                 release_firmware(adev->gfx.pfp_fw);
977                 adev->gfx.pfp_fw = NULL;
978                 release_firmware(adev->gfx.me_fw);
979                 adev->gfx.me_fw = NULL;
980                 release_firmware(adev->gfx.ce_fw);
981                 adev->gfx.ce_fw = NULL;
982                 release_firmware(adev->gfx.mec_fw);
983                 adev->gfx.mec_fw = NULL;
984                 release_firmware(adev->gfx.mec2_fw);
985                 adev->gfx.mec2_fw = NULL;
986                 release_firmware(adev->gfx.rlc_fw);
987                 adev->gfx.rlc_fw = NULL;
988         }
989         return err;
990 }
991
992 static void gfx_v7_0_free_microcode(struct amdgpu_device *adev)
993 {
994         release_firmware(adev->gfx.pfp_fw);
995         adev->gfx.pfp_fw = NULL;
996         release_firmware(adev->gfx.me_fw);
997         adev->gfx.me_fw = NULL;
998         release_firmware(adev->gfx.ce_fw);
999         adev->gfx.ce_fw = NULL;
1000         release_firmware(adev->gfx.mec_fw);
1001         adev->gfx.mec_fw = NULL;
1002         release_firmware(adev->gfx.mec2_fw);
1003         adev->gfx.mec2_fw = NULL;
1004         release_firmware(adev->gfx.rlc_fw);
1005         adev->gfx.rlc_fw = NULL;
1006 }
1007
1008 /**
1009  * gfx_v7_0_tiling_mode_table_init - init the hw tiling table
1010  *
1011  * @adev: amdgpu_device pointer
1012  *
1013  * Starting with SI, the tiling setup is done globally in a
1014  * set of 32 tiling modes.  Rather than selecting each set of
1015  * parameters per surface as on older asics, we just select
1016  * which index in the tiling table we want to use, and the
1017  * surface uses those parameters (CIK).
1018  */
1019 static void gfx_v7_0_tiling_mode_table_init(struct amdgpu_device *adev)
1020 {
1021         const u32 num_tile_mode_states =
1022                         ARRAY_SIZE(adev->gfx.config.tile_mode_array);
1023         const u32 num_secondary_tile_mode_states =
1024                         ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
1025         u32 reg_offset, split_equal_to_row_size;
1026         uint32_t *tile, *macrotile;
1027
1028         tile = adev->gfx.config.tile_mode_array;
1029         macrotile = adev->gfx.config.macrotile_mode_array;
1030
1031         switch (adev->gfx.config.mem_row_size_in_kb) {
1032         case 1:
1033                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1034                 break;
1035         case 2:
1036         default:
1037                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1038                 break;
1039         case 4:
1040                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1041                 break;
1042         }
1043
1044         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1045                 tile[reg_offset] = 0;
1046         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1047                 macrotile[reg_offset] = 0;
1048
1049         switch (adev->asic_type) {
1050         case CHIP_BONAIRE:
1051                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1052                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1053                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1054                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1055                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1056                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1057                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1058                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1059                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1060                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1061                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1062                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1063                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1064                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1065                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1066                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1067                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1068                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1069                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1070                            TILE_SPLIT(split_equal_to_row_size));
1071                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1072                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1073                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1074                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1075                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1076                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1077                            TILE_SPLIT(split_equal_to_row_size));
1078                 tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1079                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1080                            PIPE_CONFIG(ADDR_SURF_P4_16x16));
1081                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1082                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1083                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1084                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1085                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1086                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1087                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1088                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1089                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1090                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1091                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1092                 tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1093                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1094                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1095                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1096                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1097                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1098                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1099                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1100                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1101                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1102                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1103                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1104                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1105                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1106                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1107                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1108                 tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1109                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1110                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1111                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1112                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1113                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1114                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1115                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1116                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1117                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1118                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1119                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1120                 tile[21] =  (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1121                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1122                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1123                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1124                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1125                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1126                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1127                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1128                 tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1129                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1130                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1131                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1132                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1133                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1134                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1135                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1136                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1137                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1138                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1139                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1140                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1141                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1142                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1143                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1144                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1145                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1146                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1147                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1148                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1149                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1150                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1151                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1152                 tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1153
1154                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1155                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1156                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1157                                 NUM_BANKS(ADDR_SURF_16_BANK));
1158                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1159                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1160                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1161                                 NUM_BANKS(ADDR_SURF_16_BANK));
1162                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1163                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1164                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1165                                 NUM_BANKS(ADDR_SURF_16_BANK));
1166                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1167                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1168                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1169                                 NUM_BANKS(ADDR_SURF_16_BANK));
1170                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1171                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1172                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1173                                 NUM_BANKS(ADDR_SURF_16_BANK));
1174                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1175                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1176                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1177                                 NUM_BANKS(ADDR_SURF_8_BANK));
1178                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1179                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1180                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1181                                 NUM_BANKS(ADDR_SURF_4_BANK));
1182                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1183                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1184                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1185                                 NUM_BANKS(ADDR_SURF_16_BANK));
1186                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1187                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1188                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1189                                 NUM_BANKS(ADDR_SURF_16_BANK));
1190                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1191                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1192                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1193                                 NUM_BANKS(ADDR_SURF_16_BANK));
1194                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1195                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1196                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1197                                 NUM_BANKS(ADDR_SURF_16_BANK));
1198                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1199                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1200                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1201                                 NUM_BANKS(ADDR_SURF_16_BANK));
1202                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1203                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1204                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1205                                 NUM_BANKS(ADDR_SURF_8_BANK));
1206                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1207                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1208                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1209                                 NUM_BANKS(ADDR_SURF_4_BANK));
1210
1211                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1212                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1213                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1214                         if (reg_offset != 7)
1215                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1216                 break;
1217         case CHIP_HAWAII:
1218                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1219                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1220                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1221                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1222                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1223                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1224                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1225                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1226                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1227                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1228                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1229                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1230                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1231                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1232                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1233                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1234                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1235                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1236                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1237                            TILE_SPLIT(split_equal_to_row_size));
1238                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1239                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1240                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1241                            TILE_SPLIT(split_equal_to_row_size));
1242                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1243                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1244                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1245                            TILE_SPLIT(split_equal_to_row_size));
1246                 tile[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1247                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1248                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1249                            TILE_SPLIT(split_equal_to_row_size));
1250                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1251                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
1252                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1253                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1254                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1255                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1256                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1257                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1258                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1259                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1260                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1261                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1262                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1263                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1264                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1265                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1266                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1267                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1268                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1269                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1270                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1271                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1272                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1273                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1274                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1275                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1276                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1277                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1278                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1279                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1280                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1281                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1282                 tile[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1283                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1284                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1285                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1286                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1287                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1288                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1289                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1290                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1291                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1292                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1293                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1294                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1295                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1296                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1297                 tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1298                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1299                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1300                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1301                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1302                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1303                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1304                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1305                 tile[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1306                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1307                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1308                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1309                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1310                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1311                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1312                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1313                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1314                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1315                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1316                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1317                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1318                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1319                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1320                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1321                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1322                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1323                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1324                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1325                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1326                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1327                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1328                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1329                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1330                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1331                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1332                 tile[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1333                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1334                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1335                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1336
1337                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1338                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1339                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1340                                 NUM_BANKS(ADDR_SURF_16_BANK));
1341                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1342                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1343                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1344                                 NUM_BANKS(ADDR_SURF_16_BANK));
1345                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1346                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1347                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1348                                 NUM_BANKS(ADDR_SURF_16_BANK));
1349                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1350                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1351                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1352                                 NUM_BANKS(ADDR_SURF_16_BANK));
1353                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1354                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1355                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1356                                 NUM_BANKS(ADDR_SURF_8_BANK));
1357                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1358                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1359                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1360                                 NUM_BANKS(ADDR_SURF_4_BANK));
1361                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1362                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1363                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1364                                 NUM_BANKS(ADDR_SURF_4_BANK));
1365                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1366                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1367                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1368                                 NUM_BANKS(ADDR_SURF_16_BANK));
1369                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1370                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1371                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1372                                 NUM_BANKS(ADDR_SURF_16_BANK));
1373                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1374                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1375                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1376                                 NUM_BANKS(ADDR_SURF_16_BANK));
1377                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1378                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1379                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1380                                 NUM_BANKS(ADDR_SURF_8_BANK));
1381                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1382                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1383                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1384                                 NUM_BANKS(ADDR_SURF_16_BANK));
1385                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1386                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1387                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1388                                 NUM_BANKS(ADDR_SURF_8_BANK));
1389                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1390                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1391                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1392                                 NUM_BANKS(ADDR_SURF_4_BANK));
1393
1394                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1395                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1396                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1397                         if (reg_offset != 7)
1398                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1399                 break;
1400         case CHIP_KABINI:
1401         case CHIP_KAVERI:
1402         case CHIP_MULLINS:
1403         default:
1404                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1405                            PIPE_CONFIG(ADDR_SURF_P2) |
1406                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1407                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1408                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1409                            PIPE_CONFIG(ADDR_SURF_P2) |
1410                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1411                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1412                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1413                            PIPE_CONFIG(ADDR_SURF_P2) |
1414                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1415                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1416                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1417                            PIPE_CONFIG(ADDR_SURF_P2) |
1418                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1419                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1420                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1421                            PIPE_CONFIG(ADDR_SURF_P2) |
1422                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1423                            TILE_SPLIT(split_equal_to_row_size));
1424                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1425                            PIPE_CONFIG(ADDR_SURF_P2) |
1426                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1427                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1428                            PIPE_CONFIG(ADDR_SURF_P2) |
1429                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1430                            TILE_SPLIT(split_equal_to_row_size));
1431                 tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1432                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1433                            PIPE_CONFIG(ADDR_SURF_P2));
1434                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1435                            PIPE_CONFIG(ADDR_SURF_P2) |
1436                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1437                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1438                             PIPE_CONFIG(ADDR_SURF_P2) |
1439                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1440                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1441                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1442                             PIPE_CONFIG(ADDR_SURF_P2) |
1443                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1444                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1445                 tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1446                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1447                             PIPE_CONFIG(ADDR_SURF_P2) |
1448                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1449                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1450                             PIPE_CONFIG(ADDR_SURF_P2) |
1451                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1452                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1453                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1454                             PIPE_CONFIG(ADDR_SURF_P2) |
1455                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1456                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1457                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1458                             PIPE_CONFIG(ADDR_SURF_P2) |
1459                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1460                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1461                 tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1462                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1463                             PIPE_CONFIG(ADDR_SURF_P2) |
1464                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1465                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1466                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1467                             PIPE_CONFIG(ADDR_SURF_P2) |
1468                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1469                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1470                             PIPE_CONFIG(ADDR_SURF_P2) |
1471                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1472                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1473                 tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1474                             PIPE_CONFIG(ADDR_SURF_P2) |
1475                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1476                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1477                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1478                             PIPE_CONFIG(ADDR_SURF_P2) |
1479                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1480                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1481                 tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1482                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1483                             PIPE_CONFIG(ADDR_SURF_P2) |
1484                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1485                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1486                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1487                             PIPE_CONFIG(ADDR_SURF_P2) |
1488                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1489                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1490                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1491                             PIPE_CONFIG(ADDR_SURF_P2) |
1492                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1493                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1494                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1495                             PIPE_CONFIG(ADDR_SURF_P2) |
1496                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1497                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1498                             PIPE_CONFIG(ADDR_SURF_P2) |
1499                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1500                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1501                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1502                             PIPE_CONFIG(ADDR_SURF_P2) |
1503                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1504                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1505                 tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1506
1507                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1508                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1509                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1510                                 NUM_BANKS(ADDR_SURF_8_BANK));
1511                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1512                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1513                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1514                                 NUM_BANKS(ADDR_SURF_8_BANK));
1515                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1516                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1517                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1518                                 NUM_BANKS(ADDR_SURF_8_BANK));
1519                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1520                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1521                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1522                                 NUM_BANKS(ADDR_SURF_8_BANK));
1523                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1524                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1525                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1526                                 NUM_BANKS(ADDR_SURF_8_BANK));
1527                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1528                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1529                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1530                                 NUM_BANKS(ADDR_SURF_8_BANK));
1531                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1532                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1533                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1534                                 NUM_BANKS(ADDR_SURF_8_BANK));
1535                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1536                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1537                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1538                                 NUM_BANKS(ADDR_SURF_16_BANK));
1539                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1540                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1541                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1542                                 NUM_BANKS(ADDR_SURF_16_BANK));
1543                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1544                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1545                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1546                                 NUM_BANKS(ADDR_SURF_16_BANK));
1547                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1548                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1549                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1550                                 NUM_BANKS(ADDR_SURF_16_BANK));
1551                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1552                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1553                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1554                                 NUM_BANKS(ADDR_SURF_16_BANK));
1555                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1556                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1557                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1558                                 NUM_BANKS(ADDR_SURF_16_BANK));
1559                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1560                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1561                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1562                                 NUM_BANKS(ADDR_SURF_8_BANK));
1563
1564                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1565                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1566                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1567                         if (reg_offset != 7)
1568                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1569                 break;
1570         }
1571 }
1572
1573 /**
1574  * gfx_v7_0_select_se_sh - select which SE, SH to address
1575  *
1576  * @adev: amdgpu_device pointer
1577  * @se_num: shader engine to address
1578  * @sh_num: sh block to address
1579  *
1580  * Select which SE, SH combinations to address. Certain
1581  * registers are instanced per SE or SH.  0xffffffff means
1582  * broadcast to all SEs or SHs (CIK).
1583  */
1584 static void gfx_v7_0_select_se_sh(struct amdgpu_device *adev,
1585                                   u32 se_num, u32 sh_num, u32 instance)
1586 {
1587         u32 data;
1588
1589         if (instance == 0xffffffff)
1590                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1591         else
1592                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
1593
1594         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1595                 data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1596                         GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
1597         else if (se_num == 0xffffffff)
1598                 data |= GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK |
1599                         (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT);
1600         else if (sh_num == 0xffffffff)
1601                 data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1602                         (se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1603         else
1604                 data |= (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT) |
1605                         (se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1606         WREG32(mmGRBM_GFX_INDEX, data);
1607 }
1608
1609 /**
1610  * gfx_v7_0_create_bitmask - create a bitmask
1611  *
1612  * @bit_width: length of the mask
1613  *
1614  * create a variable length bit mask (CIK).
1615  * Returns the bitmask.
1616  */
1617 static u32 gfx_v7_0_create_bitmask(u32 bit_width)
1618 {
1619         return (u32)((1ULL << bit_width) - 1);
1620 }
1621
1622 /**
1623  * gfx_v7_0_get_rb_active_bitmap - computes the mask of enabled RBs
1624  *
1625  * @adev: amdgpu_device pointer
1626  *
1627  * Calculates the bitmask of enabled RBs (CIK).
1628  * Returns the enabled RB bitmask.
1629  */
1630 static u32 gfx_v7_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1631 {
1632         u32 data, mask;
1633
1634         data = RREG32(mmCC_RB_BACKEND_DISABLE);
1635         data |= RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1636
1637         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1638         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1639
1640         mask = gfx_v7_0_create_bitmask(adev->gfx.config.max_backends_per_se /
1641                                        adev->gfx.config.max_sh_per_se);
1642
1643         return (~data) & mask;
1644 }
1645
1646 static void
1647 gfx_v7_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
1648 {
1649         switch (adev->asic_type) {
1650         case CHIP_BONAIRE:
1651                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
1652                           SE_XSEL(1) | SE_YSEL(1);
1653                 *rconf1 |= 0x0;
1654                 break;
1655         case CHIP_HAWAII:
1656                 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
1657                           RB_XSEL2(1) | PKR_MAP(2) | PKR_XSEL(1) |
1658                           PKR_YSEL(1) | SE_MAP(2) | SE_XSEL(2) |
1659                           SE_YSEL(3);
1660                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
1661                            SE_PAIR_YSEL(2);
1662                 break;
1663         case CHIP_KAVERI:
1664                 *rconf |= RB_MAP_PKR0(2);
1665                 *rconf1 |= 0x0;
1666                 break;
1667         case CHIP_KABINI:
1668         case CHIP_MULLINS:
1669                 *rconf |= 0x0;
1670                 *rconf1 |= 0x0;
1671                 break;
1672         default:
1673                 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
1674                 break;
1675         }
1676 }
1677
1678 static void
1679 gfx_v7_0_write_harvested_raster_configs(struct amdgpu_device *adev,
1680                                         u32 raster_config, u32 raster_config_1,
1681                                         unsigned rb_mask, unsigned num_rb)
1682 {
1683         unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
1684         unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
1685         unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
1686         unsigned rb_per_se = num_rb / num_se;
1687         unsigned se_mask[4];
1688         unsigned se;
1689
1690         se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
1691         se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
1692         se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
1693         se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
1694
1695         WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
1696         WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
1697         WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
1698
1699         if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
1700                              (!se_mask[2] && !se_mask[3]))) {
1701                 raster_config_1 &= ~SE_PAIR_MAP_MASK;
1702
1703                 if (!se_mask[0] && !se_mask[1]) {
1704                         raster_config_1 |=
1705                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
1706                 } else {
1707                         raster_config_1 |=
1708                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
1709                 }
1710         }
1711
1712         for (se = 0; se < num_se; se++) {
1713                 unsigned raster_config_se = raster_config;
1714                 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
1715                 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
1716                 int idx = (se / 2) * 2;
1717
1718                 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
1719                         raster_config_se &= ~SE_MAP_MASK;
1720
1721                         if (!se_mask[idx]) {
1722                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
1723                         } else {
1724                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
1725                         }
1726                 }
1727
1728                 pkr0_mask &= rb_mask;
1729                 pkr1_mask &= rb_mask;
1730                 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
1731                         raster_config_se &= ~PKR_MAP_MASK;
1732
1733                         if (!pkr0_mask) {
1734                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
1735                         } else {
1736                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
1737                         }
1738                 }
1739
1740                 if (rb_per_se >= 2) {
1741                         unsigned rb0_mask = 1 << (se * rb_per_se);
1742                         unsigned rb1_mask = rb0_mask << 1;
1743
1744                         rb0_mask &= rb_mask;
1745                         rb1_mask &= rb_mask;
1746                         if (!rb0_mask || !rb1_mask) {
1747                                 raster_config_se &= ~RB_MAP_PKR0_MASK;
1748
1749                                 if (!rb0_mask) {
1750                                         raster_config_se |=
1751                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
1752                                 } else {
1753                                         raster_config_se |=
1754                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
1755                                 }
1756                         }
1757
1758                         if (rb_per_se > 2) {
1759                                 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
1760                                 rb1_mask = rb0_mask << 1;
1761                                 rb0_mask &= rb_mask;
1762                                 rb1_mask &= rb_mask;
1763                                 if (!rb0_mask || !rb1_mask) {
1764                                         raster_config_se &= ~RB_MAP_PKR1_MASK;
1765
1766                                         if (!rb0_mask) {
1767                                                 raster_config_se |=
1768                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
1769                                         } else {
1770                                                 raster_config_se |=
1771                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
1772                                         }
1773                                 }
1774                         }
1775                 }
1776
1777                 /* GRBM_GFX_INDEX has a different offset on CI+ */
1778                 gfx_v7_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
1779                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
1780                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1781         }
1782
1783         /* GRBM_GFX_INDEX has a different offset on CI+ */
1784         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1785 }
1786
1787 /**
1788  * gfx_v7_0_setup_rb - setup the RBs on the asic
1789  *
1790  * @adev: amdgpu_device pointer
1791  * @se_num: number of SEs (shader engines) for the asic
1792  * @sh_per_se: number of SH blocks per SE for the asic
1793  *
1794  * Configures per-SE/SH RB registers (CIK).
1795  */
1796 static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
1797 {
1798         int i, j;
1799         u32 data;
1800         u32 raster_config = 0, raster_config_1 = 0;
1801         u32 active_rbs = 0;
1802         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1803                                         adev->gfx.config.max_sh_per_se;
1804         unsigned num_rb_pipes;
1805
1806         mutex_lock(&adev->grbm_idx_mutex);
1807         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1808                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1809                         gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
1810                         data = gfx_v7_0_get_rb_active_bitmap(adev);
1811                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1812                                                rb_bitmap_width_per_sh);
1813                 }
1814         }
1815         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1816
1817         adev->gfx.config.backend_enable_mask = active_rbs;
1818         adev->gfx.config.num_rbs = hweight32(active_rbs);
1819
1820         num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
1821                              adev->gfx.config.max_shader_engines, 16);
1822
1823         gfx_v7_0_raster_config(adev, &raster_config, &raster_config_1);
1824
1825         if (!adev->gfx.config.backend_enable_mask ||
1826                         adev->gfx.config.num_rbs >= num_rb_pipes) {
1827                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
1828                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1829         } else {
1830                 gfx_v7_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
1831                                                         adev->gfx.config.backend_enable_mask,
1832                                                         num_rb_pipes);
1833         }
1834         mutex_unlock(&adev->grbm_idx_mutex);
1835 }
1836
1837 /**
1838  * gmc_v7_0_init_compute_vmid - gart enable
1839  *
1840  * @rdev: amdgpu_device pointer
1841  *
1842  * Initialize compute vmid sh_mem registers
1843  *
1844  */
1845 #define DEFAULT_SH_MEM_BASES    (0x6000)
1846 #define FIRST_COMPUTE_VMID      (8)
1847 #define LAST_COMPUTE_VMID       (16)
1848 static void gmc_v7_0_init_compute_vmid(struct amdgpu_device *adev)
1849 {
1850         int i;
1851         uint32_t sh_mem_config;
1852         uint32_t sh_mem_bases;
1853
1854         /*
1855          * Configure apertures:
1856          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1857          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1858          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1859         */
1860         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1861         sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1862                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1863         sh_mem_config |= MTYPE_NONCACHED << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT;
1864         mutex_lock(&adev->srbm_mutex);
1865         for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1866                 cik_srbm_select(adev, 0, 0, 0, i);
1867                 /* CP and shaders */
1868                 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
1869                 WREG32(mmSH_MEM_APE1_BASE, 1);
1870                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
1871                 WREG32(mmSH_MEM_BASES, sh_mem_bases);
1872         }
1873         cik_srbm_select(adev, 0, 0, 0, 0);
1874         mutex_unlock(&adev->srbm_mutex);
1875 }
1876
1877 static void gfx_v7_0_config_init(struct amdgpu_device *adev)
1878 {
1879         adev->gfx.config.double_offchip_lds_buf = 1;
1880 }
1881
1882 /**
1883  * gfx_v7_0_gpu_init - setup the 3D engine
1884  *
1885  * @adev: amdgpu_device pointer
1886  *
1887  * Configures the 3D engine and tiling configuration
1888  * registers so that the 3D engine is usable.
1889  */
1890 static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
1891 {
1892         u32 sh_mem_cfg, sh_static_mem_cfg, sh_mem_base;
1893         u32 tmp;
1894         int i;
1895
1896         WREG32(mmGRBM_CNTL, (0xff << GRBM_CNTL__READ_TIMEOUT__SHIFT));
1897
1898         WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1899         WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1900         WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
1901
1902         gfx_v7_0_tiling_mode_table_init(adev);
1903
1904         gfx_v7_0_setup_rb(adev);
1905         gfx_v7_0_get_cu_info(adev);
1906         gfx_v7_0_config_init(adev);
1907
1908         /* set HW defaults for 3D engine */
1909         WREG32(mmCP_MEQ_THRESHOLDS,
1910                (0x30 << CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT) |
1911                (0x60 << CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT));
1912
1913         mutex_lock(&adev->grbm_idx_mutex);
1914         /*
1915          * making sure that the following register writes will be broadcasted
1916          * to all the shaders
1917          */
1918         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1919
1920         /* XXX SH_MEM regs */
1921         /* where to put LDS, scratch, GPUVM in FSA64 space */
1922         sh_mem_cfg = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1923                                    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1924         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, DEFAULT_MTYPE,
1925                                    MTYPE_NC);
1926         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, APE1_MTYPE,
1927                                    MTYPE_UC);
1928         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, PRIVATE_ATC, 0);
1929
1930         sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
1931                                    SWIZZLE_ENABLE, 1);
1932         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1933                                    ELEMENT_SIZE, 1);
1934         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1935                                    INDEX_STRIDE, 3);
1936
1937         mutex_lock(&adev->srbm_mutex);
1938         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
1939                 if (i == 0)
1940                         sh_mem_base = 0;
1941                 else
1942                         sh_mem_base = adev->mc.shared_aperture_start >> 48;
1943                 cik_srbm_select(adev, 0, 0, 0, i);
1944                 /* CP and shaders */
1945                 WREG32(mmSH_MEM_CONFIG, sh_mem_cfg);
1946                 WREG32(mmSH_MEM_APE1_BASE, 1);
1947                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
1948                 WREG32(mmSH_MEM_BASES, sh_mem_base);
1949                 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
1950         }
1951         cik_srbm_select(adev, 0, 0, 0, 0);
1952         mutex_unlock(&adev->srbm_mutex);
1953
1954         gmc_v7_0_init_compute_vmid(adev);
1955
1956         WREG32(mmSX_DEBUG_1, 0x20);
1957
1958         WREG32(mmTA_CNTL_AUX, 0x00010000);
1959
1960         tmp = RREG32(mmSPI_CONFIG_CNTL);
1961         tmp |= 0x03000000;
1962         WREG32(mmSPI_CONFIG_CNTL, tmp);
1963
1964         WREG32(mmSQ_CONFIG, 1);
1965
1966         WREG32(mmDB_DEBUG, 0);
1967
1968         tmp = RREG32(mmDB_DEBUG2) & ~0xf00fffff;
1969         tmp |= 0x00000400;
1970         WREG32(mmDB_DEBUG2, tmp);
1971
1972         tmp = RREG32(mmDB_DEBUG3) & ~0x0002021c;
1973         tmp |= 0x00020200;
1974         WREG32(mmDB_DEBUG3, tmp);
1975
1976         tmp = RREG32(mmCB_HW_CONTROL) & ~0x00010000;
1977         tmp |= 0x00018208;
1978         WREG32(mmCB_HW_CONTROL, tmp);
1979
1980         WREG32(mmSPI_CONFIG_CNTL_1, (4 << SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT));
1981
1982         WREG32(mmPA_SC_FIFO_SIZE,
1983                 ((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1984                 (adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1985                 (adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1986                 (adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)));
1987
1988         WREG32(mmVGT_NUM_INSTANCES, 1);
1989
1990         WREG32(mmCP_PERFMON_CNTL, 0);
1991
1992         WREG32(mmSQ_CONFIG, 0);
1993
1994         WREG32(mmPA_SC_FORCE_EOV_MAX_CNTS,
1995                 ((4095 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT) |
1996                 (255 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT)));
1997
1998         WREG32(mmVGT_CACHE_INVALIDATION,
1999                 (VC_AND_TC << VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT) |
2000                 (ES_AND_GS_AUTO << VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT));
2001
2002         WREG32(mmVGT_GS_VERTEX_REUSE, 16);
2003         WREG32(mmPA_SC_LINE_STIPPLE_STATE, 0);
2004
2005         WREG32(mmPA_CL_ENHANCE, PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK |
2006                         (3 << PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT));
2007         WREG32(mmPA_SC_ENHANCE, PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK);
2008
2009         tmp = RREG32(mmSPI_ARB_PRIORITY);
2010         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
2011         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
2012         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
2013         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
2014         WREG32(mmSPI_ARB_PRIORITY, tmp);
2015
2016         mutex_unlock(&adev->grbm_idx_mutex);
2017
2018         udelay(50);
2019 }
2020
2021 /*
2022  * GPU scratch registers helpers function.
2023  */
2024 /**
2025  * gfx_v7_0_scratch_init - setup driver info for CP scratch regs
2026  *
2027  * @adev: amdgpu_device pointer
2028  *
2029  * Set up the number and offset of the CP scratch registers.
2030  * NOTE: use of CP scratch registers is a legacy inferface and
2031  * is not used by default on newer asics (r6xx+).  On newer asics,
2032  * memory buffers are used for fences rather than scratch regs.
2033  */
2034 static void gfx_v7_0_scratch_init(struct amdgpu_device *adev)
2035 {
2036         adev->gfx.scratch.num_reg = 7;
2037         adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
2038         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
2039 }
2040
2041 /**
2042  * gfx_v7_0_ring_test_ring - basic gfx ring test
2043  *
2044  * @adev: amdgpu_device pointer
2045  * @ring: amdgpu_ring structure holding ring information
2046  *
2047  * Allocate a scratch register and write to it using the gfx ring (CIK).
2048  * Provides a basic gfx ring test to verify that the ring is working.
2049  * Used by gfx_v7_0_cp_gfx_resume();
2050  * Returns 0 on success, error on failure.
2051  */
2052 static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
2053 {
2054         struct amdgpu_device *adev = ring->adev;
2055         uint32_t scratch;
2056         uint32_t tmp = 0;
2057         unsigned i;
2058         int r;
2059
2060         r = amdgpu_gfx_scratch_get(adev, &scratch);
2061         if (r) {
2062                 DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
2063                 return r;
2064         }
2065         WREG32(scratch, 0xCAFEDEAD);
2066         r = amdgpu_ring_alloc(ring, 3);
2067         if (r) {
2068                 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n", ring->idx, r);
2069                 amdgpu_gfx_scratch_free(adev, scratch);
2070                 return r;
2071         }
2072         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2073         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
2074         amdgpu_ring_write(ring, 0xDEADBEEF);
2075         amdgpu_ring_commit(ring);
2076
2077         for (i = 0; i < adev->usec_timeout; i++) {
2078                 tmp = RREG32(scratch);
2079                 if (tmp == 0xDEADBEEF)
2080                         break;
2081                 DRM_UDELAY(1);
2082         }
2083         if (i < adev->usec_timeout) {
2084                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2085         } else {
2086                 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2087                           ring->idx, scratch, tmp);
2088                 r = -EINVAL;
2089         }
2090         amdgpu_gfx_scratch_free(adev, scratch);
2091         return r;
2092 }
2093
2094 /**
2095  * gfx_v7_0_ring_emit_hdp - emit an hdp flush on the cp
2096  *
2097  * @adev: amdgpu_device pointer
2098  * @ridx: amdgpu ring index
2099  *
2100  * Emits an hdp flush on the cp.
2101  */
2102 static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2103 {
2104         u32 ref_and_mask;
2105         int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
2106
2107         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
2108                 switch (ring->me) {
2109                 case 1:
2110                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
2111                         break;
2112                 case 2:
2113                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
2114                         break;
2115                 default:
2116                         return;
2117                 }
2118         } else {
2119                 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
2120         }
2121
2122         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2123         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
2124                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
2125                                  WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
2126         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
2127         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
2128         amdgpu_ring_write(ring, ref_and_mask);
2129         amdgpu_ring_write(ring, ref_and_mask);
2130         amdgpu_ring_write(ring, 0x20); /* poll interval */
2131 }
2132
2133 static void gfx_v7_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
2134 {
2135         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2136         amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
2137                 EVENT_INDEX(4));
2138
2139         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2140         amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
2141                 EVENT_INDEX(0));
2142 }
2143
2144
2145 /**
2146  * gfx_v7_0_ring_emit_hdp_invalidate - emit an hdp invalidate on the cp
2147  *
2148  * @adev: amdgpu_device pointer
2149  * @ridx: amdgpu ring index
2150  *
2151  * Emits an hdp invalidate on the cp.
2152  */
2153 static void gfx_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
2154 {
2155         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2156         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
2157                                  WRITE_DATA_DST_SEL(0) |
2158                                  WR_CONFIRM));
2159         amdgpu_ring_write(ring, mmHDP_DEBUG0);
2160         amdgpu_ring_write(ring, 0);
2161         amdgpu_ring_write(ring, 1);
2162 }
2163
2164 /**
2165  * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring
2166  *
2167  * @adev: amdgpu_device pointer
2168  * @fence: amdgpu fence object
2169  *
2170  * Emits a fence sequnce number on the gfx ring and flushes
2171  * GPU caches.
2172  */
2173 static void gfx_v7_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
2174                                          u64 seq, unsigned flags)
2175 {
2176         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2177         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2178         /* Workaround for cache flush problems. First send a dummy EOP
2179          * event down the pipe with seq one below.
2180          */
2181         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2182         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2183                                  EOP_TC_ACTION_EN |
2184                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2185                                  EVENT_INDEX(5)));
2186         amdgpu_ring_write(ring, addr & 0xfffffffc);
2187         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2188                                 DATA_SEL(1) | INT_SEL(0));
2189         amdgpu_ring_write(ring, lower_32_bits(seq - 1));
2190         amdgpu_ring_write(ring, upper_32_bits(seq - 1));
2191
2192         /* Then send the real EOP event down the pipe. */
2193         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2194         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2195                                  EOP_TC_ACTION_EN |
2196                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2197                                  EVENT_INDEX(5)));
2198         amdgpu_ring_write(ring, addr & 0xfffffffc);
2199         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2200                                 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2201         amdgpu_ring_write(ring, lower_32_bits(seq));
2202         amdgpu_ring_write(ring, upper_32_bits(seq));
2203 }
2204
2205 /**
2206  * gfx_v7_0_ring_emit_fence_compute - emit a fence on the compute ring
2207  *
2208  * @adev: amdgpu_device pointer
2209  * @fence: amdgpu fence object
2210  *
2211  * Emits a fence sequnce number on the compute ring and flushes
2212  * GPU caches.
2213  */
2214 static void gfx_v7_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
2215                                              u64 addr, u64 seq,
2216                                              unsigned flags)
2217 {
2218         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2219         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2220
2221         /* RELEASE_MEM - flush caches, send int */
2222         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
2223         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2224                                  EOP_TC_ACTION_EN |
2225                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2226                                  EVENT_INDEX(5)));
2227         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2228         amdgpu_ring_write(ring, addr & 0xfffffffc);
2229         amdgpu_ring_write(ring, upper_32_bits(addr));
2230         amdgpu_ring_write(ring, lower_32_bits(seq));
2231         amdgpu_ring_write(ring, upper_32_bits(seq));
2232 }
2233
2234 /*
2235  * IB stuff
2236  */
2237 /**
2238  * gfx_v7_0_ring_emit_ib - emit an IB (Indirect Buffer) on the ring
2239  *
2240  * @ring: amdgpu_ring structure holding ring information
2241  * @ib: amdgpu indirect buffer object
2242  *
2243  * Emits an DE (drawing engine) or CE (constant engine) IB
2244  * on the gfx ring.  IBs are usually generated by userspace
2245  * acceleration drivers and submitted to the kernel for
2246  * sheduling on the ring.  This function schedules the IB
2247  * on the gfx ring for execution by the GPU.
2248  */
2249 static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
2250                                       struct amdgpu_ib *ib,
2251                                       unsigned vm_id, bool ctx_switch)
2252 {
2253         u32 header, control = 0;
2254
2255         /* insert SWITCH_BUFFER packet before first IB in the ring frame */
2256         if (ctx_switch) {
2257                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2258                 amdgpu_ring_write(ring, 0);
2259         }
2260
2261         if (ib->flags & AMDGPU_IB_FLAG_CE)
2262                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2263         else
2264                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2265
2266         control |= ib->length_dw | (vm_id << 24);
2267
2268         amdgpu_ring_write(ring, header);
2269         amdgpu_ring_write(ring,
2270 #ifdef __BIG_ENDIAN
2271                           (2 << 0) |
2272 #endif
2273                           (ib->gpu_addr & 0xFFFFFFFC));
2274         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2275         amdgpu_ring_write(ring, control);
2276 }
2277
2278 static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
2279                                           struct amdgpu_ib *ib,
2280                                           unsigned vm_id, bool ctx_switch)
2281 {
2282         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
2283
2284         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2285         amdgpu_ring_write(ring,
2286 #ifdef __BIG_ENDIAN
2287                                           (2 << 0) |
2288 #endif
2289                                           (ib->gpu_addr & 0xFFFFFFFC));
2290         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2291         amdgpu_ring_write(ring, control);
2292 }
2293
2294 static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
2295 {
2296         uint32_t dw2 = 0;
2297
2298         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
2299         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
2300                 gfx_v7_0_ring_emit_vgt_flush(ring);
2301                 /* set load_global_config & load_global_uconfig */
2302                 dw2 |= 0x8001;
2303                 /* set load_cs_sh_regs */
2304                 dw2 |= 0x01000000;
2305                 /* set load_per_context_state & load_gfx_sh_regs */
2306                 dw2 |= 0x10002;
2307         }
2308
2309         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2310         amdgpu_ring_write(ring, dw2);
2311         amdgpu_ring_write(ring, 0);
2312 }
2313
2314 /**
2315  * gfx_v7_0_ring_test_ib - basic ring IB test
2316  *
2317  * @ring: amdgpu_ring structure holding ring information
2318  *
2319  * Allocate an IB and execute it on the gfx ring (CIK).
2320  * Provides a basic gfx ring test to verify that IBs are working.
2321  * Returns 0 on success, error on failure.
2322  */
2323 static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
2324 {
2325         struct amdgpu_device *adev = ring->adev;
2326         struct amdgpu_ib ib;
2327         struct dma_fence *f = NULL;
2328         uint32_t scratch;
2329         uint32_t tmp = 0;
2330         long r;
2331
2332         r = amdgpu_gfx_scratch_get(adev, &scratch);
2333         if (r) {
2334                 DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
2335                 return r;
2336         }
2337         WREG32(scratch, 0xCAFEDEAD);
2338         memset(&ib, 0, sizeof(ib));
2339         r = amdgpu_ib_get(adev, NULL, 256, &ib);
2340         if (r) {
2341                 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
2342                 goto err1;
2343         }
2344         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2345         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
2346         ib.ptr[2] = 0xDEADBEEF;
2347         ib.length_dw = 3;
2348
2349         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
2350         if (r)
2351                 goto err2;
2352
2353         r = dma_fence_wait_timeout(f, false, timeout);
2354         if (r == 0) {
2355                 DRM_ERROR("amdgpu: IB test timed out\n");
2356                 r = -ETIMEDOUT;
2357                 goto err2;
2358         } else if (r < 0) {
2359                 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
2360                 goto err2;
2361         }
2362         tmp = RREG32(scratch);
2363         if (tmp == 0xDEADBEEF) {
2364                 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
2365                 r = 0;
2366         } else {
2367                 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
2368                           scratch, tmp);
2369                 r = -EINVAL;
2370         }
2371
2372 err2:
2373         amdgpu_ib_free(adev, &ib, NULL);
2374         dma_fence_put(f);
2375 err1:
2376         amdgpu_gfx_scratch_free(adev, scratch);
2377         return r;
2378 }
2379
2380 /*
2381  * CP.
2382  * On CIK, gfx and compute now have independant command processors.
2383  *
2384  * GFX
2385  * Gfx consists of a single ring and can process both gfx jobs and
2386  * compute jobs.  The gfx CP consists of three microengines (ME):
2387  * PFP - Pre-Fetch Parser
2388  * ME - Micro Engine
2389  * CE - Constant Engine
2390  * The PFP and ME make up what is considered the Drawing Engine (DE).
2391  * The CE is an asynchronous engine used for updating buffer desciptors
2392  * used by the DE so that they can be loaded into cache in parallel
2393  * while the DE is processing state update packets.
2394  *
2395  * Compute
2396  * The compute CP consists of two microengines (ME):
2397  * MEC1 - Compute MicroEngine 1
2398  * MEC2 - Compute MicroEngine 2
2399  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
2400  * The queues are exposed to userspace and are programmed directly
2401  * by the compute runtime.
2402  */
2403 /**
2404  * gfx_v7_0_cp_gfx_enable - enable/disable the gfx CP MEs
2405  *
2406  * @adev: amdgpu_device pointer
2407  * @enable: enable or disable the MEs
2408  *
2409  * Halts or unhalts the gfx MEs.
2410  */
2411 static void gfx_v7_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2412 {
2413         int i;
2414
2415         if (enable) {
2416                 WREG32(mmCP_ME_CNTL, 0);
2417         } else {
2418                 WREG32(mmCP_ME_CNTL, (CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK));
2419                 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2420                         adev->gfx.gfx_ring[i].ready = false;
2421         }
2422         udelay(50);
2423 }
2424
2425 /**
2426  * gfx_v7_0_cp_gfx_load_microcode - load the gfx CP ME ucode
2427  *
2428  * @adev: amdgpu_device pointer
2429  *
2430  * Loads the gfx PFP, ME, and CE ucode.
2431  * Returns 0 for success, -EINVAL if the ucode is not available.
2432  */
2433 static int gfx_v7_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2434 {
2435         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2436         const struct gfx_firmware_header_v1_0 *ce_hdr;
2437         const struct gfx_firmware_header_v1_0 *me_hdr;
2438         const __le32 *fw_data;
2439         unsigned i, fw_size;
2440
2441         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2442                 return -EINVAL;
2443
2444         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
2445         ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
2446         me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
2447
2448         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2449         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2450         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2451         adev->gfx.pfp_fw_version = le32_to_cpu(pfp_hdr->header.ucode_version);
2452         adev->gfx.ce_fw_version = le32_to_cpu(ce_hdr->header.ucode_version);
2453         adev->gfx.me_fw_version = le32_to_cpu(me_hdr->header.ucode_version);
2454         adev->gfx.me_feature_version = le32_to_cpu(me_hdr->ucode_feature_version);
2455         adev->gfx.ce_feature_version = le32_to_cpu(ce_hdr->ucode_feature_version);
2456         adev->gfx.pfp_feature_version = le32_to_cpu(pfp_hdr->ucode_feature_version);
2457
2458         gfx_v7_0_cp_gfx_enable(adev, false);
2459
2460         /* PFP */
2461         fw_data = (const __le32 *)
2462                 (adev->gfx.pfp_fw->data +
2463                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2464         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2465         WREG32(mmCP_PFP_UCODE_ADDR, 0);
2466         for (i = 0; i < fw_size; i++)
2467                 WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2468         WREG32(mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2469
2470         /* CE */
2471         fw_data = (const __le32 *)
2472                 (adev->gfx.ce_fw->data +
2473                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2474         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
2475         WREG32(mmCP_CE_UCODE_ADDR, 0);
2476         for (i = 0; i < fw_size; i++)
2477                 WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
2478         WREG32(mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
2479
2480         /* ME */
2481         fw_data = (const __le32 *)
2482                 (adev->gfx.me_fw->data +
2483                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2484         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
2485         WREG32(mmCP_ME_RAM_WADDR, 0);
2486         for (i = 0; i < fw_size; i++)
2487                 WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
2488         WREG32(mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
2489
2490         return 0;
2491 }
2492
2493 /**
2494  * gfx_v7_0_cp_gfx_start - start the gfx ring
2495  *
2496  * @adev: amdgpu_device pointer
2497  *
2498  * Enables the ring and loads the clear state context and other
2499  * packets required to init the ring.
2500  * Returns 0 for success, error for failure.
2501  */
2502 static int gfx_v7_0_cp_gfx_start(struct amdgpu_device *adev)
2503 {
2504         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
2505         const struct cs_section_def *sect = NULL;
2506         const struct cs_extent_def *ext = NULL;
2507         int r, i;
2508
2509         /* init the CP */
2510         WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
2511         WREG32(mmCP_ENDIAN_SWAP, 0);
2512         WREG32(mmCP_DEVICE_ID, 1);
2513
2514         gfx_v7_0_cp_gfx_enable(adev, true);
2515
2516         r = amdgpu_ring_alloc(ring, gfx_v7_0_get_csb_size(adev) + 8);
2517         if (r) {
2518                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2519                 return r;
2520         }
2521
2522         /* init the CE partitions.  CE only used for gfx on CIK */
2523         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2524         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2525         amdgpu_ring_write(ring, 0x8000);
2526         amdgpu_ring_write(ring, 0x8000);
2527
2528         /* clear state buffer */
2529         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2530         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2531
2532         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2533         amdgpu_ring_write(ring, 0x80000000);
2534         amdgpu_ring_write(ring, 0x80000000);
2535
2536         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2537                 for (ext = sect->section; ext->extent != NULL; ++ext) {
2538                         if (sect->id == SECT_CONTEXT) {
2539                                 amdgpu_ring_write(ring,
2540                                                   PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2541                                 amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2542                                 for (i = 0; i < ext->reg_count; i++)
2543                                         amdgpu_ring_write(ring, ext->extent[i]);
2544                         }
2545                 }
2546         }
2547
2548         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2549         amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2550         switch (adev->asic_type) {
2551         case CHIP_BONAIRE:
2552                 amdgpu_ring_write(ring, 0x16000012);
2553                 amdgpu_ring_write(ring, 0x00000000);
2554                 break;
2555         case CHIP_KAVERI:
2556                 amdgpu_ring_write(ring, 0x00000000); /* XXX */
2557                 amdgpu_ring_write(ring, 0x00000000);
2558                 break;
2559         case CHIP_KABINI:
2560         case CHIP_MULLINS:
2561                 amdgpu_ring_write(ring, 0x00000000); /* XXX */
2562                 amdgpu_ring_write(ring, 0x00000000);
2563                 break;
2564         case CHIP_HAWAII:
2565                 amdgpu_ring_write(ring, 0x3a00161a);
2566                 amdgpu_ring_write(ring, 0x0000002e);
2567                 break;
2568         default:
2569                 amdgpu_ring_write(ring, 0x00000000);
2570                 amdgpu_ring_write(ring, 0x00000000);
2571                 break;
2572         }
2573
2574         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2575         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2576
2577         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2578         amdgpu_ring_write(ring, 0);
2579
2580         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2581         amdgpu_ring_write(ring, 0x00000316);
2582         amdgpu_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2583         amdgpu_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2584
2585         amdgpu_ring_commit(ring);
2586
2587         return 0;
2588 }
2589
2590 /**
2591  * gfx_v7_0_cp_gfx_resume - setup the gfx ring buffer registers
2592  *
2593  * @adev: amdgpu_device pointer
2594  *
2595  * Program the location and size of the gfx ring buffer
2596  * and test it to make sure it's working.
2597  * Returns 0 for success, error for failure.
2598  */
2599 static int gfx_v7_0_cp_gfx_resume(struct amdgpu_device *adev)
2600 {
2601         struct amdgpu_ring *ring;
2602         u32 tmp;
2603         u32 rb_bufsz;
2604         u64 rb_addr, rptr_addr;
2605         int r;
2606
2607         WREG32(mmCP_SEM_WAIT_TIMER, 0x0);
2608         if (adev->asic_type != CHIP_HAWAII)
2609                 WREG32(mmCP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2610
2611         /* Set the write pointer delay */
2612         WREG32(mmCP_RB_WPTR_DELAY, 0);
2613
2614         /* set the RB to use vmid 0 */
2615         WREG32(mmCP_RB_VMID, 0);
2616
2617         WREG32(mmSCRATCH_ADDR, 0);
2618
2619         /* ring 0 - compute and gfx */
2620         /* Set ring buffer size */
2621         ring = &adev->gfx.gfx_ring[0];
2622         rb_bufsz = order_base_2(ring->ring_size / 8);
2623         tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2624 #ifdef __BIG_ENDIAN
2625         tmp |= 2 << CP_RB0_CNTL__BUF_SWAP__SHIFT;
2626 #endif
2627         WREG32(mmCP_RB0_CNTL, tmp);
2628
2629         /* Initialize the ring buffer's read and write pointers */
2630         WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
2631         ring->wptr = 0;
2632         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2633
2634         /* set the wb address wether it's enabled or not */
2635         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2636         WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2637         WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2638
2639         /* scratch register shadowing is no longer supported */
2640         WREG32(mmSCRATCH_UMSK, 0);
2641
2642         mdelay(1);
2643         WREG32(mmCP_RB0_CNTL, tmp);
2644
2645         rb_addr = ring->gpu_addr >> 8;
2646         WREG32(mmCP_RB0_BASE, rb_addr);
2647         WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2648
2649         /* start the ring */
2650         gfx_v7_0_cp_gfx_start(adev);
2651         ring->ready = true;
2652         r = amdgpu_ring_test_ring(ring);
2653         if (r) {
2654                 ring->ready = false;
2655                 return r;
2656         }
2657
2658         return 0;
2659 }
2660
2661 static u64 gfx_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
2662 {
2663         return ring->adev->wb.wb[ring->rptr_offs];
2664 }
2665
2666 static u64 gfx_v7_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
2667 {
2668         struct amdgpu_device *adev = ring->adev;
2669
2670         return RREG32(mmCP_RB0_WPTR);
2671 }
2672
2673 static void gfx_v7_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2674 {
2675         struct amdgpu_device *adev = ring->adev;
2676
2677         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2678         (void)RREG32(mmCP_RB0_WPTR);
2679 }
2680
2681 static u64 gfx_v7_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
2682 {
2683         /* XXX check if swapping is necessary on BE */
2684         return ring->adev->wb.wb[ring->wptr_offs];
2685 }
2686
2687 static void gfx_v7_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
2688 {
2689         struct amdgpu_device *adev = ring->adev;
2690
2691         /* XXX check if swapping is necessary on BE */
2692         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
2693         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2694 }
2695
2696 /**
2697  * gfx_v7_0_cp_compute_enable - enable/disable the compute CP MEs
2698  *
2699  * @adev: amdgpu_device pointer
2700  * @enable: enable or disable the MEs
2701  *
2702  * Halts or unhalts the compute MEs.
2703  */
2704 static void gfx_v7_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2705 {
2706         int i;
2707
2708         if (enable) {
2709                 WREG32(mmCP_MEC_CNTL, 0);
2710         } else {
2711                 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
2712                 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2713                         adev->gfx.compute_ring[i].ready = false;
2714         }
2715         udelay(50);
2716 }
2717
2718 /**
2719  * gfx_v7_0_cp_compute_load_microcode - load the compute CP ME ucode
2720  *
2721  * @adev: amdgpu_device pointer
2722  *
2723  * Loads the compute MEC1&2 ucode.
2724  * Returns 0 for success, -EINVAL if the ucode is not available.
2725  */
2726 static int gfx_v7_0_cp_compute_load_microcode(struct amdgpu_device *adev)
2727 {
2728         const struct gfx_firmware_header_v1_0 *mec_hdr;
2729         const __le32 *fw_data;
2730         unsigned i, fw_size;
2731
2732         if (!adev->gfx.mec_fw)
2733                 return -EINVAL;
2734
2735         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2736         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2737         adev->gfx.mec_fw_version = le32_to_cpu(mec_hdr->header.ucode_version);
2738         adev->gfx.mec_feature_version = le32_to_cpu(
2739                                         mec_hdr->ucode_feature_version);
2740
2741         gfx_v7_0_cp_compute_enable(adev, false);
2742
2743         /* MEC1 */
2744         fw_data = (const __le32 *)
2745                 (adev->gfx.mec_fw->data +
2746                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2747         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
2748         WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2749         for (i = 0; i < fw_size; i++)
2750                 WREG32(mmCP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
2751         WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2752
2753         if (adev->asic_type == CHIP_KAVERI) {
2754                 const struct gfx_firmware_header_v1_0 *mec2_hdr;
2755
2756                 if (!adev->gfx.mec2_fw)
2757                         return -EINVAL;
2758
2759                 mec2_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
2760                 amdgpu_ucode_print_gfx_hdr(&mec2_hdr->header);
2761                 adev->gfx.mec2_fw_version = le32_to_cpu(mec2_hdr->header.ucode_version);
2762                 adev->gfx.mec2_feature_version = le32_to_cpu(
2763                                 mec2_hdr->ucode_feature_version);
2764
2765                 /* MEC2 */
2766                 fw_data = (const __le32 *)
2767                         (adev->gfx.mec2_fw->data +
2768                          le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
2769                 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
2770                 WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2771                 for (i = 0; i < fw_size; i++)
2772                         WREG32(mmCP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
2773                 WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2774         }
2775
2776         return 0;
2777 }
2778
2779 /**
2780  * gfx_v7_0_cp_compute_fini - stop the compute queues
2781  *
2782  * @adev: amdgpu_device pointer
2783  *
2784  * Stop the compute queues and tear down the driver queue
2785  * info.
2786  */
2787 static void gfx_v7_0_cp_compute_fini(struct amdgpu_device *adev)
2788 {
2789         int i, r;
2790
2791         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2792                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
2793
2794                 if (ring->mqd_obj) {
2795                         r = amdgpu_bo_reserve(ring->mqd_obj, true);
2796                         if (unlikely(r != 0))
2797                                 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
2798
2799                         amdgpu_bo_unpin(ring->mqd_obj);
2800                         amdgpu_bo_unreserve(ring->mqd_obj);
2801
2802                         amdgpu_bo_unref(&ring->mqd_obj);
2803                         ring->mqd_obj = NULL;
2804                 }
2805         }
2806 }
2807
2808 static void gfx_v7_0_mec_fini(struct amdgpu_device *adev)
2809 {
2810         int r;
2811
2812         if (adev->gfx.mec.hpd_eop_obj) {
2813                 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, true);
2814                 if (unlikely(r != 0))
2815                         dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
2816                 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
2817                 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2818
2819                 amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
2820                 adev->gfx.mec.hpd_eop_obj = NULL;
2821         }
2822 }
2823
2824 #define MEC_HPD_SIZE 2048
2825
2826 static int gfx_v7_0_mec_init(struct amdgpu_device *adev)
2827 {
2828         int r;
2829         u32 *hpd;
2830
2831         /*
2832          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
2833          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
2834          * Nonetheless, we assign only 1 pipe because all other pipes will
2835          * be handled by KFD
2836          */
2837         adev->gfx.mec.num_mec = 1;
2838         adev->gfx.mec.num_pipe = 1;
2839         adev->gfx.mec.num_queue = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe * 8;
2840
2841         if (adev->gfx.mec.hpd_eop_obj == NULL) {
2842                 r = amdgpu_bo_create(adev,
2843                                      adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2,
2844                                      PAGE_SIZE, true,
2845                                      AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
2846                                      &adev->gfx.mec.hpd_eop_obj);
2847                 if (r) {
2848                         dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
2849                         return r;
2850                 }
2851         }
2852
2853         r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
2854         if (unlikely(r != 0)) {
2855                 gfx_v7_0_mec_fini(adev);
2856                 return r;
2857         }
2858         r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
2859                           &adev->gfx.mec.hpd_eop_gpu_addr);
2860         if (r) {
2861                 dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
2862                 gfx_v7_0_mec_fini(adev);
2863                 return r;
2864         }
2865         r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
2866         if (r) {
2867                 dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
2868                 gfx_v7_0_mec_fini(adev);
2869                 return r;
2870         }
2871
2872         /* clear memory.  Not sure if this is required or not */
2873         memset(hpd, 0, adev->gfx.mec.num_mec *adev->gfx.mec.num_pipe * MEC_HPD_SIZE * 2);
2874
2875         amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2876         amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2877
2878         return 0;
2879 }
2880
2881 struct hqd_registers
2882 {
2883         u32 cp_mqd_base_addr;
2884         u32 cp_mqd_base_addr_hi;
2885         u32 cp_hqd_active;
2886         u32 cp_hqd_vmid;
2887         u32 cp_hqd_persistent_state;
2888         u32 cp_hqd_pipe_priority;
2889         u32 cp_hqd_queue_priority;
2890         u32 cp_hqd_quantum;
2891         u32 cp_hqd_pq_base;
2892         u32 cp_hqd_pq_base_hi;
2893         u32 cp_hqd_pq_rptr;
2894         u32 cp_hqd_pq_rptr_report_addr;
2895         u32 cp_hqd_pq_rptr_report_addr_hi;
2896         u32 cp_hqd_pq_wptr_poll_addr;
2897         u32 cp_hqd_pq_wptr_poll_addr_hi;
2898         u32 cp_hqd_pq_doorbell_control;
2899         u32 cp_hqd_pq_wptr;
2900         u32 cp_hqd_pq_control;
2901         u32 cp_hqd_ib_base_addr;
2902         u32 cp_hqd_ib_base_addr_hi;
2903         u32 cp_hqd_ib_rptr;
2904         u32 cp_hqd_ib_control;
2905         u32 cp_hqd_iq_timer;
2906         u32 cp_hqd_iq_rptr;
2907         u32 cp_hqd_dequeue_request;
2908         u32 cp_hqd_dma_offload;
2909         u32 cp_hqd_sema_cmd;
2910         u32 cp_hqd_msg_type;
2911         u32 cp_hqd_atomic0_preop_lo;
2912         u32 cp_hqd_atomic0_preop_hi;
2913         u32 cp_hqd_atomic1_preop_lo;
2914         u32 cp_hqd_atomic1_preop_hi;
2915         u32 cp_hqd_hq_scheduler0;
2916         u32 cp_hqd_hq_scheduler1;
2917         u32 cp_mqd_control;
2918 };
2919
2920 struct bonaire_mqd
2921 {
2922         u32 header;
2923         u32 dispatch_initiator;
2924         u32 dimensions[3];
2925         u32 start_idx[3];
2926         u32 num_threads[3];
2927         u32 pipeline_stat_enable;
2928         u32 perf_counter_enable;
2929         u32 pgm[2];
2930         u32 tba[2];
2931         u32 tma[2];
2932         u32 pgm_rsrc[2];
2933         u32 vmid;
2934         u32 resource_limits;
2935         u32 static_thread_mgmt01[2];
2936         u32 tmp_ring_size;
2937         u32 static_thread_mgmt23[2];
2938         u32 restart[3];
2939         u32 thread_trace_enable;
2940         u32 reserved1;
2941         u32 user_data[16];
2942         u32 vgtcs_invoke_count[2];
2943         struct hqd_registers queue_state;
2944         u32 dequeue_cntr;
2945         u32 interrupt_queue[64];
2946 };
2947
2948 /**
2949  * gfx_v7_0_cp_compute_resume - setup the compute queue registers
2950  *
2951  * @adev: amdgpu_device pointer
2952  *
2953  * Program the compute queues and test them to make sure they
2954  * are working.
2955  * Returns 0 for success, error for failure.
2956  */
2957 static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
2958 {
2959         int r, i, j;
2960         u32 tmp;
2961         bool use_doorbell = true;
2962         u64 hqd_gpu_addr;
2963         u64 mqd_gpu_addr;
2964         u64 eop_gpu_addr;
2965         u64 wb_gpu_addr;
2966         u32 *buf;
2967         struct bonaire_mqd *mqd;
2968         struct amdgpu_ring *ring;
2969
2970         /* fix up chicken bits */