drivers/gpu/drm/amd/amdgpu/df_v3_6.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "df_v3_6.h"
  25
  26 #include "df/df_3_6_default.h"
  27 #include "df/df_3_6_offset.h"
  28 #include "df/df_3_6_sh_mask.h"
  29
  30 static u32 df_v3_6_channel_number[] = {1, 2, 0, 4, 0, 8, 0,
  31                                        16, 32, 0, 0, 0, 2, 4, 8};
  32
  33 /* init df format attrs */
  34 AMDGPU_PMU_ATTR(event,          "config:0-7");
  35 AMDGPU_PMU_ATTR(instance,       "config:8-15");
  36 AMDGPU_PMU_ATTR(umask,          "config:16-23");
  37
  38 /* df format attributes  */
  39 static struct attribute *df_v3_6_format_attrs[] = {
  40         &pmu_attr_event.attr,
  41         &pmu_attr_instance.attr,
  42         &pmu_attr_umask.attr,
  43         NULL
  44 };
  45
  46 /* df format attribute group */
  47 static struct attribute_group df_v3_6_format_attr_group = {
  48         .name = "format",
  49         .attrs = df_v3_6_format_attrs,
  50 };
  51
  52 /* df event attrs */
  53 AMDGPU_PMU_ATTR(cake0_pcsout_txdata,
  54                       "event=0x7,instance=0x46,umask=0x2");
  55 AMDGPU_PMU_ATTR(cake1_pcsout_txdata,
  56                       "event=0x7,instance=0x47,umask=0x2");
  57 AMDGPU_PMU_ATTR(cake0_pcsout_txmeta,
  58                       "event=0x7,instance=0x46,umask=0x4");
  59 AMDGPU_PMU_ATTR(cake1_pcsout_txmeta,
  60                       "event=0x7,instance=0x47,umask=0x4");
  61 AMDGPU_PMU_ATTR(cake0_ftiinstat_reqalloc,
  62                       "event=0xb,instance=0x46,umask=0x4");
  63 AMDGPU_PMU_ATTR(cake1_ftiinstat_reqalloc,
  64                       "event=0xb,instance=0x47,umask=0x4");
  65 AMDGPU_PMU_ATTR(cake0_ftiinstat_rspalloc,
  66                       "event=0xb,instance=0x46,umask=0x8");
  67 AMDGPU_PMU_ATTR(cake1_ftiinstat_rspalloc,
  68                       "event=0xb,instance=0x47,umask=0x8");
  69
  70 /* df event attributes  */
  71 static struct attribute *df_v3_6_event_attrs[] = {
  72         &pmu_attr_cake0_pcsout_txdata.attr,
  73         &pmu_attr_cake1_pcsout_txdata.attr,
  74         &pmu_attr_cake0_pcsout_txmeta.attr,
  75         &pmu_attr_cake1_pcsout_txmeta.attr,
  76         &pmu_attr_cake0_ftiinstat_reqalloc.attr,
  77         &pmu_attr_cake1_ftiinstat_reqalloc.attr,
  78         &pmu_attr_cake0_ftiinstat_rspalloc.attr,
  79         &pmu_attr_cake1_ftiinstat_rspalloc.attr,
  80         NULL
  81 };
  82
  83 /* df event attribute group */
  84 static struct attribute_group df_v3_6_event_attr_group = {
  85         .name = "events",
  86         .attrs = df_v3_6_event_attrs
  87 };
  88
  89 /* df event attr groups  */
  90 const struct attribute_group *df_v3_6_attr_groups[] = {
  91                 &df_v3_6_format_attr_group,
  92                 &df_v3_6_event_attr_group,
  93                 NULL
  94 };
  95
  96 static uint64_t df_v3_6_get_fica(struct amdgpu_device *adev,
  97                                  uint32_t ficaa_val)
  98 {
  99         unsigned long flags, address, data;
 100         uint32_t ficadl_val, ficadh_val;
 101
 102         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 103         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 104
 105         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 106         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 107         WREG32(data, ficaa_val);
 108
 109         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
 110         ficadl_val = RREG32(data);
 111
 112         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 113         ficadh_val = RREG32(data);
 114
 115         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 116
 117         return (((ficadh_val & 0xFFFFFFFFFFFFFFFF) << 32) | ficadl_val);
 118 }
 119
 120 static void df_v3_6_set_fica(struct amdgpu_device *adev, uint32_t ficaa_val,
 121                              uint32_t ficadl_val, uint32_t ficadh_val)
 122 {
 123         unsigned long flags, address, data;
 124
 125         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 126         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 127
 128         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 129         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessAddress3);
 130         WREG32(data, ficaa_val);
 131
 132         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataLo3);
 133         WREG32(data, ficadl_val);
 134
 135         WREG32(address, smnDF_PIE_AON_FabricIndirectConfigAccessDataHi3);
 136         WREG32(data, ficadh_val);
 137
 138         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 139 }
 140
 141 /*
 142  * df_v3_6_perfmon_rreg - read perfmon lo and hi
 143  *
 144  * required to be atomic.  no mmio method provided so subsequent reads for lo
 145  * and hi require to preserve df finite state machine
 146  */
 147 static void df_v3_6_perfmon_rreg(struct amdgpu_device *adev,
 148                             uint32_t lo_addr, uint32_t *lo_val,
 149                             uint32_t hi_addr, uint32_t *hi_val)
 150 {
 151         unsigned long flags, address, data;
 152
 153         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 154         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 155
 156         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 157         WREG32(address, lo_addr);
 158         *lo_val = RREG32(data);
 159         WREG32(address, hi_addr);
 160         *hi_val = RREG32(data);
 161         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 162 }
 163
 164 /*
 165  * df_v3_6_perfmon_wreg - write to perfmon lo and hi
 166  *
 167  * required to be atomic.  no mmio method provided so subsequent reads after
 168  * data writes cannot occur to preserve data fabrics finite state machine.
 169  */
 170 static void df_v3_6_perfmon_wreg(struct amdgpu_device *adev, uint32_t lo_addr,
 171                             uint32_t lo_val, uint32_t hi_addr, uint32_t hi_val)
 172 {
 173         unsigned long flags, address, data;
 174
 175         address = adev->nbio.funcs->get_pcie_index_offset(adev);
 176         data = adev->nbio.funcs->get_pcie_data_offset(adev);
 177
 178         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 179         WREG32(address, lo_addr);
 180         WREG32(data, lo_val);
 181         WREG32(address, hi_addr);
 182         WREG32(data, hi_val);
 183         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 184 }
 185
 186 /* get the number of df counters available */
 187 static ssize_t df_v3_6_get_df_cntr_avail(struct device *dev,
 188                 struct device_attribute *attr,
 189                 char *buf)
 190 {
 191         struct amdgpu_device *adev;
 192         struct drm_device *ddev;
 193         int i, count;
 194
 195         ddev = dev_get_drvdata(dev);
 196         adev = ddev->dev_private;
 197         count = 0;
 198
 199         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 200                 if (adev->df_perfmon_config_assign_mask[i] == 0)
 201                         count++;
 202         }
 203
 204         return snprintf(buf, PAGE_SIZE, "%i\n", count);
 205 }
 206
 207 /* device attr for available perfmon counters */
 208 static DEVICE_ATTR(df_cntr_avail, S_IRUGO, df_v3_6_get_df_cntr_avail, NULL);
 209
 210 /* init perfmons */
 211 static void df_v3_6_sw_init(struct amdgpu_device *adev)
 212 {
 213         int i, ret;
 214
 215         ret = device_create_file(adev->dev, &dev_attr_df_cntr_avail);
 216         if (ret)
 217                 DRM_ERROR("failed to create file for available df counters\n");
 218
 219         for (i = 0; i < AMDGPU_MAX_DF_PERFMONS; i++)
 220                 adev->df_perfmon_config_assign_mask[i] = 0;
 221 }
 222
 223 static void df_v3_6_sw_fini(struct amdgpu_device *adev)
 224 {
 225
 226         device_remove_file(adev->dev, &dev_attr_df_cntr_avail);
 227
 228 }
 229
 230 static void df_v3_6_enable_broadcast_mode(struct amdgpu_device *adev,
 231                                           bool enable)
 232 {
 233         u32 tmp;
 234
 235         if (enable) {
 236                 tmp = RREG32_SOC15(DF, 0, mmFabricConfigAccessControl);
 237                 tmp &= ~FabricConfigAccessControl__CfgRegInstAccEn_MASK;
 238                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl, tmp);
 239         } else
 240                 WREG32_SOC15(DF, 0, mmFabricConfigAccessControl,
 241                              mmFabricConfigAccessControl_DEFAULT);
 242 }
 243
 244 static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev)
 245 {
 246         u32 tmp;
 247
 248         tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
 249         tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
 250         tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
 251
 252         return tmp;
 253 }
 254
 255 static u32 df_v3_6_get_hbm_channel_number(struct amdgpu_device *adev)
 256 {
 257         int fb_channel_number;
 258
 259         fb_channel_number = adev->df_funcs->get_fb_channel_number(adev);
 260         if (fb_channel_number >= ARRAY_SIZE(df_v3_6_channel_number))
 261                 fb_channel_number = 0;
 262
 263         return df_v3_6_channel_number[fb_channel_number];
 264 }
 265
 266 static void df_v3_6_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 267                                                      bool enable)
 268 {
 269         u32 tmp;
 270
 271         if (adev->cg_flags & AMD_CG_SUPPORT_DF_MGCG) {
 272                 /* Put DF on broadcast mode */
 273                 adev->df_funcs->enable_broadcast_mode(adev, true);
 274
 275                 if (enable) {
 276                         tmp = RREG32_SOC15(DF, 0,
 277                                         mmDF_PIE_AON0_DfGlobalClkGater);
 278                         tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
 279                         tmp |= DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY;
 280                         WREG32_SOC15(DF, 0,
 281                                         mmDF_PIE_AON0_DfGlobalClkGater, tmp);
 282                 } else {
 283                         tmp = RREG32_SOC15(DF, 0,
 284                                         mmDF_PIE_AON0_DfGlobalClkGater);
 285                         tmp &= ~DF_PIE_AON0_DfGlobalClkGater__MGCGMode_MASK;
 286                         tmp |= DF_V3_6_MGCG_DISABLE;
 287                         WREG32_SOC15(DF, 0,
 288                                         mmDF_PIE_AON0_DfGlobalClkGater, tmp);
 289                 }
 290
 291                 /* Exit broadcast mode */
 292                 adev->df_funcs->enable_broadcast_mode(adev, false);
 293         }
 294 }
 295
 296 static void df_v3_6_get_clockgating_state(struct amdgpu_device *adev,
 297                                           u32 *flags)
 298 {
 299         u32 tmp;
 300
 301         /* AMD_CG_SUPPORT_DF_MGCG */
 302         tmp = RREG32_SOC15(DF, 0, mmDF_PIE_AON0_DfGlobalClkGater);
 303         if (tmp & DF_V3_6_MGCG_ENABLE_15_CYCLE_DELAY)
 304                 *flags |= AMD_CG_SUPPORT_DF_MGCG;
 305 }
 306
 307 /* get assigned df perfmon ctr as int */
 308 static int df_v3_6_pmc_config_2_cntr(struct amdgpu_device *adev,
 309                                       uint64_t config)
 310 {
 311         int i;
 312
 313         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 314                 if ((config & 0x0FFFFFFUL) ==
 315                                         adev->df_perfmon_config_assign_mask[i])
 316                         return i;
 317         }
 318
 319         return -EINVAL;
 320 }
 321
 322 /* get address based on counter assignment */
 323 static void df_v3_6_pmc_get_addr(struct amdgpu_device *adev,
 324                                  uint64_t config,
 325                                  int is_ctrl,
 326                                  uint32_t *lo_base_addr,
 327                                  uint32_t *hi_base_addr)
 328 {
 329         int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 330
 331         if (target_cntr < 0)
 332                 return;
 333
 334         switch (target_cntr) {
 335
 336         case 0:
 337                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo0 : smnPerfMonCtrLo0;
 338                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi0 : smnPerfMonCtrHi0;
 339                 break;
 340         case 1:
 341                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo1 : smnPerfMonCtrLo1;
 342                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi1 : smnPerfMonCtrHi1;
 343                 break;
 344         case 2:
 345                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo2 : smnPerfMonCtrLo2;
 346                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi2 : smnPerfMonCtrHi2;
 347                 break;
 348         case 3:
 349                 *lo_base_addr = is_ctrl ? smnPerfMonCtlLo3 : smnPerfMonCtrLo3;
 350                 *hi_base_addr = is_ctrl ? smnPerfMonCtlHi3 : smnPerfMonCtrHi3;
 351                 break;
 352
 353         }
 354
 355 }
 356
 357 /* get read counter address */
 358 static void df_v3_6_pmc_get_read_settings(struct amdgpu_device *adev,
 359                                           uint64_t config,
 360                                           uint32_t *lo_base_addr,
 361                                           uint32_t *hi_base_addr)
 362 {
 363         df_v3_6_pmc_get_addr(adev, config, 0, lo_base_addr, hi_base_addr);
 364 }
 365
 366 /* get control counter settings i.e. address and values to set */
 367 static int df_v3_6_pmc_get_ctrl_settings(struct amdgpu_device *adev,
 368                                           uint64_t config,
 369                                           uint32_t *lo_base_addr,
 370                                           uint32_t *hi_base_addr,
 371                                           uint32_t *lo_val,
 372                                           uint32_t *hi_val)
 373 {
 374
 375         uint32_t eventsel, instance, unitmask;
 376         uint32_t instance_10, instance_5432, instance_76;
 377
 378         df_v3_6_pmc_get_addr(adev, config, 1, lo_base_addr, hi_base_addr);
 379
 380         if ((*lo_base_addr == 0) || (*hi_base_addr == 0)) {
 381                 DRM_ERROR("[DF PMC] addressing not retrieved! Lo: %x, Hi: %x",
 382                                 *lo_base_addr, *hi_base_addr);
 383                 return -ENXIO;
 384         }
 385
 386         eventsel = DF_V3_6_GET_EVENT(config) & 0x3f;
 387         unitmask = DF_V3_6_GET_UNITMASK(config) & 0xf;
 388         instance = DF_V3_6_GET_INSTANCE(config);
 389
 390         instance_10 = instance & 0x3;
 391         instance_5432 = (instance >> 2) & 0xf;
 392         instance_76 = (instance >> 6) & 0x3;
 393
 394         *lo_val = (unitmask << 8) | (instance_10 << 6) | eventsel | (1 << 22);
 395         *hi_val = (instance_76 << 29) | instance_5432;
 396
 397         DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
 398                 config, *lo_base_addr, *hi_base_addr, *lo_val, *hi_val);
 399
 400         return 0;
 401 }
 402
 403 /* add df performance counters for read */
 404 static int df_v3_6_pmc_add_cntr(struct amdgpu_device *adev,
 405                                    uint64_t config)
 406 {
 407         int i, target_cntr;
 408
 409         target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 410
 411         if (target_cntr >= 0)
 412                 return 0;
 413
 414         for (i = 0; i < DF_V3_6_MAX_COUNTERS; i++) {
 415                 if (adev->df_perfmon_config_assign_mask[i] == 0U) {
 416                         adev->df_perfmon_config_assign_mask[i] =
 417                                                         config & 0x0FFFFFFUL;
 418                         return 0;
 419                 }
 420         }
 421
 422         return -ENOSPC;
 423 }
 424
 425 /* release performance counter */
 426 static void df_v3_6_pmc_release_cntr(struct amdgpu_device *adev,
 427                                      uint64_t config)
 428 {
 429         int target_cntr = df_v3_6_pmc_config_2_cntr(adev, config);
 430
 431         if (target_cntr >= 0)
 432                 adev->df_perfmon_config_assign_mask[target_cntr] = 0ULL;
 433 }
 434
 435
 436 static void df_v3_6_reset_perfmon_cntr(struct amdgpu_device *adev,
 437                                          uint64_t config)
 438 {
 439         uint32_t lo_base_addr, hi_base_addr;
 440
 441         df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 442                                       &hi_base_addr);
 443
 444         if ((lo_base_addr == 0) || (hi_base_addr == 0))
 445                 return;
 446
 447         df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
 448 }
 449
 450 static int df_v3_6_pmc_start(struct amdgpu_device *adev, uint64_t config,
 451                              int is_enable)
 452 {
 453         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 454         int ret = 0;
 455
 456         switch (adev->asic_type) {
 457         case CHIP_VEGA20:
 458
 459                 df_v3_6_reset_perfmon_cntr(adev, config);
 460
 461                 if (is_enable) {
 462                         ret = df_v3_6_pmc_add_cntr(adev, config);
 463                 } else {
 464                         ret = df_v3_6_pmc_get_ctrl_settings(adev,
 465                                         config,
 466                                         &lo_base_addr,
 467                                         &hi_base_addr,
 468                                         &lo_val,
 469                                         &hi_val);
 470
 471                         if (ret)
 472                                 return ret;
 473
 474                         df_v3_6_perfmon_wreg(adev, lo_base_addr, lo_val,
 475                                         hi_base_addr, hi_val);
 476                 }
 477
 478                 break;
 479         default:
 480                 break;
 481         }
 482
 483         return ret;
 484 }
 485
 486 static int df_v3_6_pmc_stop(struct amdgpu_device *adev, uint64_t config,
 487                             int is_disable)
 488 {
 489         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 490         int ret = 0;
 491
 492         switch (adev->asic_type) {
 493         case CHIP_VEGA20:
 494                 ret = df_v3_6_pmc_get_ctrl_settings(adev,
 495                         config,
 496                         &lo_base_addr,
 497                         &hi_base_addr,
 498                         &lo_val,
 499                         &hi_val);
 500
 501                 if (ret)
 502                         return ret;
 503
 504                 df_v3_6_perfmon_wreg(adev, lo_base_addr, 0, hi_base_addr, 0);
 505
 506                 if (is_disable)
 507                         df_v3_6_pmc_release_cntr(adev, config);
 508
 509                 break;
 510         default:
 511                 break;
 512         }
 513
 514         return ret;
 515 }
 516
 517 static void df_v3_6_pmc_get_count(struct amdgpu_device *adev,
 518                                   uint64_t config,
 519                                   uint64_t *count)
 520 {
 521         uint32_t lo_base_addr, hi_base_addr, lo_val, hi_val;
 522         *count = 0;
 523
 524         switch (adev->asic_type) {
 525         case CHIP_VEGA20:
 526
 527                 df_v3_6_pmc_get_read_settings(adev, config, &lo_base_addr,
 528                                       &hi_base_addr);
 529
 530                 if ((lo_base_addr == 0) || (hi_base_addr == 0))
 531                         return;
 532
 533                 df_v3_6_perfmon_rreg(adev, lo_base_addr, &lo_val,
 534                                 hi_base_addr, &hi_val);
 535
 536                 *count  = ((hi_val | 0ULL) << 32) | (lo_val | 0ULL);
 537
 538                 if (*count >= DF_V3_6_PERFMON_OVERFLOW)
 539                         *count = 0;
 540
 541                 DRM_DEBUG_DRIVER("config=%llx addr=%08x:%08x val=%08x:%08x",
 542                          config, lo_base_addr, hi_base_addr, lo_val, hi_val);
 543
 544                 break;
 545
 546         default:
 547                 break;
 548         }
 549 }
 550
 551 const struct amdgpu_df_funcs df_v3_6_funcs = {
 552         .sw_init = df_v3_6_sw_init,
 553         .sw_fini = df_v3_6_sw_fini,
 554         .enable_broadcast_mode = df_v3_6_enable_broadcast_mode,
 555         .get_fb_channel_number = df_v3_6_get_fb_channel_number,
 556         .get_hbm_channel_number = df_v3_6_get_hbm_channel_number,
 557         .update_medium_grain_clock_gating =
 558                         df_v3_6_update_medium_grain_clock_gating,
 559         .get_clockgating_state = df_v3_6_get_clockgating_state,
 560         .pmc_start = df_v3_6_pmc_start,
 561         .pmc_stop = df_v3_6_pmc_stop,
 562         .pmc_get_count = df_v3_6_pmc_get_count,
 563         .get_fica = df_v3_6_get_fica,
 564         .set_fica = df_v3_6_set_fica
 565 };