tools/perf/arch/x86/util/intel-bts.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel-bts.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2015, Intel Corporation.
   5  */
   6
   7 #include <errno.h>
   8 #include <linux/kernel.h>
   9 #include <linux/types.h>
  10 #include <linux/bitops.h>
  11 #include <linux/log2.h>
  12
  13 #include "../../util/cpumap.h"
  14 #include "../../util/evsel.h"
  15 #include "../../util/evlist.h"
  16 #include "../../util/session.h"
  17 #include "../../util/util.h"
  18 #include "../../util/pmu.h"
  19 #include "../../util/debug.h"
  20 #include "../../util/tsc.h"
  21 #include "../../util/auxtrace.h"
  22 #include "../../util/intel-bts.h"
  23
  24 #define KiB(x) ((x) * 1024)
  25 #define MiB(x) ((x) * 1024 * 1024)
  26 #define KiB_MASK(x) (KiB(x) - 1)
  27 #define MiB_MASK(x) (MiB(x) - 1)
  28
  29 struct intel_bts_snapshot_ref {
  30         void    *ref_buf;
  31         size_t  ref_offset;
  32         bool    wrapped;
  33 };
  34
  35 struct intel_bts_recording {
  36         struct auxtrace_record          itr;
  37         struct perf_pmu                 *intel_bts_pmu;
  38         struct perf_evlist              *evlist;
  39         bool                            snapshot_mode;
  40         size_t                          snapshot_size;
  41         int                             snapshot_ref_cnt;
  42         struct intel_bts_snapshot_ref   *snapshot_refs;
  43 };
  44
  45 struct branch {
  46         u64 from;
  47         u64 to;
  48         u64 misc;
  49 };
  50
  51 static size_t
  52 intel_bts_info_priv_size(struct auxtrace_record *itr __maybe_unused,
  53                          struct perf_evlist *evlist __maybe_unused)
  54 {
  55         return INTEL_BTS_AUXTRACE_PRIV_SIZE;
  56 }
  57
  58 static int intel_bts_info_fill(struct auxtrace_record *itr,
  59                                struct perf_session *session,
  60                                struct auxtrace_info_event *auxtrace_info,
  61                                size_t priv_size)
  62 {
  63         struct intel_bts_recording *btsr =
  64                         container_of(itr, struct intel_bts_recording, itr);
  65         struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
  66         struct perf_event_mmap_page *pc;
  67         struct perf_tsc_conversion tc = { .time_mult = 0, };
  68         bool cap_user_time_zero = false;
  69         int err;
  70
  71         if (priv_size != INTEL_BTS_AUXTRACE_PRIV_SIZE)
  72                 return -EINVAL;
  73
  74         if (!session->evlist->nr_mmaps)
  75                 return -EINVAL;
  76
  77         pc = session->evlist->mmap[0].base;
  78         if (pc) {
  79                 err = perf_read_tsc_conversion(pc, &tc);
  80                 if (err) {
  81                         if (err != -EOPNOTSUPP)
  82                                 return err;
  83                 } else {
  84                         cap_user_time_zero = tc.time_mult != 0;
  85                 }
  86                 if (!cap_user_time_zero)
  87                         ui__warning("Intel BTS: TSC not available\n");
  88         }
  89
  90         auxtrace_info->type = PERF_AUXTRACE_INTEL_BTS;
  91         auxtrace_info->priv[INTEL_BTS_PMU_TYPE] = intel_bts_pmu->type;
  92         auxtrace_info->priv[INTEL_BTS_TIME_SHIFT] = tc.time_shift;
  93         auxtrace_info->priv[INTEL_BTS_TIME_MULT] = tc.time_mult;
  94         auxtrace_info->priv[INTEL_BTS_TIME_ZERO] = tc.time_zero;
  95         auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO] = cap_user_time_zero;
  96         auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE] = btsr->snapshot_mode;
  97
  98         return 0;
  99 }
 100
 101 static int intel_bts_recording_options(struct auxtrace_record *itr,
 102                                        struct perf_evlist *evlist,
 103                                        struct record_opts *opts)
 104 {
 105         struct intel_bts_recording *btsr =
 106                         container_of(itr, struct intel_bts_recording, itr);
 107         struct perf_pmu *intel_bts_pmu = btsr->intel_bts_pmu;
 108         struct perf_evsel *evsel, *intel_bts_evsel = NULL;
 109         const struct cpu_map *cpus = evlist->cpus;
 110         bool privileged = geteuid() == 0 || perf_event_paranoid() < 0;
 111
 112         btsr->evlist = evlist;
 113         btsr->snapshot_mode = opts->auxtrace_snapshot_mode;
 114
 115         evlist__for_each_entry(evlist, evsel) {
 116                 if (evsel->attr.type == intel_bts_pmu->type) {
 117                         if (intel_bts_evsel) {
 118                                 pr_err("There may be only one " INTEL_BTS_PMU_NAME " event\n");
 119                                 return -EINVAL;
 120                         }
 121                         evsel->attr.freq = 0;
 122                         evsel->attr.sample_period = 1;
 123                         intel_bts_evsel = evsel;
 124                         opts->full_auxtrace = true;
 125                 }
 126         }
 127
 128         if (opts->auxtrace_snapshot_mode && !opts->full_auxtrace) {
 129                 pr_err("Snapshot mode (-S option) requires " INTEL_BTS_PMU_NAME " PMU event (-e " INTEL_BTS_PMU_NAME ")\n");
 130                 return -EINVAL;
 131         }
 132
 133         if (!opts->full_auxtrace)
 134                 return 0;
 135
 136         if (opts->full_auxtrace && !cpu_map__empty(cpus)) {
 137                 pr_err(INTEL_BTS_PMU_NAME " does not support per-cpu recording\n");
 138                 return -EINVAL;
 139         }
 140
 141         /* Set default sizes for snapshot mode */
 142         if (opts->auxtrace_snapshot_mode) {
 143                 if (!opts->auxtrace_snapshot_size && !opts->auxtrace_mmap_pages) {
 144                         if (privileged) {
 145                                 opts->auxtrace_mmap_pages = MiB(4) / page_size;
 146                         } else {
 147                                 opts->auxtrace_mmap_pages = KiB(128) / page_size;
 148                                 if (opts->mmap_pages == UINT_MAX)
 149                                         opts->mmap_pages = KiB(256) / page_size;
 150                         }
 151                 } else if (!opts->auxtrace_mmap_pages && !privileged &&
 152                            opts->mmap_pages == UINT_MAX) {
 153                         opts->mmap_pages = KiB(256) / page_size;
 154                 }
 155                 if (!opts->auxtrace_snapshot_size)
 156                         opts->auxtrace_snapshot_size =
 157                                 opts->auxtrace_mmap_pages * (size_t)page_size;
 158                 if (!opts->auxtrace_mmap_pages) {
 159                         size_t sz = opts->auxtrace_snapshot_size;
 160
 161                         sz = round_up(sz, page_size) / page_size;
 162                         opts->auxtrace_mmap_pages = roundup_pow_of_two(sz);
 163                 }
 164                 if (opts->auxtrace_snapshot_size >
 165                                 opts->auxtrace_mmap_pages * (size_t)page_size) {
 166                         pr_err("Snapshot size %zu must not be greater than AUX area tracing mmap size %zu\n",
 167                                opts->auxtrace_snapshot_size,
 168                                opts->auxtrace_mmap_pages * (size_t)page_size);
 169                         return -EINVAL;
 170                 }
 171                 if (!opts->auxtrace_snapshot_size || !opts->auxtrace_mmap_pages) {
 172                         pr_err("Failed to calculate default snapshot size and/or AUX area tracing mmap pages\n");
 173                         return -EINVAL;
 174                 }
 175                 pr_debug2("Intel BTS snapshot size: %zu\n",
 176                           opts->auxtrace_snapshot_size);
 177         }
 178
 179         /* Set default sizes for full trace mode */
 180         if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) {
 181                 if (privileged) {
 182                         opts->auxtrace_mmap_pages = MiB(4) / page_size;
 183                 } else {
 184                         opts->auxtrace_mmap_pages = KiB(128) / page_size;
 185                         if (opts->mmap_pages == UINT_MAX)
 186                                 opts->mmap_pages = KiB(256) / page_size;
 187                 }
 188         }
 189
 190         /* Validate auxtrace_mmap_pages */
 191         if (opts->auxtrace_mmap_pages) {
 192                 size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size;
 193                 size_t min_sz;
 194
 195                 if (opts->auxtrace_snapshot_mode)
 196                         min_sz = KiB(4);
 197                 else
 198                         min_sz = KiB(8);
 199
 200                 if (sz < min_sz || !is_power_of_2(sz)) {
 201                         pr_err("Invalid mmap size for Intel BTS: must be at least %zuKiB and a power of 2\n",
 202                                min_sz / 1024);
 203                         return -EINVAL;
 204                 }
 205         }
 206
 207         if (intel_bts_evsel) {
 208                 /*
 209                  * To obtain the auxtrace buffer file descriptor, the auxtrace event
 210                  * must come first.
 211                  */
 212                 perf_evlist__to_front(evlist, intel_bts_evsel);
 213                 /*
 214                  * In the case of per-cpu mmaps, we need the CPU on the
 215                  * AUX event.
 216                  */
 217                 if (!cpu_map__empty(cpus))
 218                         perf_evsel__set_sample_bit(intel_bts_evsel, CPU);
 219         }
 220
 221         /* Add dummy event to keep tracking */
 222         if (opts->full_auxtrace) {
 223                 struct perf_evsel *tracking_evsel;
 224                 int err;
 225
 226                 err = parse_events(evlist, "dummy:u", NULL);
 227                 if (err)
 228                         return err;
 229
 230                 tracking_evsel = perf_evlist__last(evlist);
 231
 232                 perf_evlist__set_tracking_event(evlist, tracking_evsel);
 233
 234                 tracking_evsel->attr.freq = 0;
 235                 tracking_evsel->attr.sample_period = 1;
 236         }
 237
 238         return 0;
 239 }
 240
 241 static int intel_bts_parse_snapshot_options(struct auxtrace_record *itr,
 242                                             struct record_opts *opts,
 243                                             const char *str)
 244 {
 245         struct intel_bts_recording *btsr =
 246                         container_of(itr, struct intel_bts_recording, itr);
 247         unsigned long long snapshot_size = 0;
 248         char *endptr;
 249
 250         if (str) {
 251                 snapshot_size = strtoull(str, &endptr, 0);
 252                 if (*endptr || snapshot_size > SIZE_MAX)
 253                         return -1;
 254         }
 255
 256         opts->auxtrace_snapshot_mode = true;
 257         opts->auxtrace_snapshot_size = snapshot_size;
 258
 259         btsr->snapshot_size = snapshot_size;
 260
 261         return 0;
 262 }
 263
 264 static u64 intel_bts_reference(struct auxtrace_record *itr __maybe_unused)
 265 {
 266         return rdtsc();
 267 }
 268
 269 static int intel_bts_alloc_snapshot_refs(struct intel_bts_recording *btsr,
 270                                          int idx)
 271 {
 272         const size_t sz = sizeof(struct intel_bts_snapshot_ref);
 273         int cnt = btsr->snapshot_ref_cnt, new_cnt = cnt * 2;
 274         struct intel_bts_snapshot_ref *refs;
 275
 276         if (!new_cnt)
 277                 new_cnt = 16;
 278
 279         while (new_cnt <= idx)
 280                 new_cnt *= 2;
 281
 282         refs = calloc(new_cnt, sz);
 283         if (!refs)
 284                 return -ENOMEM;
 285
 286         memcpy(refs, btsr->snapshot_refs, cnt * sz);
 287
 288         btsr->snapshot_refs = refs;
 289         btsr->snapshot_ref_cnt = new_cnt;
 290
 291         return 0;
 292 }
 293
 294 static void intel_bts_free_snapshot_refs(struct intel_bts_recording *btsr)
 295 {
 296         int i;
 297
 298         for (i = 0; i < btsr->snapshot_ref_cnt; i++)
 299                 zfree(&btsr->snapshot_refs[i].ref_buf);
 300         zfree(&btsr->snapshot_refs);
 301 }
 302
 303 static void intel_bts_recording_free(struct auxtrace_record *itr)
 304 {
 305         struct intel_bts_recording *btsr =
 306                         container_of(itr, struct intel_bts_recording, itr);
 307
 308         intel_bts_free_snapshot_refs(btsr);
 309         free(btsr);
 310 }
 311
 312 static int intel_bts_snapshot_start(struct auxtrace_record *itr)
 313 {
 314         struct intel_bts_recording *btsr =
 315                         container_of(itr, struct intel_bts_recording, itr);
 316         struct perf_evsel *evsel;
 317
 318         evlist__for_each_entry(btsr->evlist, evsel) {
 319                 if (evsel->attr.type == btsr->intel_bts_pmu->type)
 320                         return perf_evsel__disable(evsel);
 321         }
 322         return -EINVAL;
 323 }
 324
 325 static int intel_bts_snapshot_finish(struct auxtrace_record *itr)
 326 {
 327         struct intel_bts_recording *btsr =
 328                         container_of(itr, struct intel_bts_recording, itr);
 329         struct perf_evsel *evsel;
 330
 331         evlist__for_each_entry(btsr->evlist, evsel) {
 332                 if (evsel->attr.type == btsr->intel_bts_pmu->type)
 333                         return perf_evsel__enable(evsel);
 334         }
 335         return -EINVAL;
 336 }
 337
 338 static bool intel_bts_first_wrap(u64 *data, size_t buf_size)
 339 {
 340         int i, a, b;
 341
 342         b = buf_size >> 3;
 343         a = b - 512;
 344         if (a < 0)
 345                 a = 0;
 346
 347         for (i = a; i < b; i++) {
 348                 if (data[i])
 349                         return true;
 350         }
 351
 352         return false;
 353 }
 354
 355 static int intel_bts_find_snapshot(struct auxtrace_record *itr, int idx,
 356                                    struct auxtrace_mmap *mm, unsigned char *data,
 357                                    u64 *head, u64 *old)
 358 {
 359         struct intel_bts_recording *btsr =
 360                         container_of(itr, struct intel_bts_recording, itr);
 361         bool wrapped;
 362         int err;
 363
 364         pr_debug3("%s: mmap index %d old head %zu new head %zu\n",
 365                   __func__, idx, (size_t)*old, (size_t)*head);
 366
 367         if (idx >= btsr->snapshot_ref_cnt) {
 368                 err = intel_bts_alloc_snapshot_refs(btsr, idx);
 369                 if (err)
 370                         goto out_err;
 371         }
 372
 373         wrapped = btsr->snapshot_refs[idx].wrapped;
 374         if (!wrapped && intel_bts_first_wrap((u64 *)data, mm->len)) {
 375                 btsr->snapshot_refs[idx].wrapped = true;
 376                 wrapped = true;
 377         }
 378
 379         /*
 380          * In full trace mode 'head' continually increases.  However in snapshot
 381          * mode 'head' is an offset within the buffer.  Here 'old' and 'head'
 382          * are adjusted to match the full trace case which expects that 'old' is
 383          * always less than 'head'.
 384          */
 385         if (wrapped) {
 386                 *old = *head;
 387                 *head += mm->len;
 388         } else {
 389                 if (mm->mask)
 390                         *old &= mm->mask;
 391                 else
 392                         *old %= mm->len;
 393                 if (*old > *head)
 394                         *head += mm->len;
 395         }
 396
 397         pr_debug3("%s: wrap-around %sdetected, adjusted old head %zu adjusted new head %zu\n",
 398                   __func__, wrapped ? "" : "not ", (size_t)*old, (size_t)*head);
 399
 400         return 0;
 401
 402 out_err:
 403         pr_err("%s: failed, error %d\n", __func__, err);
 404         return err;
 405 }
 406
 407 static int intel_bts_read_finish(struct auxtrace_record *itr, int idx)
 408 {
 409         struct intel_bts_recording *btsr =
 410                         container_of(itr, struct intel_bts_recording, itr);
 411         struct perf_evsel *evsel;
 412
 413         evlist__for_each_entry(btsr->evlist, evsel) {
 414                 if (evsel->attr.type == btsr->intel_bts_pmu->type)
 415                         return perf_evlist__enable_event_idx(btsr->evlist,
 416                                                              evsel, idx);
 417         }
 418         return -EINVAL;
 419 }
 420
 421 struct auxtrace_record *intel_bts_recording_init(int *err)
 422 {
 423         struct perf_pmu *intel_bts_pmu = perf_pmu__find(INTEL_BTS_PMU_NAME);
 424         struct intel_bts_recording *btsr;
 425
 426         if (!intel_bts_pmu)
 427                 return NULL;
 428
 429         if (setenv("JITDUMP_USE_ARCH_TIMESTAMP", "1", 1)) {
 430                 *err = -errno;
 431                 return NULL;
 432         }
 433
 434         btsr = zalloc(sizeof(struct intel_bts_recording));
 435         if (!btsr) {
 436                 *err = -ENOMEM;
 437                 return NULL;
 438         }
 439
 440         btsr->intel_bts_pmu = intel_bts_pmu;
 441         btsr->itr.recording_options = intel_bts_recording_options;
 442         btsr->itr.info_priv_size = intel_bts_info_priv_size;
 443         btsr->itr.info_fill = intel_bts_info_fill;
 444         btsr->itr.free = intel_bts_recording_free;
 445         btsr->itr.snapshot_start = intel_bts_snapshot_start;
 446         btsr->itr.snapshot_finish = intel_bts_snapshot_finish;
 447         btsr->itr.find_snapshot = intel_bts_find_snapshot;
 448         btsr->itr.parse_snapshot_options = intel_bts_parse_snapshot_options;
 449         btsr->itr.reference = intel_bts_reference;
 450         btsr->itr.read_finish = intel_bts_read_finish;
 451         btsr->itr.alignment = sizeof(struct branch);
 452         return &btsr->itr;
 453 }