92ae5de0bbacadc4764e2fd53dc5548ec6adb749
[sfrench/cifs-2.6.git] / arch / powerpc / perf / imc-pmu.c
1 /*
2  * In-Memory Collection (IMC) Performance Monitor counter support.
3  *
4  * Copyright (C) 2017 Madhavan Srinivasan, IBM Corporation.
5  *           (C) 2017 Anju T Sudhakar, IBM Corporation.
6  *           (C) 2017 Hemant K Shaw, IBM Corporation.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version
11  * 2 of the License, or later version.
12  */
13 #include <linux/perf_event.h>
14 #include <linux/slab.h>
15 #include <asm/opal.h>
16 #include <asm/imc-pmu.h>
17 #include <asm/cputhreads.h>
18 #include <asm/smp.h>
19 #include <linux/string.h>
20
21 /* Nest IMC data structures and variables */
22
23 /*
24  * Used to avoid races in counting the nest-pmu units during hotplug
25  * register and unregister
26  */
27 static DEFINE_MUTEX(nest_init_lock);
28 static DEFINE_PER_CPU(struct imc_pmu_ref *, local_nest_imc_refc);
29 static struct imc_pmu *per_nest_pmu_arr[IMC_MAX_PMUS];
30 static cpumask_t nest_imc_cpumask;
31 struct imc_pmu_ref *nest_imc_refc;
32 static int nest_pmus;
33
34 /* Core IMC data structures and variables */
35
36 static cpumask_t core_imc_cpumask;
37 struct imc_pmu_ref *core_imc_refc;
38 static struct imc_pmu *core_imc_pmu;
39
40 /* Thread IMC data structures and variables */
41
42 static DEFINE_PER_CPU(u64 *, thread_imc_mem);
43 static struct imc_pmu *thread_imc_pmu;
44 static int thread_imc_mem_size;
45
46 struct imc_pmu *imc_event_to_pmu(struct perf_event *event)
47 {
48         return container_of(event->pmu, struct imc_pmu, pmu);
49 }
50
51 PMU_FORMAT_ATTR(event, "config:0-40");
52 PMU_FORMAT_ATTR(offset, "config:0-31");
53 PMU_FORMAT_ATTR(rvalue, "config:32");
54 PMU_FORMAT_ATTR(mode, "config:33-40");
55 static struct attribute *imc_format_attrs[] = {
56         &format_attr_event.attr,
57         &format_attr_offset.attr,
58         &format_attr_rvalue.attr,
59         &format_attr_mode.attr,
60         NULL,
61 };
62
63 static struct attribute_group imc_format_group = {
64         .name = "format",
65         .attrs = imc_format_attrs,
66 };
67
68 /* Get the cpumask printed to a buffer "buf" */
69 static ssize_t imc_pmu_cpumask_get_attr(struct device *dev,
70                                         struct device_attribute *attr,
71                                         char *buf)
72 {
73         struct pmu *pmu = dev_get_drvdata(dev);
74         struct imc_pmu *imc_pmu = container_of(pmu, struct imc_pmu, pmu);
75         cpumask_t *active_mask;
76
77         switch(imc_pmu->domain){
78         case IMC_DOMAIN_NEST:
79                 active_mask = &nest_imc_cpumask;
80                 break;
81         case IMC_DOMAIN_CORE:
82                 active_mask = &core_imc_cpumask;
83                 break;
84         default:
85                 return 0;
86         }
87
88         return cpumap_print_to_pagebuf(true, buf, active_mask);
89 }
90
91 static DEVICE_ATTR(cpumask, S_IRUGO, imc_pmu_cpumask_get_attr, NULL);
92
93 static struct attribute *imc_pmu_cpumask_attrs[] = {
94         &dev_attr_cpumask.attr,
95         NULL,
96 };
97
98 static struct attribute_group imc_pmu_cpumask_attr_group = {
99         .attrs = imc_pmu_cpumask_attrs,
100 };
101
102 /* device_str_attr_create : Populate event "name" and string "str" in attribute */
103 static struct attribute *device_str_attr_create(const char *name, const char *str)
104 {
105         struct perf_pmu_events_attr *attr;
106
107         attr = kzalloc(sizeof(*attr), GFP_KERNEL);
108         if (!attr)
109                 return NULL;
110         sysfs_attr_init(&attr->attr.attr);
111
112         attr->event_str = str;
113         attr->attr.attr.name = name;
114         attr->attr.attr.mode = 0444;
115         attr->attr.show = perf_event_sysfs_show;
116
117         return &attr->attr.attr;
118 }
119
120 struct imc_events *imc_parse_event(struct device_node *np, const char *scale,
121                                   const char *unit, const char *prefix, u32 base)
122 {
123         struct imc_events *event;
124         const char *s;
125         u32 reg;
126
127         event = kzalloc(sizeof(struct imc_events), GFP_KERNEL);
128         if (!event)
129                 return NULL;
130
131         if (of_property_read_u32(np, "reg", &reg))
132                 goto error;
133         /* Add the base_reg value to the "reg" */
134         event->value = base + reg;
135
136         if (of_property_read_string(np, "event-name", &s))
137                 goto error;
138
139         event->name = kasprintf(GFP_KERNEL, "%s%s", prefix, s);
140         if (!event->name)
141                 goto error;
142
143         if (of_property_read_string(np, "scale", &s))
144                 s = scale;
145
146         if (s) {
147                 event->scale = kstrdup(s, GFP_KERNEL);
148                 if (!event->scale)
149                         goto error;
150         }
151
152         if (of_property_read_string(np, "unit", &s))
153                 s = unit;
154
155         if (s) {
156                 event->unit = kstrdup(s, GFP_KERNEL);
157                 if (!event->unit)
158                         goto error;
159         }
160
161         return event;
162 error:
163         kfree(event->unit);
164         kfree(event->scale);
165         kfree(event->name);
166         kfree(event);
167
168         return NULL;
169 }
170
171 /*
172  * update_events_in_group: Update the "events" information in an attr_group
173  *                         and assign the attr_group to the pmu "pmu".
174  */
175 static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
176 {
177         struct attribute_group *attr_group;
178         struct attribute **attrs, *dev_str;
179         struct device_node *np, *pmu_events;
180         struct imc_events *ev;
181         u32 handle, base_reg;
182         int i=0, j=0, ct;
183         const char *prefix, *g_scale, *g_unit;
184         const char *ev_val_str, *ev_scale_str, *ev_unit_str;
185
186         if (!of_property_read_u32(node, "events", &handle))
187                 pmu_events = of_find_node_by_phandle(handle);
188         else
189                 return 0;
190
191         /* Did not find any node with a given phandle */
192         if (!pmu_events)
193                 return 0;
194
195         /* Get a count of number of child nodes */
196         ct = of_get_child_count(pmu_events);
197
198         /* Get the event prefix */
199         if (of_property_read_string(node, "events-prefix", &prefix))
200                 return 0;
201
202         /* Get a global unit and scale data if available */
203         if (of_property_read_string(node, "scale", &g_scale))
204                 g_scale = NULL;
205
206         if (of_property_read_string(node, "unit", &g_unit))
207                 g_unit = NULL;
208
209         /* "reg" property gives out the base offset of the counters data */
210         of_property_read_u32(node, "reg", &base_reg);
211
212         /* Allocate memory for the events */
213         pmu->events = kcalloc(ct, sizeof(struct imc_events), GFP_KERNEL);
214         if (!pmu->events)
215                 return -ENOMEM;
216
217         ct = 0;
218         /* Parse the events and update the struct */
219         for_each_child_of_node(pmu_events, np) {
220                 ev = imc_parse_event(np, g_scale, g_unit, prefix, base_reg);
221                 if (ev)
222                         pmu->events[ct++] = ev;
223         }
224
225         /* Allocate memory for attribute group */
226         attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL);
227         if (!attr_group)
228                 return -ENOMEM;
229
230         /*
231          * Allocate memory for attributes.
232          * Since we have count of events for this pmu, we also allocate
233          * memory for the scale and unit attribute for now.
234          * "ct" has the total event structs added from the events-parent node.
235          * So allocate three times the "ct" (this includes event, event_scale and
236          * event_unit).
237          */
238         attrs = kcalloc(((ct * 3) + 1), sizeof(struct attribute *), GFP_KERNEL);
239         if (!attrs) {
240                 kfree(attr_group);
241                 kfree(pmu->events);
242                 return -ENOMEM;
243         }
244
245         attr_group->name = "events";
246         attr_group->attrs = attrs;
247         do {
248                 ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i]->value);
249                 dev_str = device_str_attr_create(pmu->events[i]->name, ev_val_str);
250                 if (!dev_str)
251                         continue;
252
253                 attrs[j++] = dev_str;
254                 if (pmu->events[i]->scale) {
255                         ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale",pmu->events[i]->name);
256                         dev_str = device_str_attr_create(ev_scale_str, pmu->events[i]->scale);
257                         if (!dev_str)
258                                 continue;
259
260                         attrs[j++] = dev_str;
261                 }
262
263                 if (pmu->events[i]->unit) {
264                         ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit",pmu->events[i]->name);
265                         dev_str = device_str_attr_create(ev_unit_str, pmu->events[i]->unit);
266                         if (!dev_str)
267                                 continue;
268
269                         attrs[j++] = dev_str;
270                 }
271         } while (++i < ct);
272
273         /* Save the event attribute */
274         pmu->attr_groups[IMC_EVENT_ATTR] = attr_group;
275
276         kfree(pmu->events);
277         return 0;
278 }
279
280 /* get_nest_pmu_ref: Return the imc_pmu_ref struct for the given node */
281 static struct imc_pmu_ref *get_nest_pmu_ref(int cpu)
282 {
283         return per_cpu(local_nest_imc_refc, cpu);
284 }
285
286 static void nest_change_cpu_context(int old_cpu, int new_cpu)
287 {
288         struct imc_pmu **pn = per_nest_pmu_arr;
289         int i;
290
291         if (old_cpu < 0 || new_cpu < 0)
292                 return;
293
294         for (i = 0; *pn && i < IMC_MAX_PMUS; i++, pn++)
295                 perf_pmu_migrate_context(&(*pn)->pmu, old_cpu, new_cpu);
296 }
297
298 static int ppc_nest_imc_cpu_offline(unsigned int cpu)
299 {
300         int nid, target = -1;
301         const struct cpumask *l_cpumask;
302         struct imc_pmu_ref *ref;
303
304         /*
305          * Check in the designated list for this cpu. Dont bother
306          * if not one of them.
307          */
308         if (!cpumask_test_and_clear_cpu(cpu, &nest_imc_cpumask))
309                 return 0;
310
311         /*
312          * Now that this cpu is one of the designated,
313          * find a next cpu a) which is online and b) in same chip.
314          */
315         nid = cpu_to_node(cpu);
316         l_cpumask = cpumask_of_node(nid);
317         target = cpumask_any_but(l_cpumask, cpu);
318
319         /*
320          * Update the cpumask with the target cpu and
321          * migrate the context if needed
322          */
323         if (target >= 0 && target < nr_cpu_ids) {
324                 cpumask_set_cpu(target, &nest_imc_cpumask);
325                 nest_change_cpu_context(cpu, target);
326         } else {
327                 opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
328                                        get_hard_smp_processor_id(cpu));
329                 /*
330                  * If this is the last cpu in this chip then, skip the reference
331                  * count mutex lock and make the reference count on this chip zero.
332                  */
333                 ref = get_nest_pmu_ref(cpu);
334                 if (!ref)
335                         return -EINVAL;
336
337                 ref->refc = 0;
338         }
339         return 0;
340 }
341
342 static int ppc_nest_imc_cpu_online(unsigned int cpu)
343 {
344         const struct cpumask *l_cpumask;
345         static struct cpumask tmp_mask;
346         int res;
347
348         /* Get the cpumask of this node */
349         l_cpumask = cpumask_of_node(cpu_to_node(cpu));
350
351         /*
352          * If this is not the first online CPU on this node, then
353          * just return.
354          */
355         if (cpumask_and(&tmp_mask, l_cpumask, &nest_imc_cpumask))
356                 return 0;
357
358         /*
359          * If this is the first online cpu on this node
360          * disable the nest counters by making an OPAL call.
361          */
362         res = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
363                                      get_hard_smp_processor_id(cpu));
364         if (res)
365                 return res;
366
367         /* Make this CPU the designated target for counter collection */
368         cpumask_set_cpu(cpu, &nest_imc_cpumask);
369         return 0;
370 }
371
372 static int nest_pmu_cpumask_init(void)
373 {
374         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE,
375                                  "perf/powerpc/imc:online",
376                                  ppc_nest_imc_cpu_online,
377                                  ppc_nest_imc_cpu_offline);
378 }
379
380 static void nest_imc_counters_release(struct perf_event *event)
381 {
382         int rc, node_id;
383         struct imc_pmu_ref *ref;
384
385         if (event->cpu < 0)
386                 return;
387
388         node_id = cpu_to_node(event->cpu);
389
390         /*
391          * See if we need to disable the nest PMU.
392          * If no events are currently in use, then we have to take a
393          * mutex to ensure that we don't race with another task doing
394          * enable or disable the nest counters.
395          */
396         ref = get_nest_pmu_ref(event->cpu);
397         if (!ref)
398                 return;
399
400         /* Take the mutex lock for this node and then decrement the reference count */
401         mutex_lock(&ref->lock);
402         if (ref->refc == 0) {
403                 /*
404                  * The scenario where this is true is, when perf session is
405                  * started, followed by offlining of all cpus in a given node.
406                  *
407                  * In the cpuhotplug offline path, ppc_nest_imc_cpu_offline()
408                  * function set the ref->count to zero, if the cpu which is
409                  * about to offline is the last cpu in a given node and make
410                  * an OPAL call to disable the engine in that node.
411                  *
412                  */
413                 mutex_unlock(&ref->lock);
414                 return;
415         }
416         ref->refc--;
417         if (ref->refc == 0) {
418                 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
419                                             get_hard_smp_processor_id(event->cpu));
420                 if (rc) {
421                         mutex_unlock(&ref->lock);
422                         pr_err("nest-imc: Unable to stop the counters for core %d\n", node_id);
423                         return;
424                 }
425         } else if (ref->refc < 0) {
426                 WARN(1, "nest-imc: Invalid event reference count\n");
427                 ref->refc = 0;
428         }
429         mutex_unlock(&ref->lock);
430 }
431
432 static int nest_imc_event_init(struct perf_event *event)
433 {
434         int chip_id, rc, node_id;
435         u32 l_config, config = event->attr.config;
436         struct imc_mem_info *pcni;
437         struct imc_pmu *pmu;
438         struct imc_pmu_ref *ref;
439         bool flag = false;
440
441         if (event->attr.type != event->pmu->type)
442                 return -ENOENT;
443
444         /* Sampling not supported */
445         if (event->hw.sample_period)
446                 return -EINVAL;
447
448         /* unsupported modes and filters */
449         if (event->attr.exclude_user   ||
450             event->attr.exclude_kernel ||
451             event->attr.exclude_hv     ||
452             event->attr.exclude_idle   ||
453             event->attr.exclude_host   ||
454             event->attr.exclude_guest)
455                 return -EINVAL;
456
457         if (event->cpu < 0)
458                 return -EINVAL;
459
460         pmu = imc_event_to_pmu(event);
461
462         /* Sanity check for config (event offset) */
463         if ((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size)
464                 return -EINVAL;
465
466         /*
467          * Nest HW counter memory resides in a per-chip reserve-memory (HOMER).
468          * Get the base memory addresss for this cpu.
469          */
470         chip_id = topology_physical_package_id(event->cpu);
471         pcni = pmu->mem_info;
472         do {
473                 if (pcni->id == chip_id) {
474                         flag = true;
475                         break;
476                 }
477                 pcni++;
478         } while (pcni);
479
480         if (!flag)
481                 return -ENODEV;
482
483         /*
484          * Add the event offset to the base address.
485          */
486         l_config = config & IMC_EVENT_OFFSET_MASK;
487         event->hw.event_base = (u64)pcni->vbase + l_config;
488         node_id = cpu_to_node(event->cpu);
489
490         /*
491          * Get the imc_pmu_ref struct for this node.
492          * Take the mutex lock and then increment the count of nest pmu events
493          * inited.
494          */
495         ref = get_nest_pmu_ref(event->cpu);
496         if (!ref)
497                 return -EINVAL;
498
499         mutex_lock(&ref->lock);
500         if (ref->refc == 0) {
501                 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_NEST,
502                                              get_hard_smp_processor_id(event->cpu));
503                 if (rc) {
504                         mutex_unlock(&ref->lock);
505                         pr_err("nest-imc: Unable to start the counters for node %d\n",
506                                                                         node_id);
507                         return rc;
508                 }
509         }
510         ++ref->refc;
511         mutex_unlock(&ref->lock);
512
513         event->destroy = nest_imc_counters_release;
514         return 0;
515 }
516
517 /*
518  * core_imc_mem_init : Initializes memory for the current core.
519  *
520  * Uses alloc_pages_node() and uses the returned address as an argument to
521  * an opal call to configure the pdbar. The address sent as an argument is
522  * converted to physical address before the opal call is made. This is the
523  * base address at which the core imc counters are populated.
524  */
525 static int core_imc_mem_init(int cpu, int size)
526 {
527         int phys_id, rc = 0, core_id = (cpu / threads_per_core);
528         struct imc_mem_info *mem_info;
529
530         /*
531          * alloc_pages_node() will allocate memory for core in the
532          * local node only.
533          */
534         phys_id = topology_physical_package_id(cpu);
535         mem_info = &core_imc_pmu->mem_info[core_id];
536         mem_info->id = core_id;
537
538         /* We need only vbase for core counters */
539         mem_info->vbase = page_address(alloc_pages_node(phys_id,
540                                           GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
541                                           __GFP_NOWARN, get_order(size)));
542         if (!mem_info->vbase)
543                 return -ENOMEM;
544
545         /* Init the mutex */
546         core_imc_refc[core_id].id = core_id;
547         mutex_init(&core_imc_refc[core_id].lock);
548
549         rc = opal_imc_counters_init(OPAL_IMC_COUNTERS_CORE,
550                                 __pa((void *)mem_info->vbase),
551                                 get_hard_smp_processor_id(cpu));
552         if (rc) {
553                 free_pages((u64)mem_info->vbase, get_order(size));
554                 mem_info->vbase = NULL;
555         }
556
557         return rc;
558 }
559
560 static bool is_core_imc_mem_inited(int cpu)
561 {
562         struct imc_mem_info *mem_info;
563         int core_id = (cpu / threads_per_core);
564
565         mem_info = &core_imc_pmu->mem_info[core_id];
566         if (!mem_info->vbase)
567                 return false;
568
569         return true;
570 }
571
572 static int ppc_core_imc_cpu_online(unsigned int cpu)
573 {
574         const struct cpumask *l_cpumask;
575         static struct cpumask tmp_mask;
576         int ret = 0;
577
578         /* Get the cpumask for this core */
579         l_cpumask = cpu_sibling_mask(cpu);
580
581         /* If a cpu for this core is already set, then, don't do anything */
582         if (cpumask_and(&tmp_mask, l_cpumask, &core_imc_cpumask))
583                 return 0;
584
585         if (!is_core_imc_mem_inited(cpu)) {
586                 ret = core_imc_mem_init(cpu, core_imc_pmu->counter_mem_size);
587                 if (ret) {
588                         pr_info("core_imc memory allocation for cpu %d failed\n", cpu);
589                         return ret;
590                 }
591         }
592
593         /* set the cpu in the mask */
594         cpumask_set_cpu(cpu, &core_imc_cpumask);
595         return 0;
596 }
597
598 static int ppc_core_imc_cpu_offline(unsigned int cpu)
599 {
600         unsigned int ncpu, core_id;
601         struct imc_pmu_ref *ref;
602
603         /*
604          * clear this cpu out of the mask, if not present in the mask,
605          * don't bother doing anything.
606          */
607         if (!cpumask_test_and_clear_cpu(cpu, &core_imc_cpumask))
608                 return 0;
609
610         /* Find any online cpu in that core except the current "cpu" */
611         ncpu = cpumask_any_but(cpu_sibling_mask(cpu), cpu);
612
613         if (ncpu >= 0 && ncpu < nr_cpu_ids) {
614                 cpumask_set_cpu(ncpu, &core_imc_cpumask);
615                 perf_pmu_migrate_context(&core_imc_pmu->pmu, cpu, ncpu);
616         } else {
617                 /*
618                  * If this is the last cpu in this core then, skip taking refernce
619                  * count mutex lock for this core and directly zero "refc" for
620                  * this core.
621                  */
622                 opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
623                                        get_hard_smp_processor_id(cpu));
624                 core_id = cpu / threads_per_core;
625                 ref = &core_imc_refc[core_id];
626                 if (!ref)
627                         return -EINVAL;
628
629                 ref->refc = 0;
630         }
631         return 0;
632 }
633
634 static int core_imc_pmu_cpumask_init(void)
635 {
636         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
637                                  "perf/powerpc/imc_core:online",
638                                  ppc_core_imc_cpu_online,
639                                  ppc_core_imc_cpu_offline);
640 }
641
642 static void core_imc_counters_release(struct perf_event *event)
643 {
644         int rc, core_id;
645         struct imc_pmu_ref *ref;
646
647         if (event->cpu < 0)
648                 return;
649         /*
650          * See if we need to disable the IMC PMU.
651          * If no events are currently in use, then we have to take a
652          * mutex to ensure that we don't race with another task doing
653          * enable or disable the core counters.
654          */
655         core_id = event->cpu / threads_per_core;
656
657         /* Take the mutex lock and decrement the refernce count for this core */
658         ref = &core_imc_refc[core_id];
659         if (!ref)
660                 return;
661
662         mutex_lock(&ref->lock);
663         if (ref->refc == 0) {
664                 /*
665                  * The scenario where this is true is, when perf session is
666                  * started, followed by offlining of all cpus in a given core.
667                  *
668                  * In the cpuhotplug offline path, ppc_core_imc_cpu_offline()
669                  * function set the ref->count to zero, if the cpu which is
670                  * about to offline is the last cpu in a given core and make
671                  * an OPAL call to disable the engine in that core.
672                  *
673                  */
674                 mutex_unlock(&ref->lock);
675                 return;
676         }
677         ref->refc--;
678         if (ref->refc == 0) {
679                 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
680                                             get_hard_smp_processor_id(event->cpu));
681                 if (rc) {
682                         mutex_unlock(&ref->lock);
683                         pr_err("IMC: Unable to stop the counters for core %d\n", core_id);
684                         return;
685                 }
686         } else if (ref->refc < 0) {
687                 WARN(1, "core-imc: Invalid event reference count\n");
688                 ref->refc = 0;
689         }
690         mutex_unlock(&ref->lock);
691 }
692
693 static int core_imc_event_init(struct perf_event *event)
694 {
695         int core_id, rc;
696         u64 config = event->attr.config;
697         struct imc_mem_info *pcmi;
698         struct imc_pmu *pmu;
699         struct imc_pmu_ref *ref;
700
701         if (event->attr.type != event->pmu->type)
702                 return -ENOENT;
703
704         /* Sampling not supported */
705         if (event->hw.sample_period)
706                 return -EINVAL;
707
708         /* unsupported modes and filters */
709         if (event->attr.exclude_user   ||
710             event->attr.exclude_kernel ||
711             event->attr.exclude_hv     ||
712             event->attr.exclude_idle   ||
713             event->attr.exclude_host   ||
714             event->attr.exclude_guest)
715                 return -EINVAL;
716
717         if (event->cpu < 0)
718                 return -EINVAL;
719
720         event->hw.idx = -1;
721         pmu = imc_event_to_pmu(event);
722
723         /* Sanity check for config (event offset) */
724         if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
725                 return -EINVAL;
726
727         if (!is_core_imc_mem_inited(event->cpu))
728                 return -ENODEV;
729
730         core_id = event->cpu / threads_per_core;
731         pcmi = &core_imc_pmu->mem_info[core_id];
732         if ((!pcmi->vbase))
733                 return -ENODEV;
734
735         /* Get the core_imc mutex for this core */
736         ref = &core_imc_refc[core_id];
737         if (!ref)
738                 return -EINVAL;
739
740         /*
741          * Core pmu units are enabled only when it is used.
742          * See if this is triggered for the first time.
743          * If yes, take the mutex lock and enable the core counters.
744          * If not, just increment the count in core_imc_refc struct.
745          */
746         mutex_lock(&ref->lock);
747         if (ref->refc == 0) {
748                 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
749                                              get_hard_smp_processor_id(event->cpu));
750                 if (rc) {
751                         mutex_unlock(&ref->lock);
752                         pr_err("core-imc: Unable to start the counters for core %d\n",
753                                                                         core_id);
754                         return rc;
755                 }
756         }
757         ++ref->refc;
758         mutex_unlock(&ref->lock);
759
760         event->hw.event_base = (u64)pcmi->vbase + (config & IMC_EVENT_OFFSET_MASK);
761         event->destroy = core_imc_counters_release;
762         return 0;
763 }
764
765 /*
766  * Allocates a page of memory for each of the online cpus, and write the
767  * physical base address of that page to the LDBAR for that cpu.
768  *
769  * LDBAR Register Layout:
770  *
771  *  0          4         8         12        16        20        24        28
772  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
773  *   | |       [   ]    [                   Counter Address [8:50]
774  *   | * Mode    |
775  *   |           * PB Scope
776  *   * Enable/Disable
777  *
778  *  32        36        40        44        48        52        56        60
779  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
780  *           Counter Address [8:50]              ]
781  *
782  */
783 static int thread_imc_mem_alloc(int cpu_id, int size)
784 {
785         u64 ldbar_value, *local_mem = per_cpu(thread_imc_mem, cpu_id);
786         int phys_id = topology_physical_package_id(cpu_id);
787
788         if (!local_mem) {
789                 /*
790                  * This case could happen only once at start, since we dont
791                  * free the memory in cpu offline path.
792                  */
793                 local_mem = page_address(alloc_pages_node(phys_id,
794                                   GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
795                                   __GFP_NOWARN, get_order(size)));
796                 if (!local_mem)
797                         return -ENOMEM;
798
799                 per_cpu(thread_imc_mem, cpu_id) = local_mem;
800         }
801
802         ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) | THREAD_IMC_ENABLE;
803
804         mtspr(SPRN_LDBAR, ldbar_value);
805         return 0;
806 }
807
808 static int ppc_thread_imc_cpu_online(unsigned int cpu)
809 {
810         return thread_imc_mem_alloc(cpu, thread_imc_mem_size);
811 }
812
813 static int ppc_thread_imc_cpu_offline(unsigned int cpu)
814 {
815         mtspr(SPRN_LDBAR, 0);
816         return 0;
817 }
818
819 static int thread_imc_cpu_init(void)
820 {
821         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
822                           "perf/powerpc/imc_thread:online",
823                           ppc_thread_imc_cpu_online,
824                           ppc_thread_imc_cpu_offline);
825 }
826
827 void thread_imc_pmu_sched_task(struct perf_event_context *ctx,
828                                       bool sched_in)
829 {
830         int core_id;
831         struct imc_pmu_ref *ref;
832
833         if (!is_core_imc_mem_inited(smp_processor_id()))
834                 return;
835
836         core_id = smp_processor_id() / threads_per_core;
837         /*
838          * imc pmus are enabled only when it is used.
839          * See if this is triggered for the first time.
840          * If yes, take the mutex lock and enable the counters.
841          * If not, just increment the count in ref count struct.
842          */
843         ref = &core_imc_refc[core_id];
844         if (!ref)
845                 return;
846
847         if (sched_in) {
848                 mutex_lock(&ref->lock);
849                 if (ref->refc == 0) {
850                         if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
851                              get_hard_smp_processor_id(smp_processor_id()))) {
852                                 mutex_unlock(&ref->lock);
853                                 pr_err("thread-imc: Unable to start the counter\
854                                                         for core %d\n", core_id);
855                                 return;
856                         }
857                 }
858                 ++ref->refc;
859                 mutex_unlock(&ref->lock);
860         } else {
861                 mutex_lock(&ref->lock);
862                 ref->refc--;
863                 if (ref->refc == 0) {
864                         if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
865                             get_hard_smp_processor_id(smp_processor_id()))) {
866                                 mutex_unlock(&ref->lock);
867                                 pr_err("thread-imc: Unable to stop the counters\
868                                                         for core %d\n", core_id);
869                                 return;
870                         }
871                 } else if (ref->refc < 0) {
872                         ref->refc = 0;
873                 }
874                 mutex_unlock(&ref->lock);
875         }
876
877         return;
878 }
879
880 static int thread_imc_event_init(struct perf_event *event)
881 {
882         u32 config = event->attr.config;
883         struct task_struct *target;
884         struct imc_pmu *pmu;
885
886         if (event->attr.type != event->pmu->type)
887                 return -ENOENT;
888
889         /* Sampling not supported */
890         if (event->hw.sample_period)
891                 return -EINVAL;
892
893         event->hw.idx = -1;
894         pmu = imc_event_to_pmu(event);
895
896         /* Sanity check for config offset */
897         if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
898                 return -EINVAL;
899
900         target = event->hw.target;
901         if (!target)
902                 return -EINVAL;
903
904         event->pmu->task_ctx_nr = perf_sw_context;
905         return 0;
906 }
907
908 static bool is_thread_imc_pmu(struct perf_event *event)
909 {
910         if (!strncmp(event->pmu->name, "thread_imc", strlen("thread_imc")))
911                 return true;
912
913         return false;
914 }
915
916 static u64 * get_event_base_addr(struct perf_event *event)
917 {
918         u64 addr;
919
920         if (is_thread_imc_pmu(event)) {
921                 addr = (u64)per_cpu(thread_imc_mem, smp_processor_id());
922                 return (u64 *)(addr + (event->attr.config & IMC_EVENT_OFFSET_MASK));
923         }
924
925         return (u64 *)event->hw.event_base;
926 }
927
928 static void thread_imc_pmu_start_txn(struct pmu *pmu,
929                                      unsigned int txn_flags)
930 {
931         if (txn_flags & ~PERF_PMU_TXN_ADD)
932                 return;
933         perf_pmu_disable(pmu);
934 }
935
936 static void thread_imc_pmu_cancel_txn(struct pmu *pmu)
937 {
938         perf_pmu_enable(pmu);
939 }
940
941 static int thread_imc_pmu_commit_txn(struct pmu *pmu)
942 {
943         perf_pmu_enable(pmu);
944         return 0;
945 }
946
947 static u64 imc_read_counter(struct perf_event *event)
948 {
949         u64 *addr, data;
950
951         /*
952          * In-Memory Collection (IMC) counters are free flowing counters.
953          * So we take a snapshot of the counter value on enable and save it
954          * to calculate the delta at later stage to present the event counter
955          * value.
956          */
957         addr = get_event_base_addr(event);
958         data = be64_to_cpu(READ_ONCE(*addr));
959         local64_set(&event->hw.prev_count, data);
960
961         return data;
962 }
963
964 static void imc_event_update(struct perf_event *event)
965 {
966         u64 counter_prev, counter_new, final_count;
967
968         counter_prev = local64_read(&event->hw.prev_count);
969         counter_new = imc_read_counter(event);
970         final_count = counter_new - counter_prev;
971
972         /* Update the delta to the event count */
973         local64_add(final_count, &event->count);
974 }
975
976 static void imc_event_start(struct perf_event *event, int flags)
977 {
978         /*
979          * In Memory Counters are free flowing counters. HW or the microcode
980          * keeps adding to the counter offset in memory. To get event
981          * counter value, we snapshot the value here and we calculate
982          * delta at later point.
983          */
984         imc_read_counter(event);
985 }
986
987 static void imc_event_stop(struct perf_event *event, int flags)
988 {
989         /*
990          * Take a snapshot and calculate the delta and update
991          * the event counter values.
992          */
993         imc_event_update(event);
994 }
995
996 static int imc_event_add(struct perf_event *event, int flags)
997 {
998         if (flags & PERF_EF_START)
999                 imc_event_start(event, flags);
1000
1001         return 0;
1002 }
1003
1004 static int thread_imc_event_add(struct perf_event *event, int flags)
1005 {
1006         if (flags & PERF_EF_START)
1007                 imc_event_start(event, flags);
1008
1009         /* Enable the sched_task to start the engine */
1010         perf_sched_cb_inc(event->ctx->pmu);
1011         return 0;
1012 }
1013
1014 static void thread_imc_event_del(struct perf_event *event, int flags)
1015 {
1016         /*
1017          * Take a snapshot and calculate the delta and update
1018          * the event counter values.
1019          */
1020         imc_event_update(event);
1021         perf_sched_cb_dec(event->ctx->pmu);
1022 }
1023
1024 /* update_pmu_ops : Populate the appropriate operations for "pmu" */
1025 static int update_pmu_ops(struct imc_pmu *pmu)
1026 {
1027         pmu->pmu.task_ctx_nr = perf_invalid_context;
1028         pmu->pmu.add = imc_event_add;
1029         pmu->pmu.del = imc_event_stop;
1030         pmu->pmu.start = imc_event_start;
1031         pmu->pmu.stop = imc_event_stop;
1032         pmu->pmu.read = imc_event_update;
1033         pmu->pmu.attr_groups = pmu->attr_groups;
1034         pmu->attr_groups[IMC_FORMAT_ATTR] = &imc_format_group;
1035
1036         switch (pmu->domain) {
1037         case IMC_DOMAIN_NEST:
1038                 pmu->pmu.event_init = nest_imc_event_init;
1039                 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1040                 break;
1041         case IMC_DOMAIN_CORE:
1042                 pmu->pmu.event_init = core_imc_event_init;
1043                 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1044                 break;
1045         case IMC_DOMAIN_THREAD:
1046                 pmu->pmu.event_init = thread_imc_event_init;
1047                 pmu->pmu.sched_task = thread_imc_pmu_sched_task;
1048                 pmu->pmu.add = thread_imc_event_add;
1049                 pmu->pmu.del = thread_imc_event_del;
1050                 pmu->pmu.start_txn = thread_imc_pmu_start_txn;
1051                 pmu->pmu.cancel_txn = thread_imc_pmu_cancel_txn;
1052                 pmu->pmu.commit_txn = thread_imc_pmu_commit_txn;
1053                 break;
1054         default:
1055                 break;
1056         }
1057
1058         return 0;
1059 }
1060
1061 /* init_nest_pmu_ref: Initialize the imc_pmu_ref struct for all the nodes */
1062 static int init_nest_pmu_ref(void)
1063 {
1064         int nid, i, cpu;
1065
1066         nest_imc_refc = kcalloc(num_possible_nodes(), sizeof(*nest_imc_refc),
1067                                                                 GFP_KERNEL);
1068
1069         if (!nest_imc_refc)
1070                 return -ENOMEM;
1071
1072         i = 0;
1073         for_each_node(nid) {
1074                 /*
1075                  * Mutex lock to avoid races while tracking the number of
1076                  * sessions using the chip's nest pmu units.
1077                  */
1078                 mutex_init(&nest_imc_refc[i].lock);
1079
1080                 /*
1081                  * Loop to init the "id" with the node_id. Variable "i" initialized to
1082                  * 0 and will be used as index to the array. "i" will not go off the
1083                  * end of the array since the "for_each_node" loops for "N_POSSIBLE"
1084                  * nodes only.
1085                  */
1086                 nest_imc_refc[i++].id = nid;
1087         }
1088
1089         /*
1090          * Loop to init the per_cpu "local_nest_imc_refc" with the proper
1091          * "nest_imc_refc" index. This makes get_nest_pmu_ref() alot simple.
1092          */
1093         for_each_possible_cpu(cpu) {
1094                 nid = cpu_to_node(cpu);
1095                 for (i = 0; i < num_possible_nodes(); i++) {
1096                         if (nest_imc_refc[i].id == nid) {
1097                                 per_cpu(local_nest_imc_refc, cpu) = &nest_imc_refc[i];
1098                                 break;
1099                         }
1100                 }
1101         }
1102         return 0;
1103 }
1104
1105 static void cleanup_all_core_imc_memory(void)
1106 {
1107         int i, nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
1108         struct imc_mem_info *ptr = core_imc_pmu->mem_info;
1109         int size = core_imc_pmu->counter_mem_size;
1110
1111         /* mem_info will never be NULL */
1112         for (i = 0; i < nr_cores; i++) {
1113                 if (ptr[i].vbase)
1114                         free_pages((u64)ptr->vbase, get_order(size));
1115         }
1116
1117         kfree(ptr);
1118         kfree(core_imc_refc);
1119 }
1120
1121 static void thread_imc_ldbar_disable(void *dummy)
1122 {
1123         /*
1124          * By Zeroing LDBAR, we disable thread-imc
1125          * updates.
1126          */
1127         mtspr(SPRN_LDBAR, 0);
1128 }
1129
1130 void thread_imc_disable(void)
1131 {
1132         on_each_cpu(thread_imc_ldbar_disable, NULL, 1);
1133 }
1134
1135 static void cleanup_all_thread_imc_memory(void)
1136 {
1137         int i, order = get_order(thread_imc_mem_size);
1138
1139         for_each_online_cpu(i) {
1140                 if (per_cpu(thread_imc_mem, i))
1141                         free_pages((u64)per_cpu(thread_imc_mem, i), order);
1142
1143         }
1144 }
1145
1146 /*
1147  * Common function to unregister cpu hotplug callback and
1148  * free the memory.
1149  * TODO: Need to handle pmu unregistering, which will be
1150  * done in followup series.
1151  */
1152 static void imc_common_cpuhp_mem_free(struct imc_pmu *pmu_ptr)
1153 {
1154         if (pmu_ptr->domain == IMC_DOMAIN_NEST) {
1155                 mutex_lock(&nest_init_lock);
1156                 if (nest_pmus == 1) {
1157                         cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE);
1158                         kfree(nest_imc_refc);
1159                 }
1160
1161                 if (nest_pmus > 0)
1162                         nest_pmus--;
1163                 mutex_unlock(&nest_init_lock);
1164         }
1165
1166         /* Free core_imc memory */
1167         if (pmu_ptr->domain == IMC_DOMAIN_CORE) {
1168                 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE);
1169                 cleanup_all_core_imc_memory();
1170         }
1171
1172         /* Free thread_imc memory */
1173         if (pmu_ptr->domain == IMC_DOMAIN_THREAD) {
1174                 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE);
1175                 cleanup_all_thread_imc_memory();
1176         }
1177
1178         /* Only free the attr_groups which are dynamically allocated  */
1179         if (pmu_ptr->attr_groups[IMC_EVENT_ATTR])
1180                 kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]->attrs);
1181         kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]);
1182         kfree(pmu_ptr);
1183         return;
1184 }
1185
1186
1187 /*
1188  * imc_mem_init : Function to support memory allocation for core imc.
1189  */
1190 static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
1191                                                                 int pmu_index)
1192 {
1193         const char *s;
1194         int nr_cores, cpu, res;
1195
1196         if (of_property_read_string(parent, "name", &s))
1197                 return -ENODEV;
1198
1199         switch (pmu_ptr->domain) {
1200         case IMC_DOMAIN_NEST:
1201                 /* Update the pmu name */
1202                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s_imc", "nest_", s);
1203                 if (!pmu_ptr->pmu.name)
1204                         return -ENOMEM;
1205
1206                 /* Needed for hotplug/migration */
1207                 per_nest_pmu_arr[pmu_index] = pmu_ptr;
1208                 break;
1209         case IMC_DOMAIN_CORE:
1210                 /* Update the pmu name */
1211                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1212                 if (!pmu_ptr->pmu.name)
1213                         return -ENOMEM;
1214
1215                 nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
1216                 pmu_ptr->mem_info = kcalloc(nr_cores, sizeof(struct imc_mem_info),
1217                                                                 GFP_KERNEL);
1218
1219                 if (!pmu_ptr->mem_info)
1220                         return -ENOMEM;
1221
1222                 core_imc_refc = kcalloc(nr_cores, sizeof(struct imc_pmu_ref),
1223                                                                 GFP_KERNEL);
1224
1225                 if (!core_imc_refc)
1226                         return -ENOMEM;
1227
1228                 core_imc_pmu = pmu_ptr;
1229                 break;
1230         case IMC_DOMAIN_THREAD:
1231                 /* Update the pmu name */
1232                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1233                 if (!pmu_ptr->pmu.name)
1234                         return -ENOMEM;
1235
1236                 thread_imc_mem_size = pmu_ptr->counter_mem_size;
1237                 for_each_online_cpu(cpu) {
1238                         res = thread_imc_mem_alloc(cpu, pmu_ptr->counter_mem_size);
1239                         if (res)
1240                                 return res;
1241                 }
1242
1243                 thread_imc_pmu = pmu_ptr;
1244                 break;
1245         default:
1246                 return -EINVAL;
1247         }
1248
1249         return 0;
1250 }
1251
1252 /*
1253  * init_imc_pmu : Setup and register the IMC pmu device.
1254  *
1255  * @parent:     Device tree unit node
1256  * @pmu_ptr:    memory allocated for this pmu
1257  * @pmu_idx:    Count of nest pmc registered
1258  *
1259  * init_imc_pmu() setup pmu cpumask and registers for a cpu hotplug callback.
1260  * Handles failure cases and accordingly frees memory.
1261  */
1262 int init_imc_pmu(struct device_node *parent, struct imc_pmu *pmu_ptr, int pmu_idx)
1263 {
1264         int ret;
1265
1266         ret = imc_mem_init(pmu_ptr, parent, pmu_idx);
1267         if (ret)
1268                 goto err_free;
1269
1270         switch (pmu_ptr->domain) {
1271         case IMC_DOMAIN_NEST:
1272                 /*
1273                 * Nest imc pmu need only one cpu per chip, we initialize the
1274                 * cpumask for the first nest imc pmu and use the same for the
1275                 * rest. To handle the cpuhotplug callback unregister, we track
1276                 * the number of nest pmus in "nest_pmus".
1277                 */
1278                 mutex_lock(&nest_init_lock);
1279                 if (nest_pmus == 0) {
1280                         ret = init_nest_pmu_ref();
1281                         if (ret) {
1282                                 mutex_unlock(&nest_init_lock);
1283                                 goto err_free;
1284                         }
1285                         /* Register for cpu hotplug notification. */
1286                         ret = nest_pmu_cpumask_init();
1287                         if (ret) {
1288                                 mutex_unlock(&nest_init_lock);
1289                                 goto err_free;
1290                         }
1291                 }
1292                 nest_pmus++;
1293                 mutex_unlock(&nest_init_lock);
1294                 break;
1295         case IMC_DOMAIN_CORE:
1296                 ret = core_imc_pmu_cpumask_init();
1297                 if (ret) {
1298                         cleanup_all_core_imc_memory();
1299                         return ret;
1300                 }
1301
1302                 break;
1303         case IMC_DOMAIN_THREAD:
1304                 ret = thread_imc_cpu_init();
1305                 if (ret) {
1306                         cleanup_all_thread_imc_memory();
1307                         return ret;
1308                 }
1309
1310                 break;
1311         default:
1312                 return  -1;     /* Unknown domain */
1313         }
1314
1315         ret = update_events_in_group(parent, pmu_ptr);
1316         if (ret)
1317                 goto err_free;
1318
1319         ret = update_pmu_ops(pmu_ptr);
1320         if (ret)
1321                 goto err_free;
1322
1323         ret = perf_pmu_register(&pmu_ptr->pmu, pmu_ptr->pmu.name, -1);
1324         if (ret)
1325                 goto err_free;
1326
1327         pr_info("%s performance monitor hardware support registered\n",
1328                                                         pmu_ptr->pmu.name);
1329
1330         return 0;
1331
1332 err_free:
1333         imc_common_cpuhp_mem_free(pmu_ptr);
1334         return ret;
1335 }