powerpc/perf: Fix core-imc hotplug callback failure during imc initialization
[sfrench/cifs-2.6.git] / arch / powerpc / perf / imc-pmu.c
1 /*
2  * In-Memory Collection (IMC) Performance Monitor counter support.
3  *
4  * Copyright (C) 2017 Madhavan Srinivasan, IBM Corporation.
5  *           (C) 2017 Anju T Sudhakar, IBM Corporation.
6  *           (C) 2017 Hemant K Shaw, IBM Corporation.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License
10  * as published by the Free Software Foundation; either version
11  * 2 of the License, or later version.
12  */
13 #include <linux/perf_event.h>
14 #include <linux/slab.h>
15 #include <asm/opal.h>
16 #include <asm/imc-pmu.h>
17 #include <asm/cputhreads.h>
18 #include <asm/smp.h>
19 #include <linux/string.h>
20
21 /* Nest IMC data structures and variables */
22
23 /*
24  * Used to avoid races in counting the nest-pmu units during hotplug
25  * register and unregister
26  */
27 static DEFINE_MUTEX(nest_init_lock);
28 static DEFINE_PER_CPU(struct imc_pmu_ref *, local_nest_imc_refc);
29 static struct imc_pmu *per_nest_pmu_arr[IMC_MAX_PMUS];
30 static cpumask_t nest_imc_cpumask;
31 struct imc_pmu_ref *nest_imc_refc;
32 static int nest_pmus;
33
34 /* Core IMC data structures and variables */
35
36 static cpumask_t core_imc_cpumask;
37 struct imc_pmu_ref *core_imc_refc;
38 static struct imc_pmu *core_imc_pmu;
39
40 /* Thread IMC data structures and variables */
41
42 static DEFINE_PER_CPU(u64 *, thread_imc_mem);
43 static struct imc_pmu *thread_imc_pmu;
44 static int thread_imc_mem_size;
45
46 struct imc_pmu *imc_event_to_pmu(struct perf_event *event)
47 {
48         return container_of(event->pmu, struct imc_pmu, pmu);
49 }
50
51 PMU_FORMAT_ATTR(event, "config:0-40");
52 PMU_FORMAT_ATTR(offset, "config:0-31");
53 PMU_FORMAT_ATTR(rvalue, "config:32");
54 PMU_FORMAT_ATTR(mode, "config:33-40");
55 static struct attribute *imc_format_attrs[] = {
56         &format_attr_event.attr,
57         &format_attr_offset.attr,
58         &format_attr_rvalue.attr,
59         &format_attr_mode.attr,
60         NULL,
61 };
62
63 static struct attribute_group imc_format_group = {
64         .name = "format",
65         .attrs = imc_format_attrs,
66 };
67
68 /* Get the cpumask printed to a buffer "buf" */
69 static ssize_t imc_pmu_cpumask_get_attr(struct device *dev,
70                                         struct device_attribute *attr,
71                                         char *buf)
72 {
73         struct pmu *pmu = dev_get_drvdata(dev);
74         struct imc_pmu *imc_pmu = container_of(pmu, struct imc_pmu, pmu);
75         cpumask_t *active_mask;
76
77         switch(imc_pmu->domain){
78         case IMC_DOMAIN_NEST:
79                 active_mask = &nest_imc_cpumask;
80                 break;
81         case IMC_DOMAIN_CORE:
82                 active_mask = &core_imc_cpumask;
83                 break;
84         default:
85                 return 0;
86         }
87
88         return cpumap_print_to_pagebuf(true, buf, active_mask);
89 }
90
91 static DEVICE_ATTR(cpumask, S_IRUGO, imc_pmu_cpumask_get_attr, NULL);
92
93 static struct attribute *imc_pmu_cpumask_attrs[] = {
94         &dev_attr_cpumask.attr,
95         NULL,
96 };
97
98 static struct attribute_group imc_pmu_cpumask_attr_group = {
99         .attrs = imc_pmu_cpumask_attrs,
100 };
101
102 /* device_str_attr_create : Populate event "name" and string "str" in attribute */
103 static struct attribute *device_str_attr_create(const char *name, const char *str)
104 {
105         struct perf_pmu_events_attr *attr;
106
107         attr = kzalloc(sizeof(*attr), GFP_KERNEL);
108         if (!attr)
109                 return NULL;
110         sysfs_attr_init(&attr->attr.attr);
111
112         attr->event_str = str;
113         attr->attr.attr.name = name;
114         attr->attr.attr.mode = 0444;
115         attr->attr.show = perf_event_sysfs_show;
116
117         return &attr->attr.attr;
118 }
119
120 struct imc_events *imc_parse_event(struct device_node *np, const char *scale,
121                                   const char *unit, const char *prefix, u32 base)
122 {
123         struct imc_events *event;
124         const char *s;
125         u32 reg;
126
127         event = kzalloc(sizeof(struct imc_events), GFP_KERNEL);
128         if (!event)
129                 return NULL;
130
131         if (of_property_read_u32(np, "reg", &reg))
132                 goto error;
133         /* Add the base_reg value to the "reg" */
134         event->value = base + reg;
135
136         if (of_property_read_string(np, "event-name", &s))
137                 goto error;
138
139         event->name = kasprintf(GFP_KERNEL, "%s%s", prefix, s);
140         if (!event->name)
141                 goto error;
142
143         if (of_property_read_string(np, "scale", &s))
144                 s = scale;
145
146         if (s) {
147                 event->scale = kstrdup(s, GFP_KERNEL);
148                 if (!event->scale)
149                         goto error;
150         }
151
152         if (of_property_read_string(np, "unit", &s))
153                 s = unit;
154
155         if (s) {
156                 event->unit = kstrdup(s, GFP_KERNEL);
157                 if (!event->unit)
158                         goto error;
159         }
160
161         return event;
162 error:
163         kfree(event->unit);
164         kfree(event->scale);
165         kfree(event->name);
166         kfree(event);
167
168         return NULL;
169 }
170
171 /*
172  * update_events_in_group: Update the "events" information in an attr_group
173  *                         and assign the attr_group to the pmu "pmu".
174  */
175 static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
176 {
177         struct attribute_group *attr_group;
178         struct attribute **attrs, *dev_str;
179         struct device_node *np, *pmu_events;
180         struct imc_events *ev;
181         u32 handle, base_reg;
182         int i=0, j=0, ct;
183         const char *prefix, *g_scale, *g_unit;
184         const char *ev_val_str, *ev_scale_str, *ev_unit_str;
185
186         if (!of_property_read_u32(node, "events", &handle))
187                 pmu_events = of_find_node_by_phandle(handle);
188         else
189                 return 0;
190
191         /* Did not find any node with a given phandle */
192         if (!pmu_events)
193                 return 0;
194
195         /* Get a count of number of child nodes */
196         ct = of_get_child_count(pmu_events);
197
198         /* Get the event prefix */
199         if (of_property_read_string(node, "events-prefix", &prefix))
200                 return 0;
201
202         /* Get a global unit and scale data if available */
203         if (of_property_read_string(node, "scale", &g_scale))
204                 g_scale = NULL;
205
206         if (of_property_read_string(node, "unit", &g_unit))
207                 g_unit = NULL;
208
209         /* "reg" property gives out the base offset of the counters data */
210         of_property_read_u32(node, "reg", &base_reg);
211
212         /* Allocate memory for the events */
213         pmu->events = kcalloc(ct, sizeof(struct imc_events), GFP_KERNEL);
214         if (!pmu->events)
215                 return -ENOMEM;
216
217         ct = 0;
218         /* Parse the events and update the struct */
219         for_each_child_of_node(pmu_events, np) {
220                 ev = imc_parse_event(np, g_scale, g_unit, prefix, base_reg);
221                 if (ev)
222                         pmu->events[ct++] = ev;
223         }
224
225         /* Allocate memory for attribute group */
226         attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL);
227         if (!attr_group)
228                 return -ENOMEM;
229
230         /*
231          * Allocate memory for attributes.
232          * Since we have count of events for this pmu, we also allocate
233          * memory for the scale and unit attribute for now.
234          * "ct" has the total event structs added from the events-parent node.
235          * So allocate three times the "ct" (this includes event, event_scale and
236          * event_unit).
237          */
238         attrs = kcalloc(((ct * 3) + 1), sizeof(struct attribute *), GFP_KERNEL);
239         if (!attrs) {
240                 kfree(attr_group);
241                 kfree(pmu->events);
242                 return -ENOMEM;
243         }
244
245         attr_group->name = "events";
246         attr_group->attrs = attrs;
247         do {
248                 ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i]->value);
249                 dev_str = device_str_attr_create(pmu->events[i]->name, ev_val_str);
250                 if (!dev_str)
251                         continue;
252
253                 attrs[j++] = dev_str;
254                 if (pmu->events[i]->scale) {
255                         ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale",pmu->events[i]->name);
256                         dev_str = device_str_attr_create(ev_scale_str, pmu->events[i]->scale);
257                         if (!dev_str)
258                                 continue;
259
260                         attrs[j++] = dev_str;
261                 }
262
263                 if (pmu->events[i]->unit) {
264                         ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit",pmu->events[i]->name);
265                         dev_str = device_str_attr_create(ev_unit_str, pmu->events[i]->unit);
266                         if (!dev_str)
267                                 continue;
268
269                         attrs[j++] = dev_str;
270                 }
271         } while (++i < ct);
272
273         /* Save the event attribute */
274         pmu->attr_groups[IMC_EVENT_ATTR] = attr_group;
275
276         kfree(pmu->events);
277         return 0;
278 }
279
280 /* get_nest_pmu_ref: Return the imc_pmu_ref struct for the given node */
281 static struct imc_pmu_ref *get_nest_pmu_ref(int cpu)
282 {
283         return per_cpu(local_nest_imc_refc, cpu);
284 }
285
286 static void nest_change_cpu_context(int old_cpu, int new_cpu)
287 {
288         struct imc_pmu **pn = per_nest_pmu_arr;
289         int i;
290
291         if (old_cpu < 0 || new_cpu < 0)
292                 return;
293
294         for (i = 0; *pn && i < IMC_MAX_PMUS; i++, pn++)
295                 perf_pmu_migrate_context(&(*pn)->pmu, old_cpu, new_cpu);
296 }
297
298 static int ppc_nest_imc_cpu_offline(unsigned int cpu)
299 {
300         int nid, target = -1;
301         const struct cpumask *l_cpumask;
302         struct imc_pmu_ref *ref;
303
304         /*
305          * Check in the designated list for this cpu. Dont bother
306          * if not one of them.
307          */
308         if (!cpumask_test_and_clear_cpu(cpu, &nest_imc_cpumask))
309                 return 0;
310
311         /*
312          * Now that this cpu is one of the designated,
313          * find a next cpu a) which is online and b) in same chip.
314          */
315         nid = cpu_to_node(cpu);
316         l_cpumask = cpumask_of_node(nid);
317         target = cpumask_any_but(l_cpumask, cpu);
318
319         /*
320          * Update the cpumask with the target cpu and
321          * migrate the context if needed
322          */
323         if (target >= 0 && target < nr_cpu_ids) {
324                 cpumask_set_cpu(target, &nest_imc_cpumask);
325                 nest_change_cpu_context(cpu, target);
326         } else {
327                 opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
328                                        get_hard_smp_processor_id(cpu));
329                 /*
330                  * If this is the last cpu in this chip then, skip the reference
331                  * count mutex lock and make the reference count on this chip zero.
332                  */
333                 ref = get_nest_pmu_ref(cpu);
334                 if (!ref)
335                         return -EINVAL;
336
337                 ref->refc = 0;
338         }
339         return 0;
340 }
341
342 static int ppc_nest_imc_cpu_online(unsigned int cpu)
343 {
344         const struct cpumask *l_cpumask;
345         static struct cpumask tmp_mask;
346         int res;
347
348         /* Get the cpumask of this node */
349         l_cpumask = cpumask_of_node(cpu_to_node(cpu));
350
351         /*
352          * If this is not the first online CPU on this node, then
353          * just return.
354          */
355         if (cpumask_and(&tmp_mask, l_cpumask, &nest_imc_cpumask))
356                 return 0;
357
358         /*
359          * If this is the first online cpu on this node
360          * disable the nest counters by making an OPAL call.
361          */
362         res = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
363                                      get_hard_smp_processor_id(cpu));
364         if (res)
365                 return res;
366
367         /* Make this CPU the designated target for counter collection */
368         cpumask_set_cpu(cpu, &nest_imc_cpumask);
369         return 0;
370 }
371
372 static int nest_pmu_cpumask_init(void)
373 {
374         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE,
375                                  "perf/powerpc/imc:online",
376                                  ppc_nest_imc_cpu_online,
377                                  ppc_nest_imc_cpu_offline);
378 }
379
380 static void nest_imc_counters_release(struct perf_event *event)
381 {
382         int rc, node_id;
383         struct imc_pmu_ref *ref;
384
385         if (event->cpu < 0)
386                 return;
387
388         node_id = cpu_to_node(event->cpu);
389
390         /*
391          * See if we need to disable the nest PMU.
392          * If no events are currently in use, then we have to take a
393          * mutex to ensure that we don't race with another task doing
394          * enable or disable the nest counters.
395          */
396         ref = get_nest_pmu_ref(event->cpu);
397         if (!ref)
398                 return;
399
400         /* Take the mutex lock for this node and then decrement the reference count */
401         mutex_lock(&ref->lock);
402         if (ref->refc == 0) {
403                 /*
404                  * The scenario where this is true is, when perf session is
405                  * started, followed by offlining of all cpus in a given node.
406                  *
407                  * In the cpuhotplug offline path, ppc_nest_imc_cpu_offline()
408                  * function set the ref->count to zero, if the cpu which is
409                  * about to offline is the last cpu in a given node and make
410                  * an OPAL call to disable the engine in that node.
411                  *
412                  */
413                 mutex_unlock(&ref->lock);
414                 return;
415         }
416         ref->refc--;
417         if (ref->refc == 0) {
418                 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
419                                             get_hard_smp_processor_id(event->cpu));
420                 if (rc) {
421                         mutex_unlock(&ref->lock);
422                         pr_err("nest-imc: Unable to stop the counters for core %d\n", node_id);
423                         return;
424                 }
425         } else if (ref->refc < 0) {
426                 WARN(1, "nest-imc: Invalid event reference count\n");
427                 ref->refc = 0;
428         }
429         mutex_unlock(&ref->lock);
430 }
431
432 static int nest_imc_event_init(struct perf_event *event)
433 {
434         int chip_id, rc, node_id;
435         u32 l_config, config = event->attr.config;
436         struct imc_mem_info *pcni;
437         struct imc_pmu *pmu;
438         struct imc_pmu_ref *ref;
439         bool flag = false;
440
441         if (event->attr.type != event->pmu->type)
442                 return -ENOENT;
443
444         /* Sampling not supported */
445         if (event->hw.sample_period)
446                 return -EINVAL;
447
448         /* unsupported modes and filters */
449         if (event->attr.exclude_user   ||
450             event->attr.exclude_kernel ||
451             event->attr.exclude_hv     ||
452             event->attr.exclude_idle   ||
453             event->attr.exclude_host   ||
454             event->attr.exclude_guest)
455                 return -EINVAL;
456
457         if (event->cpu < 0)
458                 return -EINVAL;
459
460         pmu = imc_event_to_pmu(event);
461
462         /* Sanity check for config (event offset) */
463         if ((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size)
464                 return -EINVAL;
465
466         /*
467          * Nest HW counter memory resides in a per-chip reserve-memory (HOMER).
468          * Get the base memory addresss for this cpu.
469          */
470         chip_id = topology_physical_package_id(event->cpu);
471         pcni = pmu->mem_info;
472         do {
473                 if (pcni->id == chip_id) {
474                         flag = true;
475                         break;
476                 }
477                 pcni++;
478         } while (pcni);
479
480         if (!flag)
481                 return -ENODEV;
482
483         /*
484          * Add the event offset to the base address.
485          */
486         l_config = config & IMC_EVENT_OFFSET_MASK;
487         event->hw.event_base = (u64)pcni->vbase + l_config;
488         node_id = cpu_to_node(event->cpu);
489
490         /*
491          * Get the imc_pmu_ref struct for this node.
492          * Take the mutex lock and then increment the count of nest pmu events
493          * inited.
494          */
495         ref = get_nest_pmu_ref(event->cpu);
496         if (!ref)
497                 return -EINVAL;
498
499         mutex_lock(&ref->lock);
500         if (ref->refc == 0) {
501                 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_NEST,
502                                              get_hard_smp_processor_id(event->cpu));
503                 if (rc) {
504                         mutex_unlock(&ref->lock);
505                         pr_err("nest-imc: Unable to start the counters for node %d\n",
506                                                                         node_id);
507                         return rc;
508                 }
509         }
510         ++ref->refc;
511         mutex_unlock(&ref->lock);
512
513         event->destroy = nest_imc_counters_release;
514         return 0;
515 }
516
517 /*
518  * core_imc_mem_init : Initializes memory for the current core.
519  *
520  * Uses alloc_pages_node() and uses the returned address as an argument to
521  * an opal call to configure the pdbar. The address sent as an argument is
522  * converted to physical address before the opal call is made. This is the
523  * base address at which the core imc counters are populated.
524  */
525 static int core_imc_mem_init(int cpu, int size)
526 {
527         int phys_id, rc = 0, core_id = (cpu / threads_per_core);
528         struct imc_mem_info *mem_info;
529
530         /*
531          * alloc_pages_node() will allocate memory for core in the
532          * local node only.
533          */
534         phys_id = topology_physical_package_id(cpu);
535         mem_info = &core_imc_pmu->mem_info[core_id];
536         mem_info->id = core_id;
537
538         /* We need only vbase for core counters */
539         mem_info->vbase = page_address(alloc_pages_node(phys_id,
540                                           GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
541                                           __GFP_NOWARN, get_order(size)));
542         if (!mem_info->vbase)
543                 return -ENOMEM;
544
545         /* Init the mutex */
546         core_imc_refc[core_id].id = core_id;
547         mutex_init(&core_imc_refc[core_id].lock);
548
549         rc = opal_imc_counters_init(OPAL_IMC_COUNTERS_CORE,
550                                 __pa((void *)mem_info->vbase),
551                                 get_hard_smp_processor_id(cpu));
552         if (rc) {
553                 free_pages((u64)mem_info->vbase, get_order(size));
554                 mem_info->vbase = NULL;
555         }
556
557         return rc;
558 }
559
560 static bool is_core_imc_mem_inited(int cpu)
561 {
562         struct imc_mem_info *mem_info;
563         int core_id = (cpu / threads_per_core);
564
565         mem_info = &core_imc_pmu->mem_info[core_id];
566         if (!mem_info->vbase)
567                 return false;
568
569         return true;
570 }
571
572 static int ppc_core_imc_cpu_online(unsigned int cpu)
573 {
574         const struct cpumask *l_cpumask;
575         static struct cpumask tmp_mask;
576         int ret = 0;
577
578         /* Get the cpumask for this core */
579         l_cpumask = cpu_sibling_mask(cpu);
580
581         /* If a cpu for this core is already set, then, don't do anything */
582         if (cpumask_and(&tmp_mask, l_cpumask, &core_imc_cpumask))
583                 return 0;
584
585         if (!is_core_imc_mem_inited(cpu)) {
586                 ret = core_imc_mem_init(cpu, core_imc_pmu->counter_mem_size);
587                 if (ret) {
588                         pr_info("core_imc memory allocation for cpu %d failed\n", cpu);
589                         return ret;
590                 }
591         }
592
593         /* set the cpu in the mask */
594         cpumask_set_cpu(cpu, &core_imc_cpumask);
595         return 0;
596 }
597
598 static int ppc_core_imc_cpu_offline(unsigned int cpu)
599 {
600         unsigned int ncpu, core_id;
601         struct imc_pmu_ref *ref;
602
603         /*
604          * clear this cpu out of the mask, if not present in the mask,
605          * don't bother doing anything.
606          */
607         if (!cpumask_test_and_clear_cpu(cpu, &core_imc_cpumask))
608                 return 0;
609
610         /*
611          * Check whether core_imc is registered. We could end up here
612          * if the cpuhotplug callback registration fails. i.e, callback
613          * invokes the offline path for all sucessfully registered cpus.
614          * At this stage, core_imc pmu will not be registered and we
615          * should return here.
616          *
617          * We return with a zero since this is not an offline failure.
618          * And cpuhp_setup_state() returns the actual failure reason
619          * to the caller, which inturn will call the cleanup routine.
620          */
621         if (!core_imc_pmu->pmu.event_init)
622                 return 0;
623
624         /* Find any online cpu in that core except the current "cpu" */
625         ncpu = cpumask_any_but(cpu_sibling_mask(cpu), cpu);
626
627         if (ncpu >= 0 && ncpu < nr_cpu_ids) {
628                 cpumask_set_cpu(ncpu, &core_imc_cpumask);
629                 perf_pmu_migrate_context(&core_imc_pmu->pmu, cpu, ncpu);
630         } else {
631                 /*
632                  * If this is the last cpu in this core then, skip taking refernce
633                  * count mutex lock for this core and directly zero "refc" for
634                  * this core.
635                  */
636                 opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
637                                        get_hard_smp_processor_id(cpu));
638                 core_id = cpu / threads_per_core;
639                 ref = &core_imc_refc[core_id];
640                 if (!ref)
641                         return -EINVAL;
642
643                 ref->refc = 0;
644         }
645         return 0;
646 }
647
648 static int core_imc_pmu_cpumask_init(void)
649 {
650         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
651                                  "perf/powerpc/imc_core:online",
652                                  ppc_core_imc_cpu_online,
653                                  ppc_core_imc_cpu_offline);
654 }
655
656 static void core_imc_counters_release(struct perf_event *event)
657 {
658         int rc, core_id;
659         struct imc_pmu_ref *ref;
660
661         if (event->cpu < 0)
662                 return;
663         /*
664          * See if we need to disable the IMC PMU.
665          * If no events are currently in use, then we have to take a
666          * mutex to ensure that we don't race with another task doing
667          * enable or disable the core counters.
668          */
669         core_id = event->cpu / threads_per_core;
670
671         /* Take the mutex lock and decrement the refernce count for this core */
672         ref = &core_imc_refc[core_id];
673         if (!ref)
674                 return;
675
676         mutex_lock(&ref->lock);
677         if (ref->refc == 0) {
678                 /*
679                  * The scenario where this is true is, when perf session is
680                  * started, followed by offlining of all cpus in a given core.
681                  *
682                  * In the cpuhotplug offline path, ppc_core_imc_cpu_offline()
683                  * function set the ref->count to zero, if the cpu which is
684                  * about to offline is the last cpu in a given core and make
685                  * an OPAL call to disable the engine in that core.
686                  *
687                  */
688                 mutex_unlock(&ref->lock);
689                 return;
690         }
691         ref->refc--;
692         if (ref->refc == 0) {
693                 rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
694                                             get_hard_smp_processor_id(event->cpu));
695                 if (rc) {
696                         mutex_unlock(&ref->lock);
697                         pr_err("IMC: Unable to stop the counters for core %d\n", core_id);
698                         return;
699                 }
700         } else if (ref->refc < 0) {
701                 WARN(1, "core-imc: Invalid event reference count\n");
702                 ref->refc = 0;
703         }
704         mutex_unlock(&ref->lock);
705 }
706
707 static int core_imc_event_init(struct perf_event *event)
708 {
709         int core_id, rc;
710         u64 config = event->attr.config;
711         struct imc_mem_info *pcmi;
712         struct imc_pmu *pmu;
713         struct imc_pmu_ref *ref;
714
715         if (event->attr.type != event->pmu->type)
716                 return -ENOENT;
717
718         /* Sampling not supported */
719         if (event->hw.sample_period)
720                 return -EINVAL;
721
722         /* unsupported modes and filters */
723         if (event->attr.exclude_user   ||
724             event->attr.exclude_kernel ||
725             event->attr.exclude_hv     ||
726             event->attr.exclude_idle   ||
727             event->attr.exclude_host   ||
728             event->attr.exclude_guest)
729                 return -EINVAL;
730
731         if (event->cpu < 0)
732                 return -EINVAL;
733
734         event->hw.idx = -1;
735         pmu = imc_event_to_pmu(event);
736
737         /* Sanity check for config (event offset) */
738         if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
739                 return -EINVAL;
740
741         if (!is_core_imc_mem_inited(event->cpu))
742                 return -ENODEV;
743
744         core_id = event->cpu / threads_per_core;
745         pcmi = &core_imc_pmu->mem_info[core_id];
746         if ((!pcmi->vbase))
747                 return -ENODEV;
748
749         /* Get the core_imc mutex for this core */
750         ref = &core_imc_refc[core_id];
751         if (!ref)
752                 return -EINVAL;
753
754         /*
755          * Core pmu units are enabled only when it is used.
756          * See if this is triggered for the first time.
757          * If yes, take the mutex lock and enable the core counters.
758          * If not, just increment the count in core_imc_refc struct.
759          */
760         mutex_lock(&ref->lock);
761         if (ref->refc == 0) {
762                 rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
763                                              get_hard_smp_processor_id(event->cpu));
764                 if (rc) {
765                         mutex_unlock(&ref->lock);
766                         pr_err("core-imc: Unable to start the counters for core %d\n",
767                                                                         core_id);
768                         return rc;
769                 }
770         }
771         ++ref->refc;
772         mutex_unlock(&ref->lock);
773
774         event->hw.event_base = (u64)pcmi->vbase + (config & IMC_EVENT_OFFSET_MASK);
775         event->destroy = core_imc_counters_release;
776         return 0;
777 }
778
779 /*
780  * Allocates a page of memory for each of the online cpus, and write the
781  * physical base address of that page to the LDBAR for that cpu.
782  *
783  * LDBAR Register Layout:
784  *
785  *  0          4         8         12        16        20        24        28
786  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
787  *   | |       [   ]    [                   Counter Address [8:50]
788  *   | * Mode    |
789  *   |           * PB Scope
790  *   * Enable/Disable
791  *
792  *  32        36        40        44        48        52        56        60
793  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
794  *           Counter Address [8:50]              ]
795  *
796  */
797 static int thread_imc_mem_alloc(int cpu_id, int size)
798 {
799         u64 ldbar_value, *local_mem = per_cpu(thread_imc_mem, cpu_id);
800         int phys_id = topology_physical_package_id(cpu_id);
801
802         if (!local_mem) {
803                 /*
804                  * This case could happen only once at start, since we dont
805                  * free the memory in cpu offline path.
806                  */
807                 local_mem = page_address(alloc_pages_node(phys_id,
808                                   GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE |
809                                   __GFP_NOWARN, get_order(size)));
810                 if (!local_mem)
811                         return -ENOMEM;
812
813                 per_cpu(thread_imc_mem, cpu_id) = local_mem;
814         }
815
816         ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) | THREAD_IMC_ENABLE;
817
818         mtspr(SPRN_LDBAR, ldbar_value);
819         return 0;
820 }
821
822 static int ppc_thread_imc_cpu_online(unsigned int cpu)
823 {
824         return thread_imc_mem_alloc(cpu, thread_imc_mem_size);
825 }
826
827 static int ppc_thread_imc_cpu_offline(unsigned int cpu)
828 {
829         mtspr(SPRN_LDBAR, 0);
830         return 0;
831 }
832
833 static int thread_imc_cpu_init(void)
834 {
835         return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
836                           "perf/powerpc/imc_thread:online",
837                           ppc_thread_imc_cpu_online,
838                           ppc_thread_imc_cpu_offline);
839 }
840
841 void thread_imc_pmu_sched_task(struct perf_event_context *ctx,
842                                       bool sched_in)
843 {
844         int core_id;
845         struct imc_pmu_ref *ref;
846
847         if (!is_core_imc_mem_inited(smp_processor_id()))
848                 return;
849
850         core_id = smp_processor_id() / threads_per_core;
851         /*
852          * imc pmus are enabled only when it is used.
853          * See if this is triggered for the first time.
854          * If yes, take the mutex lock and enable the counters.
855          * If not, just increment the count in ref count struct.
856          */
857         ref = &core_imc_refc[core_id];
858         if (!ref)
859                 return;
860
861         if (sched_in) {
862                 mutex_lock(&ref->lock);
863                 if (ref->refc == 0) {
864                         if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
865                              get_hard_smp_processor_id(smp_processor_id()))) {
866                                 mutex_unlock(&ref->lock);
867                                 pr_err("thread-imc: Unable to start the counter\
868                                                         for core %d\n", core_id);
869                                 return;
870                         }
871                 }
872                 ++ref->refc;
873                 mutex_unlock(&ref->lock);
874         } else {
875                 mutex_lock(&ref->lock);
876                 ref->refc--;
877                 if (ref->refc == 0) {
878                         if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
879                             get_hard_smp_processor_id(smp_processor_id()))) {
880                                 mutex_unlock(&ref->lock);
881                                 pr_err("thread-imc: Unable to stop the counters\
882                                                         for core %d\n", core_id);
883                                 return;
884                         }
885                 } else if (ref->refc < 0) {
886                         ref->refc = 0;
887                 }
888                 mutex_unlock(&ref->lock);
889         }
890
891         return;
892 }
893
894 static int thread_imc_event_init(struct perf_event *event)
895 {
896         u32 config = event->attr.config;
897         struct task_struct *target;
898         struct imc_pmu *pmu;
899
900         if (event->attr.type != event->pmu->type)
901                 return -ENOENT;
902
903         /* Sampling not supported */
904         if (event->hw.sample_period)
905                 return -EINVAL;
906
907         event->hw.idx = -1;
908         pmu = imc_event_to_pmu(event);
909
910         /* Sanity check for config offset */
911         if (((config & IMC_EVENT_OFFSET_MASK) > pmu->counter_mem_size))
912                 return -EINVAL;
913
914         target = event->hw.target;
915         if (!target)
916                 return -EINVAL;
917
918         event->pmu->task_ctx_nr = perf_sw_context;
919         return 0;
920 }
921
922 static bool is_thread_imc_pmu(struct perf_event *event)
923 {
924         if (!strncmp(event->pmu->name, "thread_imc", strlen("thread_imc")))
925                 return true;
926
927         return false;
928 }
929
930 static u64 * get_event_base_addr(struct perf_event *event)
931 {
932         u64 addr;
933
934         if (is_thread_imc_pmu(event)) {
935                 addr = (u64)per_cpu(thread_imc_mem, smp_processor_id());
936                 return (u64 *)(addr + (event->attr.config & IMC_EVENT_OFFSET_MASK));
937         }
938
939         return (u64 *)event->hw.event_base;
940 }
941
942 static void thread_imc_pmu_start_txn(struct pmu *pmu,
943                                      unsigned int txn_flags)
944 {
945         if (txn_flags & ~PERF_PMU_TXN_ADD)
946                 return;
947         perf_pmu_disable(pmu);
948 }
949
950 static void thread_imc_pmu_cancel_txn(struct pmu *pmu)
951 {
952         perf_pmu_enable(pmu);
953 }
954
955 static int thread_imc_pmu_commit_txn(struct pmu *pmu)
956 {
957         perf_pmu_enable(pmu);
958         return 0;
959 }
960
961 static u64 imc_read_counter(struct perf_event *event)
962 {
963         u64 *addr, data;
964
965         /*
966          * In-Memory Collection (IMC) counters are free flowing counters.
967          * So we take a snapshot of the counter value on enable and save it
968          * to calculate the delta at later stage to present the event counter
969          * value.
970          */
971         addr = get_event_base_addr(event);
972         data = be64_to_cpu(READ_ONCE(*addr));
973         local64_set(&event->hw.prev_count, data);
974
975         return data;
976 }
977
978 static void imc_event_update(struct perf_event *event)
979 {
980         u64 counter_prev, counter_new, final_count;
981
982         counter_prev = local64_read(&event->hw.prev_count);
983         counter_new = imc_read_counter(event);
984         final_count = counter_new - counter_prev;
985
986         /* Update the delta to the event count */
987         local64_add(final_count, &event->count);
988 }
989
990 static void imc_event_start(struct perf_event *event, int flags)
991 {
992         /*
993          * In Memory Counters are free flowing counters. HW or the microcode
994          * keeps adding to the counter offset in memory. To get event
995          * counter value, we snapshot the value here and we calculate
996          * delta at later point.
997          */
998         imc_read_counter(event);
999 }
1000
1001 static void imc_event_stop(struct perf_event *event, int flags)
1002 {
1003         /*
1004          * Take a snapshot and calculate the delta and update
1005          * the event counter values.
1006          */
1007         imc_event_update(event);
1008 }
1009
1010 static int imc_event_add(struct perf_event *event, int flags)
1011 {
1012         if (flags & PERF_EF_START)
1013                 imc_event_start(event, flags);
1014
1015         return 0;
1016 }
1017
1018 static int thread_imc_event_add(struct perf_event *event, int flags)
1019 {
1020         if (flags & PERF_EF_START)
1021                 imc_event_start(event, flags);
1022
1023         /* Enable the sched_task to start the engine */
1024         perf_sched_cb_inc(event->ctx->pmu);
1025         return 0;
1026 }
1027
1028 static void thread_imc_event_del(struct perf_event *event, int flags)
1029 {
1030         /*
1031          * Take a snapshot and calculate the delta and update
1032          * the event counter values.
1033          */
1034         imc_event_update(event);
1035         perf_sched_cb_dec(event->ctx->pmu);
1036 }
1037
1038 /* update_pmu_ops : Populate the appropriate operations for "pmu" */
1039 static int update_pmu_ops(struct imc_pmu *pmu)
1040 {
1041         pmu->pmu.task_ctx_nr = perf_invalid_context;
1042         pmu->pmu.add = imc_event_add;
1043         pmu->pmu.del = imc_event_stop;
1044         pmu->pmu.start = imc_event_start;
1045         pmu->pmu.stop = imc_event_stop;
1046         pmu->pmu.read = imc_event_update;
1047         pmu->pmu.attr_groups = pmu->attr_groups;
1048         pmu->attr_groups[IMC_FORMAT_ATTR] = &imc_format_group;
1049
1050         switch (pmu->domain) {
1051         case IMC_DOMAIN_NEST:
1052                 pmu->pmu.event_init = nest_imc_event_init;
1053                 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1054                 break;
1055         case IMC_DOMAIN_CORE:
1056                 pmu->pmu.event_init = core_imc_event_init;
1057                 pmu->attr_groups[IMC_CPUMASK_ATTR] = &imc_pmu_cpumask_attr_group;
1058                 break;
1059         case IMC_DOMAIN_THREAD:
1060                 pmu->pmu.event_init = thread_imc_event_init;
1061                 pmu->pmu.sched_task = thread_imc_pmu_sched_task;
1062                 pmu->pmu.add = thread_imc_event_add;
1063                 pmu->pmu.del = thread_imc_event_del;
1064                 pmu->pmu.start_txn = thread_imc_pmu_start_txn;
1065                 pmu->pmu.cancel_txn = thread_imc_pmu_cancel_txn;
1066                 pmu->pmu.commit_txn = thread_imc_pmu_commit_txn;
1067                 break;
1068         default:
1069                 break;
1070         }
1071
1072         return 0;
1073 }
1074
1075 /* init_nest_pmu_ref: Initialize the imc_pmu_ref struct for all the nodes */
1076 static int init_nest_pmu_ref(void)
1077 {
1078         int nid, i, cpu;
1079
1080         nest_imc_refc = kcalloc(num_possible_nodes(), sizeof(*nest_imc_refc),
1081                                                                 GFP_KERNEL);
1082
1083         if (!nest_imc_refc)
1084                 return -ENOMEM;
1085
1086         i = 0;
1087         for_each_node(nid) {
1088                 /*
1089                  * Mutex lock to avoid races while tracking the number of
1090                  * sessions using the chip's nest pmu units.
1091                  */
1092                 mutex_init(&nest_imc_refc[i].lock);
1093
1094                 /*
1095                  * Loop to init the "id" with the node_id. Variable "i" initialized to
1096                  * 0 and will be used as index to the array. "i" will not go off the
1097                  * end of the array since the "for_each_node" loops for "N_POSSIBLE"
1098                  * nodes only.
1099                  */
1100                 nest_imc_refc[i++].id = nid;
1101         }
1102
1103         /*
1104          * Loop to init the per_cpu "local_nest_imc_refc" with the proper
1105          * "nest_imc_refc" index. This makes get_nest_pmu_ref() alot simple.
1106          */
1107         for_each_possible_cpu(cpu) {
1108                 nid = cpu_to_node(cpu);
1109                 for (i = 0; i < num_possible_nodes(); i++) {
1110                         if (nest_imc_refc[i].id == nid) {
1111                                 per_cpu(local_nest_imc_refc, cpu) = &nest_imc_refc[i];
1112                                 break;
1113                         }
1114                 }
1115         }
1116         return 0;
1117 }
1118
1119 static void cleanup_all_core_imc_memory(void)
1120 {
1121         int i, nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
1122         struct imc_mem_info *ptr = core_imc_pmu->mem_info;
1123         int size = core_imc_pmu->counter_mem_size;
1124
1125         /* mem_info will never be NULL */
1126         for (i = 0; i < nr_cores; i++) {
1127                 if (ptr[i].vbase)
1128                         free_pages((u64)ptr->vbase, get_order(size));
1129         }
1130
1131         kfree(ptr);
1132         kfree(core_imc_refc);
1133 }
1134
1135 static void thread_imc_ldbar_disable(void *dummy)
1136 {
1137         /*
1138          * By Zeroing LDBAR, we disable thread-imc
1139          * updates.
1140          */
1141         mtspr(SPRN_LDBAR, 0);
1142 }
1143
1144 void thread_imc_disable(void)
1145 {
1146         on_each_cpu(thread_imc_ldbar_disable, NULL, 1);
1147 }
1148
1149 static void cleanup_all_thread_imc_memory(void)
1150 {
1151         int i, order = get_order(thread_imc_mem_size);
1152
1153         for_each_online_cpu(i) {
1154                 if (per_cpu(thread_imc_mem, i))
1155                         free_pages((u64)per_cpu(thread_imc_mem, i), order);
1156
1157         }
1158 }
1159
1160 /*
1161  * Common function to unregister cpu hotplug callback and
1162  * free the memory.
1163  * TODO: Need to handle pmu unregistering, which will be
1164  * done in followup series.
1165  */
1166 static void imc_common_cpuhp_mem_free(struct imc_pmu *pmu_ptr)
1167 {
1168         if (pmu_ptr->domain == IMC_DOMAIN_NEST) {
1169                 mutex_lock(&nest_init_lock);
1170                 if (nest_pmus == 1) {
1171                         cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_NEST_IMC_ONLINE);
1172                         kfree(nest_imc_refc);
1173                 }
1174
1175                 if (nest_pmus > 0)
1176                         nest_pmus--;
1177                 mutex_unlock(&nest_init_lock);
1178         }
1179
1180         /* Free core_imc memory */
1181         if (pmu_ptr->domain == IMC_DOMAIN_CORE) {
1182                 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE);
1183                 cleanup_all_core_imc_memory();
1184         }
1185
1186         /* Free thread_imc memory */
1187         if (pmu_ptr->domain == IMC_DOMAIN_THREAD) {
1188                 cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE);
1189                 cleanup_all_thread_imc_memory();
1190         }
1191
1192         /* Only free the attr_groups which are dynamically allocated  */
1193         if (pmu_ptr->attr_groups[IMC_EVENT_ATTR])
1194                 kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]->attrs);
1195         kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]);
1196         kfree(pmu_ptr);
1197         return;
1198 }
1199
1200
1201 /*
1202  * imc_mem_init : Function to support memory allocation for core imc.
1203  */
1204 static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
1205                                                                 int pmu_index)
1206 {
1207         const char *s;
1208         int nr_cores, cpu, res;
1209
1210         if (of_property_read_string(parent, "name", &s))
1211                 return -ENODEV;
1212
1213         switch (pmu_ptr->domain) {
1214         case IMC_DOMAIN_NEST:
1215                 /* Update the pmu name */
1216                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s_imc", "nest_", s);
1217                 if (!pmu_ptr->pmu.name)
1218                         return -ENOMEM;
1219
1220                 /* Needed for hotplug/migration */
1221                 per_nest_pmu_arr[pmu_index] = pmu_ptr;
1222                 break;
1223         case IMC_DOMAIN_CORE:
1224                 /* Update the pmu name */
1225                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1226                 if (!pmu_ptr->pmu.name)
1227                         return -ENOMEM;
1228
1229                 nr_cores = DIV_ROUND_UP(num_present_cpus(), threads_per_core);
1230                 pmu_ptr->mem_info = kcalloc(nr_cores, sizeof(struct imc_mem_info),
1231                                                                 GFP_KERNEL);
1232
1233                 if (!pmu_ptr->mem_info)
1234                         return -ENOMEM;
1235
1236                 core_imc_refc = kcalloc(nr_cores, sizeof(struct imc_pmu_ref),
1237                                                                 GFP_KERNEL);
1238
1239                 if (!core_imc_refc)
1240                         return -ENOMEM;
1241
1242                 core_imc_pmu = pmu_ptr;
1243                 break;
1244         case IMC_DOMAIN_THREAD:
1245                 /* Update the pmu name */
1246                 pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
1247                 if (!pmu_ptr->pmu.name)
1248                         return -ENOMEM;
1249
1250                 thread_imc_mem_size = pmu_ptr->counter_mem_size;
1251                 for_each_online_cpu(cpu) {
1252                         res = thread_imc_mem_alloc(cpu, pmu_ptr->counter_mem_size);
1253                         if (res)
1254                                 return res;
1255                 }
1256
1257                 thread_imc_pmu = pmu_ptr;
1258                 break;
1259         default:
1260                 return -EINVAL;
1261         }
1262
1263         return 0;
1264 }
1265
1266 /*
1267  * init_imc_pmu : Setup and register the IMC pmu device.
1268  *
1269  * @parent:     Device tree unit node
1270  * @pmu_ptr:    memory allocated for this pmu
1271  * @pmu_idx:    Count of nest pmc registered
1272  *
1273  * init_imc_pmu() setup pmu cpumask and registers for a cpu hotplug callback.
1274  * Handles failure cases and accordingly frees memory.
1275  */
1276 int init_imc_pmu(struct device_node *parent, struct imc_pmu *pmu_ptr, int pmu_idx)
1277 {
1278         int ret;
1279
1280         ret = imc_mem_init(pmu_ptr, parent, pmu_idx);
1281         if (ret)
1282                 goto err_free;
1283
1284         switch (pmu_ptr->domain) {
1285         case IMC_DOMAIN_NEST:
1286                 /*
1287                 * Nest imc pmu need only one cpu per chip, we initialize the
1288                 * cpumask for the first nest imc pmu and use the same for the
1289                 * rest. To handle the cpuhotplug callback unregister, we track
1290                 * the number of nest pmus in "nest_pmus".
1291                 */
1292                 mutex_lock(&nest_init_lock);
1293                 if (nest_pmus == 0) {
1294                         ret = init_nest_pmu_ref();
1295                         if (ret) {
1296                                 mutex_unlock(&nest_init_lock);
1297                                 goto err_free;
1298                         }
1299                         /* Register for cpu hotplug notification. */
1300                         ret = nest_pmu_cpumask_init();
1301                         if (ret) {
1302                                 mutex_unlock(&nest_init_lock);
1303                                 goto err_free;
1304                         }
1305                 }
1306                 nest_pmus++;
1307                 mutex_unlock(&nest_init_lock);
1308                 break;
1309         case IMC_DOMAIN_CORE:
1310                 ret = core_imc_pmu_cpumask_init();
1311                 if (ret) {
1312                         cleanup_all_core_imc_memory();
1313                         return ret;
1314                 }
1315
1316                 break;
1317         case IMC_DOMAIN_THREAD:
1318                 ret = thread_imc_cpu_init();
1319                 if (ret) {
1320                         cleanup_all_thread_imc_memory();
1321                         return ret;
1322                 }
1323
1324                 break;
1325         default:
1326                 return  -1;     /* Unknown domain */
1327         }
1328
1329         ret = update_events_in_group(parent, pmu_ptr);
1330         if (ret)
1331                 goto err_free;
1332
1333         ret = update_pmu_ops(pmu_ptr);
1334         if (ret)
1335                 goto err_free;
1336
1337         ret = perf_pmu_register(&pmu_ptr->pmu, pmu_ptr->pmu.name, -1);
1338         if (ret)
1339                 goto err_free;
1340
1341         pr_info("%s performance monitor hardware support registered\n",
1342                                                         pmu_ptr->pmu.name);
1343
1344         return 0;
1345
1346 err_free:
1347         imc_common_cpuhp_mem_free(pmu_ptr);
1348         return ret;
1349 }