Merge branch 'x86-tsx-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...
[sfrench/cifs-2.6.git] / arch / x86 / events / intel / core.c
index 7ec265bacb6a2622ceba7d42dcc98ebca7b2ec31..35102ecdfc8d5debd884581bb1d40e2a9029a59f 100644 (file)
@@ -2000,6 +2000,39 @@ static void intel_pmu_nhm_enable_all(int added)
        intel_pmu_enable_all(added);
 }
 
+static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on)
+{
+       u64 val = on ? MSR_TFA_RTM_FORCE_ABORT : 0;
+
+       if (cpuc->tfa_shadow != val) {
+               cpuc->tfa_shadow = val;
+               wrmsrl(MSR_TSX_FORCE_ABORT, val);
+       }
+}
+
+static void intel_tfa_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)
+{
+       /*
+        * We're going to use PMC3, make sure TFA is set before we touch it.
+        */
+       if (cntr == 3 && !cpuc->is_fake)
+               intel_set_tfa(cpuc, true);
+}
+
+static void intel_tfa_pmu_enable_all(int added)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+       /*
+        * If we find PMC3 is no longer used when we enable the PMU, we can
+        * clear TFA.
+        */
+       if (!test_bit(3, cpuc->active_mask))
+               intel_set_tfa(cpuc, false);
+
+       intel_pmu_enable_all(added);
+}
+
 static void enable_counter_freeze(void)
 {
        update_debugctlmsr(get_debugctlmsr() |
@@ -2769,6 +2802,35 @@ intel_stop_scheduling(struct cpu_hw_events *cpuc)
        raw_spin_unlock(&excl_cntrs->lock);
 }
 
+static struct event_constraint *
+dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx)
+{
+       WARN_ON_ONCE(!cpuc->constraint_list);
+
+       if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) {
+               struct event_constraint *cx;
+
+               /*
+                * grab pre-allocated constraint entry
+                */
+               cx = &cpuc->constraint_list[idx];
+
+               /*
+                * initialize dynamic constraint
+                * with static constraint
+                */
+               *cx = *c;
+
+               /*
+                * mark constraint as dynamic
+                */
+               cx->flags |= PERF_X86_EVENT_DYNAMIC;
+               c = cx;
+       }
+
+       return c;
+}
+
 static struct event_constraint *
 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
                           int idx, struct event_constraint *c)
@@ -2799,27 +2861,7 @@ intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
         * only needed when constraint has not yet
         * been cloned (marked dynamic)
         */
-       if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) {
-               struct event_constraint *cx;
-
-               /*
-                * grab pre-allocated constraint entry
-                */
-               cx = &cpuc->constraint_list[idx];
-
-               /*
-                * initialize dynamic constraint
-                * with static constraint
-                */
-               *cx = *c;
-
-               /*
-                * mark constraint as dynamic, so we
-                * can free it later on
-                */
-               cx->flags |= PERF_X86_EVENT_DYNAMIC;
-               c = cx;
-       }
+       c = dyn_constraint(cpuc, c, idx);
 
        /*
         * From here on, the constraint is dynamic.
@@ -3357,6 +3399,26 @@ glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
        return c;
 }
 
+static bool allow_tsx_force_abort = true;
+
+static struct event_constraint *
+tfa_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
+                         struct perf_event *event)
+{
+       struct event_constraint *c = hsw_get_event_constraints(cpuc, idx, event);
+
+       /*
+        * Without TFA we must not use PMC3.
+        */
+       if (!allow_tsx_force_abort && test_bit(3, c->idxmsk)) {
+               c = dyn_constraint(cpuc, c, idx);
+               c->idxmsk64 &= ~(1ULL << 3);
+               c->weight--;
+       }
+
+       return c;
+}
+
 /*
  * Broadwell:
  *
@@ -3410,7 +3472,7 @@ ssize_t intel_event_sysfs_show(char *page, u64 config)
        return x86_event_sysfs_show(page, config, event);
 }
 
-struct intel_shared_regs *allocate_shared_regs(int cpu)
+static struct intel_shared_regs *allocate_shared_regs(int cpu)
 {
        struct intel_shared_regs *regs;
        int i;
@@ -3442,23 +3504,24 @@ static struct intel_excl_cntrs *allocate_excl_cntrs(int cpu)
        return c;
 }
 
-static int intel_pmu_cpu_prepare(int cpu)
-{
-       struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 
+int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
+{
        if (x86_pmu.extra_regs || x86_pmu.lbr_sel_map) {
                cpuc->shared_regs = allocate_shared_regs(cpu);
                if (!cpuc->shared_regs)
                        goto err;
        }
 
-       if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
+       if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA)) {
                size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
 
-               cpuc->constraint_list = kzalloc(sz, GFP_KERNEL);
+               cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
                if (!cpuc->constraint_list)
                        goto err_shared_regs;
+       }
 
+       if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
                cpuc->excl_cntrs = allocate_excl_cntrs(cpu);
                if (!cpuc->excl_cntrs)
                        goto err_constraint_list;
@@ -3480,6 +3543,11 @@ err:
        return -ENOMEM;
 }
 
+static int intel_pmu_cpu_prepare(int cpu)
+{
+       return intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu);
+}
+
 static void flip_smm_bit(void *data)
 {
        unsigned long set = *(unsigned long *)data;
@@ -3554,9 +3622,8 @@ static void intel_pmu_cpu_starting(int cpu)
        }
 }
 
-static void free_excl_cntrs(int cpu)
+static void free_excl_cntrs(struct cpu_hw_events *cpuc)
 {
-       struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
        struct intel_excl_cntrs *c;
 
        c = cpuc->excl_cntrs;
@@ -3564,9 +3631,10 @@ static void free_excl_cntrs(int cpu)
                if (c->core_id == -1 || --c->refcnt == 0)
                        kfree(c);
                cpuc->excl_cntrs = NULL;
-               kfree(cpuc->constraint_list);
-               cpuc->constraint_list = NULL;
        }
+
+       kfree(cpuc->constraint_list);
+       cpuc->constraint_list = NULL;
 }
 
 static void intel_pmu_cpu_dying(int cpu)
@@ -3577,9 +3645,8 @@ static void intel_pmu_cpu_dying(int cpu)
                disable_counter_freeze();
 }
 
-static void intel_pmu_cpu_dead(int cpu)
+void intel_cpuc_finish(struct cpu_hw_events *cpuc)
 {
-       struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
        struct intel_shared_regs *pc;
 
        pc = cpuc->shared_regs;
@@ -3589,7 +3656,12 @@ static void intel_pmu_cpu_dead(int cpu)
                cpuc->shared_regs = NULL;
        }
 
-       free_excl_cntrs(cpu);
+       free_excl_cntrs(cpuc);
+}
+
+static void intel_pmu_cpu_dead(int cpu)
+{
+       intel_cpuc_finish(&per_cpu(cpu_hw_events, cpu));
 }
 
 static void intel_pmu_sched_task(struct perf_event_context *ctx,
@@ -4107,8 +4179,11 @@ static struct attribute *intel_pmu_caps_attrs[] = {
        NULL
 };
 
+DEVICE_BOOL_ATTR(allow_tsx_force_abort, 0644, allow_tsx_force_abort);
+
 static struct attribute *intel_pmu_attrs[] = {
        &dev_attr_freeze_on_smi.attr,
+       NULL, /* &dev_attr_allow_tsx_force_abort.attr.attr */
        NULL,
 };
 
@@ -4607,6 +4682,15 @@ __init int intel_pmu_init(void)
                tsx_attr = hsw_tsx_events_attrs;
                intel_pmu_pebs_data_source_skl(
                        boot_cpu_data.x86_model == INTEL_FAM6_SKYLAKE_X);
+
+               if (boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
+                       x86_pmu.flags |= PMU_FL_TFA;
+                       x86_pmu.get_event_constraints = tfa_get_event_constraints;
+                       x86_pmu.enable_all = intel_tfa_pmu_enable_all;
+                       x86_pmu.commit_scheduling = intel_tfa_commit_scheduling;
+                       intel_pmu_attrs[1] = &dev_attr_allow_tsx_force_abort.attr.attr;
+               }
+
                pr_cont("Skylake events, ");
                name = "skylake";
                break;
@@ -4758,7 +4842,7 @@ static __init int fixup_ht_bug(void)
        hardlockup_detector_perf_restart();
 
        for_each_online_cpu(c)
-               free_excl_cntrs(c);
+               free_excl_cntrs(&per_cpu(cpu_hw_events, c));
 
        cpus_read_unlock();
        pr_info("PMU erratum BJ122, BV98, HSD29 workaround disabled, HT off\n");