1 // SPDX-License-Identifier: GPL-2.0
3 * Intel Speed Select Interface: Common functions
4 * Copyright (c) 2019, Intel Corporation.
7 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
10 #include <linux/cpufeature.h>
11 #include <linux/cpuhotplug.h>
13 #include <linux/hashtable.h>
14 #include <linux/miscdevice.h>
15 #include <linux/module.h>
16 #include <linux/pci.h>
17 #include <linux/sched/signal.h>
18 #include <linux/slab.h>
19 #include <linux/uaccess.h>
20 #include <uapi/linux/isst_if.h>
22 #include "isst_if_common.h"
24 #define MSR_THREAD_ID_INFO 0x53
25 #define MSR_CPU_BUS_NUMBER 0x128
27 static struct isst_if_cmd_cb punit_callbacks[ISST_IF_DEV_MAX];
29 static int punit_msr_white_list[] = {
30 MSR_TURBO_RATIO_LIMIT,
31 MSR_CONFIG_TDP_CONTROL,
32 MSR_TURBO_RATIO_LIMIT1,
33 MSR_TURBO_RATIO_LIMIT2,
36 struct isst_valid_cmd_ranges {
42 struct isst_cmd_set_req_type {
48 static const struct isst_valid_cmd_ranges isst_valid_cmds[] = {
57 static const struct isst_cmd_set_req_type isst_cmd_set_reqs[] = {
68 struct hlist_node hnode;
76 static DECLARE_HASHTABLE(isst_hash, 8);
77 static DEFINE_MUTEX(isst_hash_lock);
79 static int isst_store_new_cmd(int cmd, u32 cpu, int mbox_cmd_type, u32 param,
82 struct isst_cmd *sst_cmd;
84 sst_cmd = kmalloc(sizeof(*sst_cmd), GFP_KERNEL);
90 sst_cmd->mbox_cmd_type = mbox_cmd_type;
91 sst_cmd->param = param;
94 hash_add(isst_hash, &sst_cmd->hnode, sst_cmd->cmd);
99 static void isst_delete_hash(void)
101 struct isst_cmd *sst_cmd;
102 struct hlist_node *tmp;
105 hash_for_each_safe(isst_hash, i, tmp, sst_cmd, hnode) {
106 hash_del(&sst_cmd->hnode);
112 * isst_store_cmd() - Store command to a hash table
113 * @cmd: Mailbox command.
114 * @sub_cmd: Mailbox sub-command or MSR id.
115 * @mbox_cmd_type: Mailbox or MSR command.
116 * @param: Mailbox parameter.
117 * @data: Mailbox request data or MSR data.
119 * Stores the command to a hash table if there is no such command already
120 * stored. If already stored update the latest parameter and data for the
123 * Return: Return result of store to hash table, 0 for success, others for
126 int isst_store_cmd(int cmd, int sub_cmd, u32 cpu, int mbox_cmd_type,
129 struct isst_cmd *sst_cmd;
132 full_cmd = (cmd & GENMASK_ULL(15, 0)) << 16;
133 full_cmd |= (sub_cmd & GENMASK_ULL(15, 0));
134 mutex_lock(&isst_hash_lock);
135 hash_for_each_possible(isst_hash, sst_cmd, hnode, full_cmd) {
136 if (sst_cmd->cmd == full_cmd && sst_cmd->cpu == cpu &&
137 sst_cmd->mbox_cmd_type == mbox_cmd_type) {
138 sst_cmd->param = param;
139 sst_cmd->data = data;
140 mutex_unlock(&isst_hash_lock);
145 ret = isst_store_new_cmd(full_cmd, cpu, mbox_cmd_type, param, data);
146 mutex_unlock(&isst_hash_lock);
150 EXPORT_SYMBOL_GPL(isst_store_cmd);
152 static void isst_mbox_resume_command(struct isst_if_cmd_cb *cb,
153 struct isst_cmd *sst_cmd)
155 struct isst_if_mbox_cmd mbox_cmd;
158 mbox_cmd.command = (sst_cmd->cmd & GENMASK_ULL(31, 16)) >> 16;
159 mbox_cmd.sub_command = sst_cmd->cmd & GENMASK_ULL(15, 0);
160 mbox_cmd.parameter = sst_cmd->param;
161 mbox_cmd.req_data = sst_cmd->data;
162 mbox_cmd.logical_cpu = sst_cmd->cpu;
163 (cb->cmd_callback)((u8 *)&mbox_cmd, &wr_only, 1);
167 * isst_resume_common() - Process Resume request
169 * On resume replay all mailbox commands and MSRs.
173 void isst_resume_common(void)
175 struct isst_cmd *sst_cmd;
178 hash_for_each(isst_hash, i, sst_cmd, hnode) {
179 struct isst_if_cmd_cb *cb;
181 if (sst_cmd->mbox_cmd_type) {
182 cb = &punit_callbacks[ISST_IF_DEV_MBOX];
184 isst_mbox_resume_command(cb, sst_cmd);
186 wrmsrl_safe_on_cpu(sst_cmd->cpu, sst_cmd->cmd,
191 EXPORT_SYMBOL_GPL(isst_resume_common);
193 static void isst_restore_msr_local(int cpu)
195 struct isst_cmd *sst_cmd;
198 mutex_lock(&isst_hash_lock);
199 for (i = 0; i < ARRAY_SIZE(punit_msr_white_list); ++i) {
200 if (!punit_msr_white_list[i])
203 hash_for_each_possible(isst_hash, sst_cmd, hnode,
204 punit_msr_white_list[i]) {
205 if (!sst_cmd->mbox_cmd_type && sst_cmd->cpu == cpu)
206 wrmsrl_safe(sst_cmd->cmd, sst_cmd->data);
209 mutex_unlock(&isst_hash_lock);
213 * isst_if_mbox_cmd_invalid() - Check invalid mailbox commands
214 * @cmd: Pointer to the command structure to verify.
216 * Invalid command to PUNIT to may result in instability of the platform.
217 * This function has a whitelist of commands, which are allowed.
219 * Return: Return true if the command is invalid, else false.
221 bool isst_if_mbox_cmd_invalid(struct isst_if_mbox_cmd *cmd)
225 if (cmd->logical_cpu >= nr_cpu_ids)
228 for (i = 0; i < ARRAY_SIZE(isst_valid_cmds); ++i) {
229 if (cmd->command == isst_valid_cmds[i].cmd &&
230 (cmd->sub_command >= isst_valid_cmds[i].sub_cmd_beg &&
231 cmd->sub_command <= isst_valid_cmds[i].sub_cmd_end)) {
238 EXPORT_SYMBOL_GPL(isst_if_mbox_cmd_invalid);
241 * isst_if_mbox_cmd_set_req() - Check mailbox command is a set request
242 * @cmd: Pointer to the command structure to verify.
244 * Check if the given mail box level is set request and not a get request.
246 * Return: Return true if the command is set_req, else false.
248 bool isst_if_mbox_cmd_set_req(struct isst_if_mbox_cmd *cmd)
252 for (i = 0; i < ARRAY_SIZE(isst_cmd_set_reqs); ++i) {
253 if (cmd->command == isst_cmd_set_reqs[i].cmd &&
254 cmd->sub_command == isst_cmd_set_reqs[i].sub_cmd &&
255 cmd->parameter == isst_cmd_set_reqs[i].param) {
262 EXPORT_SYMBOL_GPL(isst_if_mbox_cmd_set_req);
264 static int isst_if_get_platform_info(void __user *argp)
266 struct isst_if_platform_info info;
268 info.api_version = ISST_IF_API_VERSION;
269 info.driver_version = ISST_IF_DRIVER_VERSION;
270 info.max_cmds_per_ioctl = ISST_IF_CMD_LIMIT;
271 info.mbox_supported = punit_callbacks[ISST_IF_DEV_MBOX].registered;
272 info.mmio_supported = punit_callbacks[ISST_IF_DEV_MMIO].registered;
274 if (copy_to_user(argp, &info, sizeof(info)))
281 struct isst_if_cpu_info {
282 /* For BUS 0 and BUS 1 only, which we need for PUNIT interface */
284 struct pci_dev *pci_dev[2];
289 static struct isst_if_cpu_info *isst_cpu_info;
290 #define ISST_MAX_PCI_DOMAINS 8
292 static struct pci_dev *_isst_if_get_pci_dev(int cpu, int bus_no, int dev, int fn)
294 struct pci_dev *matched_pci_dev = NULL;
295 struct pci_dev *pci_dev = NULL;
299 if (bus_no < 0 || bus_no > 1 || cpu < 0 || cpu >= nr_cpu_ids ||
300 cpu >= num_possible_cpus())
303 bus_number = isst_cpu_info[cpu].bus_info[bus_no];
307 for (i = 0; i < ISST_MAX_PCI_DOMAINS; ++i) {
308 struct pci_dev *_pci_dev;
311 _pci_dev = pci_get_domain_bus_and_slot(i, bus_number, PCI_DEVFN(dev, fn));
316 if (!matched_pci_dev)
317 matched_pci_dev = _pci_dev;
319 node = dev_to_node(&_pci_dev->dev);
320 if (node == NUMA_NO_NODE) {
321 pr_info("Fail to get numa node for CPU:%d bus:%d dev:%d fn:%d\n",
322 cpu, bus_no, dev, fn);
326 if (node == isst_cpu_info[cpu].numa_node) {
333 * If there is no numa matched pci_dev, then there can be following cases:
334 * 1. CONFIG_NUMA is not defined: In this case if there is only single device
335 * match, then we don't need numa information. Simply return last match.
336 * Othewise return NULL.
337 * 2. NUMA information is not exposed via _SEG method. In this case it is similar
339 * 3. Numa information doesn't match with CPU numa node and more than one match
342 if (!pci_dev && no_matches == 1)
343 pci_dev = matched_pci_dev;
349 * isst_if_get_pci_dev() - Get the PCI device instance for a CPU
350 * @cpu: Logical CPU number.
351 * @bus_number: The bus number assigned by the hardware.
352 * @dev: The device number assigned by the hardware.
353 * @fn: The function number assigned by the hardware.
355 * Using cached bus information, find out the PCI device for a bus number,
356 * device and function.
358 * Return: Return pci_dev pointer or NULL.
360 struct pci_dev *isst_if_get_pci_dev(int cpu, int bus_no, int dev, int fn)
362 struct pci_dev *pci_dev;
364 if (bus_no < 0 || bus_no > 1 || cpu < 0 || cpu >= nr_cpu_ids ||
365 cpu >= num_possible_cpus())
368 pci_dev = isst_cpu_info[cpu].pci_dev[bus_no];
370 if (pci_dev && pci_dev->devfn == PCI_DEVFN(dev, fn))
373 return _isst_if_get_pci_dev(cpu, bus_no, dev, fn);
375 EXPORT_SYMBOL_GPL(isst_if_get_pci_dev);
377 static int isst_if_cpu_online(unsigned int cpu)
382 isst_cpu_info[cpu].numa_node = cpu_to_node(cpu);
384 ret = rdmsrl_safe(MSR_CPU_BUS_NUMBER, &data);
386 /* This is not a fatal error on MSR mailbox only I/F */
387 isst_cpu_info[cpu].bus_info[0] = -1;
388 isst_cpu_info[cpu].bus_info[1] = -1;
390 isst_cpu_info[cpu].bus_info[0] = data & 0xff;
391 isst_cpu_info[cpu].bus_info[1] = (data >> 8) & 0xff;
392 isst_cpu_info[cpu].pci_dev[0] = _isst_if_get_pci_dev(cpu, 0, 0, 1);
393 isst_cpu_info[cpu].pci_dev[1] = _isst_if_get_pci_dev(cpu, 1, 30, 1);
396 ret = rdmsrl_safe(MSR_THREAD_ID_INFO, &data);
398 isst_cpu_info[cpu].punit_cpu_id = -1;
401 isst_cpu_info[cpu].punit_cpu_id = data;
403 isst_restore_msr_local(cpu);
408 static int isst_if_online_id;
410 static int isst_if_cpu_info_init(void)
414 isst_cpu_info = kcalloc(num_possible_cpus(),
415 sizeof(*isst_cpu_info),
420 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
421 "platform/x86/isst-if:online",
422 isst_if_cpu_online, NULL);
424 kfree(isst_cpu_info);
428 isst_if_online_id = ret;
433 static void isst_if_cpu_info_exit(void)
435 cpuhp_remove_state(isst_if_online_id);
436 kfree(isst_cpu_info);
439 static long isst_if_proc_phyid_req(u8 *cmd_ptr, int *write_only, int resume)
441 struct isst_if_cpu_map *cpu_map;
443 cpu_map = (struct isst_if_cpu_map *)cmd_ptr;
444 if (cpu_map->logical_cpu >= nr_cpu_ids ||
445 cpu_map->logical_cpu >= num_possible_cpus())
449 cpu_map->physical_cpu = isst_cpu_info[cpu_map->logical_cpu].punit_cpu_id;
454 static bool match_punit_msr_white_list(int msr)
458 for (i = 0; i < ARRAY_SIZE(punit_msr_white_list); ++i) {
459 if (punit_msr_white_list[i] == msr)
466 static long isst_if_msr_cmd_req(u8 *cmd_ptr, int *write_only, int resume)
468 struct isst_if_msr_cmd *msr_cmd;
471 msr_cmd = (struct isst_if_msr_cmd *)cmd_ptr;
473 if (!match_punit_msr_white_list(msr_cmd->msr))
476 if (msr_cmd->logical_cpu >= nr_cpu_ids)
479 if (msr_cmd->read_write) {
480 if (!capable(CAP_SYS_ADMIN))
483 ret = wrmsrl_safe_on_cpu(msr_cmd->logical_cpu,
488 ret = isst_store_cmd(0, msr_cmd->msr,
489 msr_cmd->logical_cpu,
490 0, 0, msr_cmd->data);
494 ret = rdmsrl_safe_on_cpu(msr_cmd->logical_cpu,
495 msr_cmd->msr, &data);
497 msr_cmd->data = data;
506 static long isst_if_exec_multi_cmd(void __user *argp, struct isst_if_cmd_cb *cb)
508 unsigned char __user *ptr;
514 /* Each multi command has u32 command count as the first field */
515 if (copy_from_user(&cmd_count, argp, sizeof(cmd_count)))
518 if (!cmd_count || cmd_count > ISST_IF_CMD_LIMIT)
521 cmd_ptr = kmalloc(cb->cmd_size, GFP_KERNEL);
525 /* cb->offset points to start of the command after the command count */
526 ptr = argp + cb->offset;
528 for (i = 0; i < cmd_count; ++i) {
531 if (signal_pending(current)) {
536 if (copy_from_user(cmd_ptr, ptr, cb->cmd_size)) {
541 ret = cb->cmd_callback(cmd_ptr, &wr_only, 0);
545 if (!wr_only && copy_to_user(ptr, cmd_ptr, cb->cmd_size)) {
558 static long isst_if_def_ioctl(struct file *file, unsigned int cmd,
561 void __user *argp = (void __user *)arg;
562 struct isst_if_cmd_cb cmd_cb;
563 struct isst_if_cmd_cb *cb;
567 case ISST_IF_GET_PLATFORM_INFO:
568 ret = isst_if_get_platform_info(argp);
570 case ISST_IF_GET_PHY_ID:
571 cmd_cb.cmd_size = sizeof(struct isst_if_cpu_map);
572 cmd_cb.offset = offsetof(struct isst_if_cpu_maps, cpu_map);
573 cmd_cb.cmd_callback = isst_if_proc_phyid_req;
574 ret = isst_if_exec_multi_cmd(argp, &cmd_cb);
577 cb = &punit_callbacks[ISST_IF_DEV_MMIO];
579 ret = isst_if_exec_multi_cmd(argp, cb);
581 case ISST_IF_MBOX_COMMAND:
582 cb = &punit_callbacks[ISST_IF_DEV_MBOX];
584 ret = isst_if_exec_multi_cmd(argp, cb);
586 case ISST_IF_MSR_COMMAND:
587 cmd_cb.cmd_size = sizeof(struct isst_if_msr_cmd);
588 cmd_cb.offset = offsetof(struct isst_if_msr_cmds, msr_cmd);
589 cmd_cb.cmd_callback = isst_if_msr_cmd_req;
590 ret = isst_if_exec_multi_cmd(argp, &cmd_cb);
599 static DEFINE_MUTEX(punit_misc_dev_lock);
600 static int misc_usage_count;
601 static int misc_device_ret;
602 static int misc_device_open;
604 static int isst_if_open(struct inode *inode, struct file *file)
608 /* Fail open, if a module is going away */
609 mutex_lock(&punit_misc_dev_lock);
610 for (i = 0; i < ISST_IF_DEV_MAX; ++i) {
611 struct isst_if_cmd_cb *cb = &punit_callbacks[i];
613 if (cb->registered && !try_module_get(cb->owner)) {
621 for (j = 0; j < i; ++j) {
622 struct isst_if_cmd_cb *cb;
624 cb = &punit_callbacks[j];
626 module_put(cb->owner);
631 mutex_unlock(&punit_misc_dev_lock);
636 static int isst_if_relase(struct inode *inode, struct file *f)
640 mutex_lock(&punit_misc_dev_lock);
642 for (i = 0; i < ISST_IF_DEV_MAX; ++i) {
643 struct isst_if_cmd_cb *cb = &punit_callbacks[i];
646 module_put(cb->owner);
648 mutex_unlock(&punit_misc_dev_lock);
653 static const struct file_operations isst_if_char_driver_ops = {
654 .open = isst_if_open,
655 .unlocked_ioctl = isst_if_def_ioctl,
656 .release = isst_if_relase,
659 static struct miscdevice isst_if_char_driver = {
660 .minor = MISC_DYNAMIC_MINOR,
661 .name = "isst_interface",
662 .fops = &isst_if_char_driver_ops,
666 * isst_if_cdev_register() - Register callback for IOCTL
667 * @device_type: The device type this callback handling.
668 * @cb: Callback structure.
670 * This function registers a callback to device type. On very first call
671 * it will register a misc device, which is used for user kernel interface.
672 * Other calls simply increment ref count. Registry will fail, if the user
673 * already opened misc device for operation. Also if the misc device
674 * creation failed, then it will not try again and all callers will get
677 * Return: Return the return value from the misc creation device or -EINVAL
678 * for unsupported device type.
680 int isst_if_cdev_register(int device_type, struct isst_if_cmd_cb *cb)
683 return misc_device_ret;
685 if (device_type >= ISST_IF_DEV_MAX)
688 mutex_lock(&punit_misc_dev_lock);
689 if (misc_device_open) {
690 mutex_unlock(&punit_misc_dev_lock);
693 if (!misc_usage_count) {
696 misc_device_ret = misc_register(&isst_if_char_driver);
700 ret = isst_if_cpu_info_init();
702 misc_deregister(&isst_if_char_driver);
703 misc_device_ret = ret;
707 memcpy(&punit_callbacks[device_type], cb, sizeof(*cb));
708 punit_callbacks[device_type].registered = 1;
711 mutex_unlock(&punit_misc_dev_lock);
713 return misc_device_ret;
715 EXPORT_SYMBOL_GPL(isst_if_cdev_register);
718 * isst_if_cdev_unregister() - Unregister callback for IOCTL
719 * @device_type: The device type to unregister.
721 * This function unregisters the previously registered callback. If this
722 * is the last callback unregistering, then misc device is removed.
726 void isst_if_cdev_unregister(int device_type)
728 mutex_lock(&punit_misc_dev_lock);
730 punit_callbacks[device_type].registered = 0;
731 if (device_type == ISST_IF_DEV_MBOX)
733 if (!misc_usage_count && !misc_device_ret) {
734 misc_deregister(&isst_if_char_driver);
735 isst_if_cpu_info_exit();
737 mutex_unlock(&punit_misc_dev_lock);
739 EXPORT_SYMBOL_GPL(isst_if_cdev_unregister);
741 MODULE_LICENSE("GPL v2");