2 * CXL Flash Device Driver
4 * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation
5 * Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation
7 * Copyright (C) 2018 IBM Corporation
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version
12 * 2 of the License, or (at your option) any later version.
15 #include <linux/file.h>
16 #include <linux/idr.h>
17 #include <linux/module.h>
18 #include <linux/mount.h>
19 #include <linux/poll.h>
20 #include <linux/sched/signal.h>
22 #include <misc/ocxl.h>
24 #include <uapi/misc/cxl.h>
30 * Pseudo-filesystem to allocate inodes.
33 #define OCXLFLASH_FS_MAGIC 0x1697698f
35 static int ocxlflash_fs_cnt;
36 static struct vfsmount *ocxlflash_vfs_mount;
38 static const struct dentry_operations ocxlflash_fs_dops = {
39 .d_dname = simple_dname,
43 * ocxlflash_fs_mount() - mount the pseudo-filesystem
44 * @fs_type: File system type.
45 * @flags: Flags for the filesystem.
46 * @dev_name: Device name associated with the filesystem.
47 * @data: Data pointer.
49 * Return: pointer to the directory entry structure
51 static struct dentry *ocxlflash_fs_mount(struct file_system_type *fs_type,
52 int flags, const char *dev_name,
55 return mount_pseudo(fs_type, "ocxlflash:", NULL, &ocxlflash_fs_dops,
59 static struct file_system_type ocxlflash_fs_type = {
62 .mount = ocxlflash_fs_mount,
63 .kill_sb = kill_anon_super,
67 * ocxlflash_release_mapping() - release the memory mapping
68 * @ctx: Context whose mapping is to be released.
70 static void ocxlflash_release_mapping(struct ocxlflash_context *ctx)
73 simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
78 * ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file
79 * @dev: Generic device of the host.
80 * @name: Name of the pseudo filesystem.
81 * @fops: File operations.
82 * @priv: Private data.
83 * @flags: Flags for the file.
85 * Return: pointer to the file on success, ERR_PTR on failure
87 static struct file *ocxlflash_getfile(struct device *dev, const char *name,
88 const struct file_operations *fops,
89 void *priv, int flags)
94 struct inode *inode = NULL;
97 if (fops->owner && !try_module_get(fops->owner)) {
98 dev_err(dev, "%s: Owner does not exist\n", __func__);
103 rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount,
105 if (unlikely(rc < 0)) {
106 dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n",
111 inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb);
114 dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n",
120 this.len = strlen(name);
122 path.dentry = d_alloc_pseudo(ocxlflash_vfs_mount->mnt_sb, &this);
124 dev_err(dev, "%s: d_alloc_pseudo failed\n", __func__);
129 path.mnt = mntget(ocxlflash_vfs_mount);
130 d_instantiate(path.dentry, inode);
132 file = alloc_file(&path, OPEN_FMODE(flags), fops);
135 dev_err(dev, "%s: alloc_file failed rc=%d\n",
141 file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
142 file->private_data = priv;
148 simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt);
150 module_put(fops->owner);
157 * ocxlflash_psa_map() - map the process specific MMIO space
158 * @ctx_cookie: Adapter context for which the mapping needs to be done.
160 * Return: MMIO pointer of the mapped region
162 static void __iomem *ocxlflash_psa_map(void *ctx_cookie)
164 struct ocxlflash_context *ctx = ctx_cookie;
165 struct device *dev = ctx->hw_afu->dev;
167 mutex_lock(&ctx->state_mutex);
168 if (ctx->state != STARTED) {
169 dev_err(dev, "%s: Context not started, state=%d\n", __func__,
171 mutex_unlock(&ctx->state_mutex);
174 mutex_unlock(&ctx->state_mutex);
176 return ioremap(ctx->psn_phys, ctx->psn_size);
180 * ocxlflash_psa_unmap() - unmap the process specific MMIO space
181 * @addr: MMIO pointer to unmap.
183 static void ocxlflash_psa_unmap(void __iomem *addr)
189 * ocxlflash_process_element() - get process element of the adapter context
190 * @ctx_cookie: Adapter context associated with the process element.
192 * Return: process element of the adapter context
194 static int ocxlflash_process_element(void *ctx_cookie)
196 struct ocxlflash_context *ctx = ctx_cookie;
202 * afu_map_irq() - map the interrupt of the adapter context
204 * @ctx: Adapter context.
205 * @num: Per-context AFU interrupt number.
206 * @handler: Interrupt handler to register.
207 * @cookie: Interrupt handler private data.
208 * @name: Name of the interrupt.
210 * Return: 0 on success, -errno on failure
212 static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num,
213 irq_handler_t handler, void *cookie, char *name)
215 struct ocxl_hw_afu *afu = ctx->hw_afu;
216 struct device *dev = afu->dev;
217 struct ocxlflash_irqs *irq;
222 if (num < 0 || num >= ctx->num_irqs) {
223 dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
228 irq = &ctx->irqs[num];
229 virq = irq_create_mapping(NULL, irq->hwirq);
230 if (unlikely(!virq)) {
231 dev_err(dev, "%s: irq_create_mapping failed\n", __func__);
236 rc = request_irq(virq, handler, 0, name, cookie);
238 dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc);
242 vtrig = ioremap(irq->ptrig, PAGE_SIZE);
243 if (unlikely(!vtrig)) {
244 dev_err(dev, "%s: Trigger page mapping failed\n", __func__);
254 free_irq(virq, cookie);
256 irq_dispose_mapping(virq);
261 * ocxlflash_map_afu_irq() - map the interrupt of the adapter context
262 * @ctx_cookie: Adapter context.
263 * @num: Per-context AFU interrupt number.
264 * @handler: Interrupt handler to register.
265 * @cookie: Interrupt handler private data.
266 * @name: Name of the interrupt.
268 * Return: 0 on success, -errno on failure
270 static int ocxlflash_map_afu_irq(void *ctx_cookie, int num,
271 irq_handler_t handler, void *cookie,
274 return afu_map_irq(0, ctx_cookie, num, handler, cookie, name);
278 * afu_unmap_irq() - unmap the interrupt
280 * @ctx: Adapter context.
281 * @num: Per-context AFU interrupt number.
282 * @cookie: Interrupt handler private data.
284 static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num,
287 struct ocxl_hw_afu *afu = ctx->hw_afu;
288 struct device *dev = afu->dev;
289 struct ocxlflash_irqs *irq;
291 if (num < 0 || num >= ctx->num_irqs) {
292 dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num);
296 irq = &ctx->irqs[num];
300 if (irq_find_mapping(NULL, irq->hwirq)) {
301 free_irq(irq->virq, cookie);
302 irq_dispose_mapping(irq->virq);
305 memset(irq, 0, sizeof(*irq));
309 * ocxlflash_unmap_afu_irq() - unmap the interrupt
310 * @ctx_cookie: Adapter context.
311 * @num: Per-context AFU interrupt number.
312 * @cookie: Interrupt handler private data.
314 static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie)
316 return afu_unmap_irq(0, ctx_cookie, num, cookie);
320 * ocxlflash_get_irq_objhndl() - get the object handle for an interrupt
321 * @ctx_cookie: Context associated with the interrupt.
322 * @irq: Interrupt number.
324 * Return: effective address of the mapped region
326 static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq)
328 struct ocxlflash_context *ctx = ctx_cookie;
330 if (irq < 0 || irq >= ctx->num_irqs)
333 return (__force u64)ctx->irqs[irq].vtrig;
337 * ocxlflash_xsl_fault() - callback when translation error is triggered
338 * @data: Private data provided at callback registration, the context.
339 * @addr: Address that triggered the error.
340 * @dsisr: Value of dsisr register.
342 static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr)
344 struct ocxlflash_context *ctx = data;
346 spin_lock(&ctx->slock);
347 ctx->fault_addr = addr;
348 ctx->fault_dsisr = dsisr;
349 ctx->pending_fault = true;
350 spin_unlock(&ctx->slock);
352 wake_up_all(&ctx->wq);
356 * start_context() - local routine to start a context
357 * @ctx: Adapter context to be started.
359 * Assign the context specific MMIO space, add and enable the PE.
361 * Return: 0 on success, -errno on failure
363 static int start_context(struct ocxlflash_context *ctx)
365 struct ocxl_hw_afu *afu = ctx->hw_afu;
366 struct ocxl_afu_config *acfg = &afu->acfg;
367 void *link_token = afu->link_token;
368 struct device *dev = afu->dev;
369 bool master = ctx->master;
370 struct mm_struct *mm;
374 mutex_lock(&ctx->state_mutex);
375 if (ctx->state != OPENED) {
376 dev_err(dev, "%s: Context state invalid, state=%d\n",
377 __func__, ctx->state);
383 ctx->psn_size = acfg->global_mmio_size;
384 ctx->psn_phys = afu->gmmio_phys;
386 ctx->psn_size = acfg->pp_mmio_stride;
387 ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size);
390 /* pid and mm not set for master contexts */
395 pid = current->mm->context.id;
399 rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0, mm,
400 ocxlflash_xsl_fault, ctx);
402 dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n",
407 ctx->state = STARTED;
409 mutex_unlock(&ctx->state_mutex);
414 * ocxlflash_start_context() - start a kernel context
415 * @ctx_cookie: Adapter context to be started.
417 * Return: 0 on success, -errno on failure
419 static int ocxlflash_start_context(void *ctx_cookie)
421 struct ocxlflash_context *ctx = ctx_cookie;
423 return start_context(ctx);
427 * ocxlflash_stop_context() - stop a context
428 * @ctx_cookie: Adapter context to be stopped.
430 * Return: 0 on success, -errno on failure
432 static int ocxlflash_stop_context(void *ctx_cookie)
434 struct ocxlflash_context *ctx = ctx_cookie;
435 struct ocxl_hw_afu *afu = ctx->hw_afu;
436 struct ocxl_afu_config *acfg = &afu->acfg;
437 struct pci_dev *pdev = afu->pdev;
438 struct device *dev = afu->dev;
439 enum ocxlflash_ctx_state state;
442 mutex_lock(&ctx->state_mutex);
445 mutex_unlock(&ctx->state_mutex);
446 if (state != STARTED)
449 rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos,
452 dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n",
454 /* If EBUSY, PE could be referenced in future by the AFU */
459 rc = ocxl_link_remove_pe(afu->link_token, ctx->pe);
461 dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n",
470 * ocxlflash_afu_reset() - reset the AFU
471 * @ctx_cookie: Adapter context.
473 static int ocxlflash_afu_reset(void *ctx_cookie)
475 struct ocxlflash_context *ctx = ctx_cookie;
476 struct device *dev = ctx->hw_afu->dev;
478 /* Pending implementation from OCXL transport services */
479 dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__);
481 /* Silently return success until it is implemented */
486 * ocxlflash_set_master() - sets the context as master
487 * @ctx_cookie: Adapter context to set as master.
489 static void ocxlflash_set_master(void *ctx_cookie)
491 struct ocxlflash_context *ctx = ctx_cookie;
497 * ocxlflash_get_context() - obtains the context associated with the host
498 * @pdev: PCI device associated with the host.
499 * @afu_cookie: Hardware AFU associated with the host.
501 * Return: returns the pointer to host adapter context
503 static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie)
505 struct ocxl_hw_afu *afu = afu_cookie;
507 return afu->ocxl_ctx;
511 * ocxlflash_dev_context_init() - allocate and initialize an adapter context
512 * @pdev: PCI device associated with the host.
513 * @afu_cookie: Hardware AFU associated with the host.
515 * Return: returns the adapter context on success, ERR_PTR on failure
517 static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie)
519 struct ocxl_hw_afu *afu = afu_cookie;
520 struct device *dev = afu->dev;
521 struct ocxlflash_context *ctx;
524 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
525 if (unlikely(!ctx)) {
526 dev_err(dev, "%s: Context allocation failed\n", __func__);
531 idr_preload(GFP_KERNEL);
532 rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT);
534 if (unlikely(rc < 0)) {
535 dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc);
539 spin_lock_init(&ctx->slock);
540 init_waitqueue_head(&ctx->wq);
541 mutex_init(&ctx->state_mutex);
549 ctx->pending_irq = false;
550 ctx->pending_fault = false;
561 * ocxlflash_release_context() - releases an adapter context
562 * @ctx_cookie: Adapter context to be released.
564 * Return: 0 on success, -errno on failure
566 static int ocxlflash_release_context(void *ctx_cookie)
568 struct ocxlflash_context *ctx = ctx_cookie;
575 dev = ctx->hw_afu->dev;
576 mutex_lock(&ctx->state_mutex);
577 if (ctx->state >= STARTED) {
578 dev_err(dev, "%s: Context in use, state=%d\n", __func__,
580 mutex_unlock(&ctx->state_mutex);
584 mutex_unlock(&ctx->state_mutex);
586 idr_remove(&ctx->hw_afu->idr, ctx->pe);
587 ocxlflash_release_mapping(ctx);
594 * ocxlflash_perst_reloads_same_image() - sets the image reload policy
595 * @afu_cookie: Hardware AFU associated with the host.
596 * @image: Whether to load the same image on PERST.
598 static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image)
600 struct ocxl_hw_afu *afu = afu_cookie;
602 afu->perst_same_image = image;
606 * ocxlflash_read_adapter_vpd() - reads the adapter VPD
607 * @pdev: PCI device associated with the host.
608 * @buf: Buffer to get the VPD data.
609 * @count: Size of buffer (maximum bytes that can be read).
611 * Return: size of VPD on success, -errno on failure
613 static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf,
616 return pci_read_vpd(pdev, 0, count, buf);
620 * free_afu_irqs() - internal service to free interrupts
621 * @ctx: Adapter context.
623 static void free_afu_irqs(struct ocxlflash_context *ctx)
625 struct ocxl_hw_afu *afu = ctx->hw_afu;
626 struct device *dev = afu->dev;
630 dev_err(dev, "%s: Interrupts not allocated\n", __func__);
634 for (i = ctx->num_irqs; i >= 0; i--)
635 ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq);
642 * alloc_afu_irqs() - internal service to allocate interrupts
643 * @ctx: Context associated with the request.
644 * @num: Number of interrupts requested.
646 * Return: 0 on success, -errno on failure
648 static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num)
650 struct ocxl_hw_afu *afu = ctx->hw_afu;
651 struct device *dev = afu->dev;
652 struct ocxlflash_irqs *irqs;
659 dev_err(dev, "%s: Interrupts already allocated\n", __func__);
664 if (num > OCXL_MAX_IRQS) {
665 dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num);
670 irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL);
671 if (unlikely(!irqs)) {
672 dev_err(dev, "%s: Context irqs allocation failed\n", __func__);
677 for (i = 0; i < num; i++) {
678 rc = ocxl_link_irq_alloc(afu->link_token, &hwirq, &addr);
680 dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n",
685 irqs[i].hwirq = hwirq;
686 irqs[i].ptrig = addr;
694 for (i = i-1; i >= 0; i--)
695 ocxl_link_free_irq(afu->link_token, irqs[i].hwirq);
701 * ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts
702 * @ctx_cookie: Context associated with the request.
703 * @num: Number of interrupts requested.
705 * Return: 0 on success, -errno on failure
707 static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num)
709 return alloc_afu_irqs(ctx_cookie, num);
713 * ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context
714 * @ctx_cookie: Adapter context.
716 static void ocxlflash_free_afu_irqs(void *ctx_cookie)
718 free_afu_irqs(ctx_cookie);
722 * ocxlflash_unconfig_afu() - unconfigure the AFU
723 * @afu: AFU associated with the host.
725 static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu)
727 if (afu->gmmio_virt) {
728 iounmap(afu->gmmio_virt);
729 afu->gmmio_virt = NULL;
734 * ocxlflash_destroy_afu() - destroy the AFU structure
735 * @afu_cookie: AFU to be freed.
737 static void ocxlflash_destroy_afu(void *afu_cookie)
739 struct ocxl_hw_afu *afu = afu_cookie;
745 ocxlflash_release_context(afu->ocxl_ctx);
746 idr_destroy(&afu->idr);
748 /* Disable the AFU */
749 pos = afu->acfg.dvsec_afu_control_pos;
750 ocxl_config_set_afu_state(afu->pdev, pos, 0);
752 ocxlflash_unconfig_afu(afu);
757 * ocxlflash_config_fn() - configure the host function
758 * @pdev: PCI device associated with the host.
759 * @afu: AFU associated with the host.
761 * Return: 0 on success, -errno on failure
763 static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
765 struct ocxl_fn_config *fcfg = &afu->fcfg;
766 struct device *dev = &pdev->dev;
767 u16 base, enabled, supported;
770 /* Read DVSEC config of the function */
771 rc = ocxl_config_read_function(pdev, fcfg);
773 dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n",
778 /* Check if function has AFUs defined, only 1 per function supported */
779 if (fcfg->max_afu_index >= 0) {
780 afu->is_present = true;
781 if (fcfg->max_afu_index != 0)
782 dev_warn(dev, "%s: Unexpected AFU index value %d\n",
783 __func__, fcfg->max_afu_index);
786 rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported);
788 dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n",
793 afu->fn_actag_base = base;
794 afu->fn_actag_enabled = enabled;
796 ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled);
797 dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n",
798 __func__, base, enabled);
800 rc = ocxl_link_setup(pdev, 0, &afu->link_token);
802 dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n",
807 rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos);
809 dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n",
816 ocxl_link_release(pdev, afu->link_token);
821 * ocxlflash_unconfig_fn() - unconfigure the host function
822 * @pdev: PCI device associated with the host.
823 * @afu: AFU associated with the host.
825 static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
827 ocxl_link_release(pdev, afu->link_token);
831 * ocxlflash_map_mmio() - map the AFU MMIO space
832 * @afu: AFU associated with the host.
834 * Return: 0 on success, -errno on failure
836 static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu)
838 struct ocxl_afu_config *acfg = &afu->acfg;
839 struct pci_dev *pdev = afu->pdev;
840 struct device *dev = afu->dev;
841 phys_addr_t gmmio, ppmmio;
844 rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash");
846 dev_err(dev, "%s: pci_request_region for global failed rc=%d\n",
850 gmmio = pci_resource_start(pdev, acfg->global_mmio_bar);
851 gmmio += acfg->global_mmio_offset;
853 rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash");
855 dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n",
859 ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar);
860 ppmmio += acfg->pp_mmio_offset;
862 afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size);
863 if (unlikely(!afu->gmmio_virt)) {
864 dev_err(dev, "%s: MMIO mapping failed\n", __func__);
869 afu->gmmio_phys = gmmio;
870 afu->ppmmio_phys = ppmmio;
874 pci_release_region(pdev, acfg->pp_mmio_bar);
876 pci_release_region(pdev, acfg->global_mmio_bar);
881 * ocxlflash_config_afu() - configure the host AFU
882 * @pdev: PCI device associated with the host.
883 * @afu: AFU associated with the host.
885 * Must be called _after_ host function configuration.
887 * Return: 0 on success, -errno on failure
889 static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu)
891 struct ocxl_afu_config *acfg = &afu->acfg;
892 struct ocxl_fn_config *fcfg = &afu->fcfg;
893 struct device *dev = &pdev->dev;
899 /* This HW AFU function does not have any AFUs defined */
900 if (!afu->is_present)
903 /* Read AFU config at index 0 */
904 rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0);
906 dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n",
911 /* Only one AFU per function is supported, so actag_base is same */
912 base = afu->fn_actag_base;
913 count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled);
914 pos = acfg->dvsec_afu_control_pos;
916 ocxl_config_set_afu_actag(pdev, pos, base, count);
917 dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count);
918 afu->afu_actag_base = base;
919 afu->afu_actag_enabled = count;
920 afu->max_pasid = 1 << acfg->pasid_supported_log;
922 ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log);
924 rc = ocxlflash_map_mmio(afu);
926 dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n",
932 ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1);
938 * ocxlflash_create_afu() - create the AFU for OCXL
939 * @pdev: PCI device associated with the host.
941 * Return: AFU on success, NULL on failure
943 static void *ocxlflash_create_afu(struct pci_dev *pdev)
945 struct device *dev = &pdev->dev;
946 struct ocxlflash_context *ctx;
947 struct ocxl_hw_afu *afu;
950 afu = kzalloc(sizeof(*afu), GFP_KERNEL);
951 if (unlikely(!afu)) {
952 dev_err(dev, "%s: HW AFU allocation failed\n", __func__);
960 rc = ocxlflash_config_fn(pdev, afu);
962 dev_err(dev, "%s: Function configuration failed rc=%d\n",
967 rc = ocxlflash_config_afu(pdev, afu);
969 dev_err(dev, "%s: AFU configuration failed rc=%d\n",
974 ctx = ocxlflash_dev_context_init(pdev, afu);
977 dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n",
986 ocxlflash_unconfig_afu(afu);
988 ocxlflash_unconfig_fn(pdev, afu);
990 idr_destroy(&afu->idr);
997 * ctx_event_pending() - check for any event pending on the context
998 * @ctx: Context to be checked.
1000 * Return: true if there is an event pending, false if none pending
1002 static inline bool ctx_event_pending(struct ocxlflash_context *ctx)
1004 if (ctx->pending_irq || ctx->pending_fault)
1011 * afu_poll() - poll the AFU for events on the context
1012 * @file: File associated with the adapter context.
1013 * @poll: Poll structure from the user.
1017 static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
1019 struct ocxlflash_context *ctx = file->private_data;
1020 struct device *dev = ctx->hw_afu->dev;
1024 poll_wait(file, &ctx->wq, poll);
1026 spin_lock_irqsave(&ctx->slock, lock_flags);
1027 if (ctx_event_pending(ctx))
1028 mask |= POLLIN | POLLRDNORM;
1029 else if (ctx->state == CLOSED)
1031 spin_unlock_irqrestore(&ctx->slock, lock_flags);
1033 dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n",
1034 __func__, ctx->pe, mask);
1040 * afu_read() - perform a read on the context for any event
1041 * @file: File associated with the adapter context.
1042 * @buf: Buffer to receive the data.
1043 * @count: Size of buffer (maximum bytes that can be read).
1046 * Return: size of the data read on success, -errno on failure
1048 static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
1051 struct ocxlflash_context *ctx = file->private_data;
1052 struct device *dev = ctx->hw_afu->dev;
1053 struct cxl_event event;
1058 DEFINE_WAIT(event_wait);
1061 dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n",
1067 spin_lock_irqsave(&ctx->slock, lock_flags);
1070 prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE);
1072 if (ctx_event_pending(ctx) || (ctx->state == CLOSED))
1075 if (file->f_flags & O_NONBLOCK) {
1076 dev_err(dev, "%s: File cannot be blocked on I/O\n",
1082 if (signal_pending(current)) {
1083 dev_err(dev, "%s: Signal pending on the process\n",
1089 spin_unlock_irqrestore(&ctx->slock, lock_flags);
1091 spin_lock_irqsave(&ctx->slock, lock_flags);
1094 finish_wait(&ctx->wq, &event_wait);
1096 memset(&event, 0, sizeof(event));
1097 event.header.process_element = ctx->pe;
1098 event.header.size = sizeof(struct cxl_event_header);
1099 if (ctx->pending_irq) {
1100 esize = sizeof(struct cxl_event_afu_interrupt);
1101 event.header.size += esize;
1102 event.header.type = CXL_EVENT_AFU_INTERRUPT;
1104 bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs);
1105 clear_bit(bit, &ctx->irq_bitmap);
1106 event.irq.irq = bit + 1;
1107 if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs))
1108 ctx->pending_irq = false;
1109 } else if (ctx->pending_fault) {
1110 event.header.size += sizeof(struct cxl_event_data_storage);
1111 event.header.type = CXL_EVENT_DATA_STORAGE;
1112 event.fault.addr = ctx->fault_addr;
1113 event.fault.dsisr = ctx->fault_dsisr;
1114 ctx->pending_fault = false;
1117 spin_unlock_irqrestore(&ctx->slock, lock_flags);
1119 if (copy_to_user(buf, &event, event.header.size)) {
1120 dev_err(dev, "%s: copy_to_user failed\n", __func__);
1125 rc = event.header.size;
1129 finish_wait(&ctx->wq, &event_wait);
1130 spin_unlock_irqrestore(&ctx->slock, lock_flags);
1135 * afu_release() - release and free the context
1136 * @inode: File inode pointer.
1137 * @file: File associated with the context.
1139 * Return: 0 on success, -errno on failure
1141 static int afu_release(struct inode *inode, struct file *file)
1143 struct ocxlflash_context *ctx = file->private_data;
1146 /* Unmap and free the interrupts associated with the context */
1147 for (i = ctx->num_irqs; i >= 0; i--)
1148 afu_unmap_irq(0, ctx, i, ctx);
1151 return ocxlflash_release_context(ctx);
1155 * ocxlflash_mmap_fault() - mmap fault handler
1156 * @vmf: VM fault associated with current fault.
1158 * Return: 0 on success, -errno on failure
1160 static int ocxlflash_mmap_fault(struct vm_fault *vmf)
1162 struct vm_area_struct *vma = vmf->vma;
1163 struct ocxlflash_context *ctx = vma->vm_file->private_data;
1164 struct device *dev = ctx->hw_afu->dev;
1165 u64 mmio_area, offset;
1167 offset = vmf->pgoff << PAGE_SHIFT;
1168 if (offset >= ctx->psn_size)
1169 return VM_FAULT_SIGBUS;
1171 mutex_lock(&ctx->state_mutex);
1172 if (ctx->state != STARTED) {
1173 dev_err(dev, "%s: Context not started, state=%d\n",
1174 __func__, ctx->state);
1175 mutex_unlock(&ctx->state_mutex);
1176 return VM_FAULT_SIGBUS;
1178 mutex_unlock(&ctx->state_mutex);
1180 mmio_area = ctx->psn_phys;
1181 mmio_area += offset;
1183 vm_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT);
1184 return VM_FAULT_NOPAGE;
1187 static const struct vm_operations_struct ocxlflash_vmops = {
1188 .fault = ocxlflash_mmap_fault,
1192 * afu_mmap() - map the fault handler operations
1193 * @file: File associated with the context.
1194 * @vma: VM area associated with mapping.
1196 * Return: 0 on success, -errno on failure
1198 static int afu_mmap(struct file *file, struct vm_area_struct *vma)
1200 struct ocxlflash_context *ctx = file->private_data;
1202 if ((vma_pages(vma) + vma->vm_pgoff) >
1203 (ctx->psn_size >> PAGE_SHIFT))
1206 vma->vm_flags |= VM_IO | VM_PFNMAP;
1207 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1208 vma->vm_ops = &ocxlflash_vmops;
1212 static const struct file_operations ocxl_afu_fops = {
1213 .owner = THIS_MODULE,
1216 .release = afu_release,
1220 #define PATCH_FOPS(NAME) \
1221 do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0)
1224 * ocxlflash_get_fd() - get file descriptor for an adapter context
1225 * @ctx_cookie: Adapter context.
1226 * @fops: File operations to be associated.
1227 * @fd: File descriptor to be returned back.
1229 * Return: pointer to the file on success, ERR_PTR on failure
1231 static struct file *ocxlflash_get_fd(void *ctx_cookie,
1232 struct file_operations *fops, int *fd)
1234 struct ocxlflash_context *ctx = ctx_cookie;
1235 struct device *dev = ctx->hw_afu->dev;
1241 /* Only allow one fd per context */
1243 dev_err(dev, "%s: Context is already mapped to an fd\n",
1249 flags = O_RDWR | O_CLOEXEC;
1251 /* This code is similar to anon_inode_getfd() */
1252 rc = get_unused_fd_flags(flags);
1253 if (unlikely(rc < 0)) {
1254 dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n",
1260 /* Patch the file ops that are not defined */
1264 PATCH_FOPS(release);
1266 } else /* Use default ops */
1267 fops = (struct file_operations *)&ocxl_afu_fops;
1269 name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe);
1270 file = ocxlflash_getfile(dev, name, fops, ctx, flags);
1274 dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n",
1279 ctx->mapping = file->f_mapping;
1284 put_unused_fd(fdtmp);
1291 * ocxlflash_fops_get_context() - get the context associated with the file
1292 * @file: File associated with the adapter context.
1294 * Return: pointer to the context
1296 static void *ocxlflash_fops_get_context(struct file *file)
1298 return file->private_data;
1302 * ocxlflash_afu_irq() - interrupt handler for user contexts
1303 * @irq: Interrupt number.
1304 * @data: Private data provided at interrupt registration, the context.
1306 * Return: Always return IRQ_HANDLED.
1308 static irqreturn_t ocxlflash_afu_irq(int irq, void *data)
1310 struct ocxlflash_context *ctx = data;
1311 struct device *dev = ctx->hw_afu->dev;
1314 dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n",
1315 __func__, ctx->pe, irq);
1317 for (i = 0; i < ctx->num_irqs; i++) {
1318 if (ctx->irqs[i].virq == irq)
1321 if (unlikely(i >= ctx->num_irqs)) {
1322 dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__);
1326 spin_lock(&ctx->slock);
1327 set_bit(i - 1, &ctx->irq_bitmap);
1328 ctx->pending_irq = true;
1329 spin_unlock(&ctx->slock);
1331 wake_up_all(&ctx->wq);
1337 * ocxlflash_start_work() - start a user context
1338 * @ctx_cookie: Context to be started.
1339 * @num_irqs: Number of interrupts requested.
1341 * Return: 0 on success, -errno on failure
1343 static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs)
1345 struct ocxlflash_context *ctx = ctx_cookie;
1346 struct ocxl_hw_afu *afu = ctx->hw_afu;
1347 struct device *dev = afu->dev;
1352 rc = alloc_afu_irqs(ctx, num_irqs);
1353 if (unlikely(rc < 0)) {
1354 dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc);
1358 for (i = 0; i < num_irqs; i++) {
1359 name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i",
1360 dev_name(dev), ctx->pe, i);
1361 rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name);
1363 if (unlikely(rc < 0)) {
1364 dev_err(dev, "%s: afu_map_irq failed rc=%d\n",
1370 rc = start_context(ctx);
1372 dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc);
1378 for (i = i-1; i >= 0; i--)
1379 afu_unmap_irq(0, ctx, i, ctx);
1385 * ocxlflash_fd_mmap() - mmap handler for adapter file descriptor
1386 * @file: File installed with adapter file descriptor.
1387 * @vma: VM area associated with mapping.
1389 * Return: 0 on success, -errno on failure
1391 static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma)
1393 return afu_mmap(file, vma);
1397 * ocxlflash_fd_release() - release the context associated with the file
1398 * @inode: File inode pointer.
1399 * @file: File associated with the adapter context.
1401 * Return: 0 on success, -errno on failure
1403 static int ocxlflash_fd_release(struct inode *inode, struct file *file)
1405 return afu_release(inode, file);
1408 /* Backend ops to ocxlflash services */
1409 const struct cxlflash_backend_ops cxlflash_ocxl_ops = {
1410 .module = THIS_MODULE,
1411 .psa_map = ocxlflash_psa_map,
1412 .psa_unmap = ocxlflash_psa_unmap,
1413 .process_element = ocxlflash_process_element,
1414 .map_afu_irq = ocxlflash_map_afu_irq,
1415 .unmap_afu_irq = ocxlflash_unmap_afu_irq,
1416 .get_irq_objhndl = ocxlflash_get_irq_objhndl,
1417 .start_context = ocxlflash_start_context,
1418 .stop_context = ocxlflash_stop_context,
1419 .afu_reset = ocxlflash_afu_reset,
1420 .set_master = ocxlflash_set_master,
1421 .get_context = ocxlflash_get_context,
1422 .dev_context_init = ocxlflash_dev_context_init,
1423 .release_context = ocxlflash_release_context,
1424 .perst_reloads_same_image = ocxlflash_perst_reloads_same_image,
1425 .read_adapter_vpd = ocxlflash_read_adapter_vpd,
1426 .allocate_afu_irqs = ocxlflash_allocate_afu_irqs,
1427 .free_afu_irqs = ocxlflash_free_afu_irqs,
1428 .create_afu = ocxlflash_create_afu,
1429 .destroy_afu = ocxlflash_destroy_afu,
1430 .get_fd = ocxlflash_get_fd,
1431 .fops_get_context = ocxlflash_fops_get_context,
1432 .start_work = ocxlflash_start_work,
1433 .fd_mmap = ocxlflash_fd_mmap,
1434 .fd_release = ocxlflash_fd_release,