drivers/pci/endpoint/functions/pci-epf-mhi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * PCI EPF driver for MHI Endpoint devices
   4  *
   5  * Copyright (C) 2023 Linaro Ltd.
   6  * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
   7  */
   8
   9 #include <linux/dmaengine.h>
  10 #include <linux/mhi_ep.h>
  11 #include <linux/module.h>
  12 #include <linux/of_dma.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/pci-epc.h>
  15 #include <linux/pci-epf.h>
  16
  17 #define MHI_VERSION_1_0 0x01000000
  18
  19 #define to_epf_mhi(cntrl) container_of(cntrl, struct pci_epf_mhi, cntrl)
  20
  21 /* Platform specific flags */
  22 #define MHI_EPF_USE_DMA BIT(0)
  23
  24 struct pci_epf_mhi_dma_transfer {
  25         struct pci_epf_mhi *epf_mhi;
  26         struct mhi_ep_buf_info buf_info;
  27         struct list_head node;
  28         dma_addr_t paddr;
  29         enum dma_data_direction dir;
  30         size_t size;
  31 };
  32
  33 struct pci_epf_mhi_ep_info {
  34         const struct mhi_ep_cntrl_config *config;
  35         struct pci_epf_header *epf_header;
  36         enum pci_barno bar_num;
  37         u32 epf_flags;
  38         u32 msi_count;
  39         u32 mru;
  40         u32 flags;
  41 };
  42
  43 #define MHI_EP_CHANNEL_CONFIG(ch_num, ch_name, direction)       \
  44         {                                                       \
  45                 .num = ch_num,                                  \
  46                 .name = ch_name,                                \
  47                 .dir = direction,                               \
  48         }
  49
  50 #define MHI_EP_CHANNEL_CONFIG_UL(ch_num, ch_name)               \
  51         MHI_EP_CHANNEL_CONFIG(ch_num, ch_name, DMA_TO_DEVICE)
  52
  53 #define MHI_EP_CHANNEL_CONFIG_DL(ch_num, ch_name)               \
  54         MHI_EP_CHANNEL_CONFIG(ch_num, ch_name, DMA_FROM_DEVICE)
  55
  56 static const struct mhi_ep_channel_config mhi_v1_channels[] = {
  57         MHI_EP_CHANNEL_CONFIG_UL(0, "LOOPBACK"),
  58         MHI_EP_CHANNEL_CONFIG_DL(1, "LOOPBACK"),
  59         MHI_EP_CHANNEL_CONFIG_UL(2, "SAHARA"),
  60         MHI_EP_CHANNEL_CONFIG_DL(3, "SAHARA"),
  61         MHI_EP_CHANNEL_CONFIG_UL(4, "DIAG"),
  62         MHI_EP_CHANNEL_CONFIG_DL(5, "DIAG"),
  63         MHI_EP_CHANNEL_CONFIG_UL(6, "SSR"),
  64         MHI_EP_CHANNEL_CONFIG_DL(7, "SSR"),
  65         MHI_EP_CHANNEL_CONFIG_UL(8, "QDSS"),
  66         MHI_EP_CHANNEL_CONFIG_DL(9, "QDSS"),
  67         MHI_EP_CHANNEL_CONFIG_UL(10, "EFS"),
  68         MHI_EP_CHANNEL_CONFIG_DL(11, "EFS"),
  69         MHI_EP_CHANNEL_CONFIG_UL(12, "MBIM"),
  70         MHI_EP_CHANNEL_CONFIG_DL(13, "MBIM"),
  71         MHI_EP_CHANNEL_CONFIG_UL(14, "QMI"),
  72         MHI_EP_CHANNEL_CONFIG_DL(15, "QMI"),
  73         MHI_EP_CHANNEL_CONFIG_UL(16, "QMI"),
  74         MHI_EP_CHANNEL_CONFIG_DL(17, "QMI"),
  75         MHI_EP_CHANNEL_CONFIG_UL(18, "IP-CTRL-1"),
  76         MHI_EP_CHANNEL_CONFIG_DL(19, "IP-CTRL-1"),
  77         MHI_EP_CHANNEL_CONFIG_UL(20, "IPCR"),
  78         MHI_EP_CHANNEL_CONFIG_DL(21, "IPCR"),
  79         MHI_EP_CHANNEL_CONFIG_UL(32, "DUN"),
  80         MHI_EP_CHANNEL_CONFIG_DL(33, "DUN"),
  81         MHI_EP_CHANNEL_CONFIG_UL(46, "IP_SW0"),
  82         MHI_EP_CHANNEL_CONFIG_DL(47, "IP_SW0"),
  83 };
  84
  85 static const struct mhi_ep_cntrl_config mhi_v1_config = {
  86         .max_channels = 128,
  87         .num_channels = ARRAY_SIZE(mhi_v1_channels),
  88         .ch_cfg = mhi_v1_channels,
  89         .mhi_version = MHI_VERSION_1_0,
  90 };
  91
  92 static struct pci_epf_header sdx55_header = {
  93         .vendorid = PCI_VENDOR_ID_QCOM,
  94         .deviceid = 0x0306,
  95         .baseclass_code = PCI_BASE_CLASS_COMMUNICATION,
  96         .subclass_code = PCI_CLASS_COMMUNICATION_MODEM & 0xff,
  97         .interrupt_pin  = PCI_INTERRUPT_INTA,
  98 };
  99
 100 static const struct pci_epf_mhi_ep_info sdx55_info = {
 101         .config = &mhi_v1_config,
 102         .epf_header = &sdx55_header,
 103         .bar_num = BAR_0,
 104         .epf_flags = PCI_BASE_ADDRESS_MEM_TYPE_32,
 105         .msi_count = 32,
 106         .mru = 0x8000,
 107 };
 108
 109 static struct pci_epf_header sm8450_header = {
 110         .vendorid = PCI_VENDOR_ID_QCOM,
 111         .deviceid = 0x0306,
 112         .baseclass_code = PCI_CLASS_OTHERS,
 113         .interrupt_pin = PCI_INTERRUPT_INTA,
 114 };
 115
 116 static const struct pci_epf_mhi_ep_info sm8450_info = {
 117         .config = &mhi_v1_config,
 118         .epf_header = &sm8450_header,
 119         .bar_num = BAR_0,
 120         .epf_flags = PCI_BASE_ADDRESS_MEM_TYPE_32,
 121         .msi_count = 32,
 122         .mru = 0x8000,
 123         .flags = MHI_EPF_USE_DMA,
 124 };
 125
 126 struct pci_epf_mhi {
 127         const struct pci_epc_features *epc_features;
 128         const struct pci_epf_mhi_ep_info *info;
 129         struct mhi_ep_cntrl mhi_cntrl;
 130         struct pci_epf *epf;
 131         struct mutex lock;
 132         void __iomem *mmio;
 133         resource_size_t mmio_phys;
 134         struct dma_chan *dma_chan_tx;
 135         struct dma_chan *dma_chan_rx;
 136         struct workqueue_struct *dma_wq;
 137         struct work_struct dma_work;
 138         struct list_head dma_list;
 139         spinlock_t list_lock;
 140         u32 mmio_size;
 141         int irq;
 142 };
 143
 144 static size_t get_align_offset(struct pci_epf_mhi *epf_mhi, u64 addr)
 145 {
 146         return addr & (epf_mhi->epc_features->align -1);
 147 }
 148
 149 static int __pci_epf_mhi_alloc_map(struct mhi_ep_cntrl *mhi_cntrl, u64 pci_addr,
 150                                  phys_addr_t *paddr, void __iomem **vaddr,
 151                                  size_t offset, size_t size)
 152 {
 153         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 154         struct pci_epf *epf = epf_mhi->epf;
 155         struct pci_epc *epc = epf->epc;
 156         int ret;
 157
 158         *vaddr = pci_epc_mem_alloc_addr(epc, paddr, size + offset);
 159         if (!*vaddr)
 160                 return -ENOMEM;
 161
 162         ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, *paddr,
 163                                pci_addr - offset, size + offset);
 164         if (ret) {
 165                 pci_epc_mem_free_addr(epc, *paddr, *vaddr, size + offset);
 166                 return ret;
 167         }
 168
 169         *paddr = *paddr + offset;
 170         *vaddr = *vaddr + offset;
 171
 172         return 0;
 173 }
 174
 175 static int pci_epf_mhi_alloc_map(struct mhi_ep_cntrl *mhi_cntrl, u64 pci_addr,
 176                                  phys_addr_t *paddr, void __iomem **vaddr,
 177                                  size_t size)
 178 {
 179         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 180         size_t offset = get_align_offset(epf_mhi, pci_addr);
 181
 182         return __pci_epf_mhi_alloc_map(mhi_cntrl, pci_addr, paddr, vaddr,
 183                                       offset, size);
 184 }
 185
 186 static void __pci_epf_mhi_unmap_free(struct mhi_ep_cntrl *mhi_cntrl,
 187                                      u64 pci_addr, phys_addr_t paddr,
 188                                      void __iomem *vaddr, size_t offset,
 189                                      size_t size)
 190 {
 191         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 192         struct pci_epf *epf = epf_mhi->epf;
 193         struct pci_epc *epc = epf->epc;
 194
 195         pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, paddr - offset);
 196         pci_epc_mem_free_addr(epc, paddr - offset, vaddr - offset,
 197                               size + offset);
 198 }
 199
 200 static void pci_epf_mhi_unmap_free(struct mhi_ep_cntrl *mhi_cntrl, u64 pci_addr,
 201                                    phys_addr_t paddr, void __iomem *vaddr,
 202                                    size_t size)
 203 {
 204         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 205         size_t offset = get_align_offset(epf_mhi, pci_addr);
 206
 207         __pci_epf_mhi_unmap_free(mhi_cntrl, pci_addr, paddr, vaddr, offset,
 208                                  size);
 209 }
 210
 211 static void pci_epf_mhi_raise_irq(struct mhi_ep_cntrl *mhi_cntrl, u32 vector)
 212 {
 213         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 214         struct pci_epf *epf = epf_mhi->epf;
 215         struct pci_epc *epc = epf->epc;
 216
 217         /*
 218          * MHI supplies 0 based MSI vectors but the API expects the vector
 219          * number to start from 1, so we need to increment the vector by 1.
 220          */
 221         pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, PCI_IRQ_MSI,
 222                           vector + 1);
 223 }
 224
 225 static int pci_epf_mhi_iatu_read(struct mhi_ep_cntrl *mhi_cntrl,
 226                                  struct mhi_ep_buf_info *buf_info)
 227 {
 228         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 229         size_t offset = get_align_offset(epf_mhi, buf_info->host_addr);
 230         void __iomem *tre_buf;
 231         phys_addr_t tre_phys;
 232         int ret;
 233
 234         mutex_lock(&epf_mhi->lock);
 235
 236         ret = __pci_epf_mhi_alloc_map(mhi_cntrl, buf_info->host_addr, &tre_phys,
 237                                       &tre_buf, offset, buf_info->size);
 238         if (ret) {
 239                 mutex_unlock(&epf_mhi->lock);
 240                 return ret;
 241         }
 242
 243         memcpy_fromio(buf_info->dev_addr, tre_buf, buf_info->size);
 244
 245         __pci_epf_mhi_unmap_free(mhi_cntrl, buf_info->host_addr, tre_phys,
 246                                  tre_buf, offset, buf_info->size);
 247
 248         mutex_unlock(&epf_mhi->lock);
 249
 250         if (buf_info->cb)
 251                 buf_info->cb(buf_info);
 252
 253         return 0;
 254 }
 255
 256 static int pci_epf_mhi_iatu_write(struct mhi_ep_cntrl *mhi_cntrl,
 257                                   struct mhi_ep_buf_info *buf_info)
 258 {
 259         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 260         size_t offset = get_align_offset(epf_mhi, buf_info->host_addr);
 261         void __iomem *tre_buf;
 262         phys_addr_t tre_phys;
 263         int ret;
 264
 265         mutex_lock(&epf_mhi->lock);
 266
 267         ret = __pci_epf_mhi_alloc_map(mhi_cntrl, buf_info->host_addr, &tre_phys,
 268                                       &tre_buf, offset, buf_info->size);
 269         if (ret) {
 270                 mutex_unlock(&epf_mhi->lock);
 271                 return ret;
 272         }
 273
 274         memcpy_toio(tre_buf, buf_info->dev_addr, buf_info->size);
 275
 276         __pci_epf_mhi_unmap_free(mhi_cntrl, buf_info->host_addr, tre_phys,
 277                                  tre_buf, offset, buf_info->size);
 278
 279         mutex_unlock(&epf_mhi->lock);
 280
 281         if (buf_info->cb)
 282                 buf_info->cb(buf_info);
 283
 284         return 0;
 285 }
 286
 287 static void pci_epf_mhi_dma_callback(void *param)
 288 {
 289         complete(param);
 290 }
 291
 292 static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
 293                                  struct mhi_ep_buf_info *buf_info)
 294 {
 295         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 296         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 297         struct dma_chan *chan = epf_mhi->dma_chan_rx;
 298         struct device *dev = &epf_mhi->epf->dev;
 299         DECLARE_COMPLETION_ONSTACK(complete);
 300         struct dma_async_tx_descriptor *desc;
 301         struct dma_slave_config config = {};
 302         dma_cookie_t cookie;
 303         dma_addr_t dst_addr;
 304         int ret;
 305
 306         if (buf_info->size < SZ_4K)
 307                 return pci_epf_mhi_iatu_read(mhi_cntrl, buf_info);
 308
 309         mutex_lock(&epf_mhi->lock);
 310
 311         config.direction = DMA_DEV_TO_MEM;
 312         config.src_addr = buf_info->host_addr;
 313
 314         ret = dmaengine_slave_config(chan, &config);
 315         if (ret) {
 316                 dev_err(dev, "Failed to configure DMA channel\n");
 317                 goto err_unlock;
 318         }
 319
 320         dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 321                                   DMA_FROM_DEVICE);
 322         ret = dma_mapping_error(dma_dev, dst_addr);
 323         if (ret) {
 324                 dev_err(dev, "Failed to map remote memory\n");
 325                 goto err_unlock;
 326         }
 327
 328         desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
 329                                            DMA_DEV_TO_MEM,
 330                                            DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
 331         if (!desc) {
 332                 dev_err(dev, "Failed to prepare DMA\n");
 333                 ret = -EIO;
 334                 goto err_unmap;
 335         }
 336
 337         desc->callback = pci_epf_mhi_dma_callback;
 338         desc->callback_param = &complete;
 339
 340         cookie = dmaengine_submit(desc);
 341         ret = dma_submit_error(cookie);
 342         if (ret) {
 343                 dev_err(dev, "Failed to do DMA submit\n");
 344                 goto err_unmap;
 345         }
 346
 347         dma_async_issue_pending(chan);
 348         ret = wait_for_completion_timeout(&complete, msecs_to_jiffies(1000));
 349         if (!ret) {
 350                 dev_err(dev, "DMA transfer timeout\n");
 351                 dmaengine_terminate_sync(chan);
 352                 ret = -ETIMEDOUT;
 353         }
 354
 355 err_unmap:
 356         dma_unmap_single(dma_dev, dst_addr, buf_info->size, DMA_FROM_DEVICE);
 357 err_unlock:
 358         mutex_unlock(&epf_mhi->lock);
 359
 360         return ret;
 361 }
 362
 363 static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
 364                                   struct mhi_ep_buf_info *buf_info)
 365 {
 366         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 367         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 368         struct dma_chan *chan = epf_mhi->dma_chan_tx;
 369         struct device *dev = &epf_mhi->epf->dev;
 370         DECLARE_COMPLETION_ONSTACK(complete);
 371         struct dma_async_tx_descriptor *desc;
 372         struct dma_slave_config config = {};
 373         dma_cookie_t cookie;
 374         dma_addr_t src_addr;
 375         int ret;
 376
 377         if (buf_info->size < SZ_4K)
 378                 return pci_epf_mhi_iatu_write(mhi_cntrl, buf_info);
 379
 380         mutex_lock(&epf_mhi->lock);
 381
 382         config.direction = DMA_MEM_TO_DEV;
 383         config.dst_addr = buf_info->host_addr;
 384
 385         ret = dmaengine_slave_config(chan, &config);
 386         if (ret) {
 387                 dev_err(dev, "Failed to configure DMA channel\n");
 388                 goto err_unlock;
 389         }
 390
 391         src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 392                                   DMA_TO_DEVICE);
 393         ret = dma_mapping_error(dma_dev, src_addr);
 394         if (ret) {
 395                 dev_err(dev, "Failed to map remote memory\n");
 396                 goto err_unlock;
 397         }
 398
 399         desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
 400                                            DMA_MEM_TO_DEV,
 401                                            DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
 402         if (!desc) {
 403                 dev_err(dev, "Failed to prepare DMA\n");
 404                 ret = -EIO;
 405                 goto err_unmap;
 406         }
 407
 408         desc->callback = pci_epf_mhi_dma_callback;
 409         desc->callback_param = &complete;
 410
 411         cookie = dmaengine_submit(desc);
 412         ret = dma_submit_error(cookie);
 413         if (ret) {
 414                 dev_err(dev, "Failed to do DMA submit\n");
 415                 goto err_unmap;
 416         }
 417
 418         dma_async_issue_pending(chan);
 419         ret = wait_for_completion_timeout(&complete, msecs_to_jiffies(1000));
 420         if (!ret) {
 421                 dev_err(dev, "DMA transfer timeout\n");
 422                 dmaengine_terminate_sync(chan);
 423                 ret = -ETIMEDOUT;
 424         }
 425
 426 err_unmap:
 427         dma_unmap_single(dma_dev, src_addr, buf_info->size, DMA_TO_DEVICE);
 428 err_unlock:
 429         mutex_unlock(&epf_mhi->lock);
 430
 431         return ret;
 432 }
 433
 434 static void pci_epf_mhi_dma_worker(struct work_struct *work)
 435 {
 436         struct pci_epf_mhi *epf_mhi = container_of(work, struct pci_epf_mhi, dma_work);
 437         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 438         struct pci_epf_mhi_dma_transfer *itr, *tmp;
 439         struct mhi_ep_buf_info *buf_info;
 440         unsigned long flags;
 441         LIST_HEAD(head);
 442
 443         spin_lock_irqsave(&epf_mhi->list_lock, flags);
 444         list_splice_tail_init(&epf_mhi->dma_list, &head);
 445         spin_unlock_irqrestore(&epf_mhi->list_lock, flags);
 446
 447         list_for_each_entry_safe(itr, tmp, &head, node) {
 448                 list_del(&itr->node);
 449                 dma_unmap_single(dma_dev, itr->paddr, itr->size, itr->dir);
 450                 buf_info = &itr->buf_info;
 451                 buf_info->cb(buf_info);
 452                 kfree(itr);
 453         }
 454 }
 455
 456 static void pci_epf_mhi_dma_async_callback(void *param)
 457 {
 458         struct pci_epf_mhi_dma_transfer *transfer = param;
 459         struct pci_epf_mhi *epf_mhi = transfer->epf_mhi;
 460
 461         spin_lock(&epf_mhi->list_lock);
 462         list_add_tail(&transfer->node, &epf_mhi->dma_list);
 463         spin_unlock(&epf_mhi->list_lock);
 464
 465         queue_work(epf_mhi->dma_wq, &epf_mhi->dma_work);
 466 }
 467
 468 static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl,
 469                                        struct mhi_ep_buf_info *buf_info)
 470 {
 471         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 472         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 473         struct pci_epf_mhi_dma_transfer *transfer = NULL;
 474         struct dma_chan *chan = epf_mhi->dma_chan_rx;
 475         struct device *dev = &epf_mhi->epf->dev;
 476         DECLARE_COMPLETION_ONSTACK(complete);
 477         struct dma_async_tx_descriptor *desc;
 478         struct dma_slave_config config = {};
 479         dma_cookie_t cookie;
 480         dma_addr_t dst_addr;
 481         int ret;
 482
 483         mutex_lock(&epf_mhi->lock);
 484
 485         config.direction = DMA_DEV_TO_MEM;
 486         config.src_addr = buf_info->host_addr;
 487
 488         ret = dmaengine_slave_config(chan, &config);
 489         if (ret) {
 490                 dev_err(dev, "Failed to configure DMA channel\n");
 491                 goto err_unlock;
 492         }
 493
 494         dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 495                                   DMA_FROM_DEVICE);
 496         ret = dma_mapping_error(dma_dev, dst_addr);
 497         if (ret) {
 498                 dev_err(dev, "Failed to map remote memory\n");
 499                 goto err_unlock;
 500         }
 501
 502         desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
 503                                            DMA_DEV_TO_MEM,
 504                                            DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
 505         if (!desc) {
 506                 dev_err(dev, "Failed to prepare DMA\n");
 507                 ret = -EIO;
 508                 goto err_unmap;
 509         }
 510
 511         transfer = kzalloc(sizeof(*transfer), GFP_KERNEL);
 512         if (!transfer) {
 513                 ret = -ENOMEM;
 514                 goto err_unmap;
 515         }
 516
 517         transfer->epf_mhi = epf_mhi;
 518         transfer->paddr = dst_addr;
 519         transfer->size = buf_info->size;
 520         transfer->dir = DMA_FROM_DEVICE;
 521         memcpy(&transfer->buf_info, buf_info, sizeof(*buf_info));
 522
 523         desc->callback = pci_epf_mhi_dma_async_callback;
 524         desc->callback_param = transfer;
 525
 526         cookie = dmaengine_submit(desc);
 527         ret = dma_submit_error(cookie);
 528         if (ret) {
 529                 dev_err(dev, "Failed to do DMA submit\n");
 530                 goto err_free_transfer;
 531         }
 532
 533         dma_async_issue_pending(chan);
 534
 535         goto err_unlock;
 536
 537 err_free_transfer:
 538         kfree(transfer);
 539 err_unmap:
 540         dma_unmap_single(dma_dev, dst_addr, buf_info->size, DMA_FROM_DEVICE);
 541 err_unlock:
 542         mutex_unlock(&epf_mhi->lock);
 543
 544         return ret;
 545 }
 546
 547 static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl,
 548                                         struct mhi_ep_buf_info *buf_info)
 549 {
 550         struct pci_epf_mhi *epf_mhi = to_epf_mhi(mhi_cntrl);
 551         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 552         struct pci_epf_mhi_dma_transfer *transfer = NULL;
 553         struct dma_chan *chan = epf_mhi->dma_chan_tx;
 554         struct device *dev = &epf_mhi->epf->dev;
 555         DECLARE_COMPLETION_ONSTACK(complete);
 556         struct dma_async_tx_descriptor *desc;
 557         struct dma_slave_config config = {};
 558         dma_cookie_t cookie;
 559         dma_addr_t src_addr;
 560         int ret;
 561
 562         mutex_lock(&epf_mhi->lock);
 563
 564         config.direction = DMA_MEM_TO_DEV;
 565         config.dst_addr = buf_info->host_addr;
 566
 567         ret = dmaengine_slave_config(chan, &config);
 568         if (ret) {
 569                 dev_err(dev, "Failed to configure DMA channel\n");
 570                 goto err_unlock;
 571         }
 572
 573         src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
 574                                   DMA_TO_DEVICE);
 575         ret = dma_mapping_error(dma_dev, src_addr);
 576         if (ret) {
 577                 dev_err(dev, "Failed to map remote memory\n");
 578                 goto err_unlock;
 579         }
 580
 581         desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
 582                                            DMA_MEM_TO_DEV,
 583                                            DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
 584         if (!desc) {
 585                 dev_err(dev, "Failed to prepare DMA\n");
 586                 ret = -EIO;
 587                 goto err_unmap;
 588         }
 589
 590         transfer = kzalloc(sizeof(*transfer), GFP_KERNEL);
 591         if (!transfer) {
 592                 ret = -ENOMEM;
 593                 goto err_unmap;
 594         }
 595
 596         transfer->epf_mhi = epf_mhi;
 597         transfer->paddr = src_addr;
 598         transfer->size = buf_info->size;
 599         transfer->dir = DMA_TO_DEVICE;
 600         memcpy(&transfer->buf_info, buf_info, sizeof(*buf_info));
 601
 602         desc->callback = pci_epf_mhi_dma_async_callback;
 603         desc->callback_param = transfer;
 604
 605         cookie = dmaengine_submit(desc);
 606         ret = dma_submit_error(cookie);
 607         if (ret) {
 608                 dev_err(dev, "Failed to do DMA submit\n");
 609                 goto err_free_transfer;
 610         }
 611
 612         dma_async_issue_pending(chan);
 613
 614         goto err_unlock;
 615
 616 err_free_transfer:
 617         kfree(transfer);
 618 err_unmap:
 619         dma_unmap_single(dma_dev, src_addr, buf_info->size, DMA_TO_DEVICE);
 620 err_unlock:
 621         mutex_unlock(&epf_mhi->lock);
 622
 623         return ret;
 624 }
 625
 626 struct epf_dma_filter {
 627         struct device *dev;
 628         u32 dma_mask;
 629 };
 630
 631 static bool pci_epf_mhi_filter(struct dma_chan *chan, void *node)
 632 {
 633         struct epf_dma_filter *filter = node;
 634         struct dma_slave_caps caps;
 635
 636         memset(&caps, 0, sizeof(caps));
 637         dma_get_slave_caps(chan, &caps);
 638
 639         return chan->device->dev == filter->dev && filter->dma_mask &
 640                                         caps.directions;
 641 }
 642
 643 static int pci_epf_mhi_dma_init(struct pci_epf_mhi *epf_mhi)
 644 {
 645         struct device *dma_dev = epf_mhi->epf->epc->dev.parent;
 646         struct device *dev = &epf_mhi->epf->dev;
 647         struct epf_dma_filter filter;
 648         dma_cap_mask_t mask;
 649         int ret;
 650
 651         dma_cap_zero(mask);
 652         dma_cap_set(DMA_SLAVE, mask);
 653
 654         filter.dev = dma_dev;
 655         filter.dma_mask = BIT(DMA_MEM_TO_DEV);
 656         epf_mhi->dma_chan_tx = dma_request_channel(mask, pci_epf_mhi_filter,
 657                                                    &filter);
 658         if (IS_ERR_OR_NULL(epf_mhi->dma_chan_tx)) {
 659                 dev_err(dev, "Failed to request tx channel\n");
 660                 return -ENODEV;
 661         }
 662
 663         filter.dma_mask = BIT(DMA_DEV_TO_MEM);
 664         epf_mhi->dma_chan_rx = dma_request_channel(mask, pci_epf_mhi_filter,
 665                                                    &filter);
 666         if (IS_ERR_OR_NULL(epf_mhi->dma_chan_rx)) {
 667                 dev_err(dev, "Failed to request rx channel\n");
 668                 ret = -ENODEV;
 669                 goto err_release_tx;
 670         }
 671
 672         epf_mhi->dma_wq = alloc_workqueue("pci_epf_mhi_dma_wq", 0, 0);
 673         if (!epf_mhi->dma_wq) {
 674                 ret = -ENOMEM;
 675                 goto err_release_rx;
 676         }
 677
 678         INIT_LIST_HEAD(&epf_mhi->dma_list);
 679         INIT_WORK(&epf_mhi->dma_work, pci_epf_mhi_dma_worker);
 680         spin_lock_init(&epf_mhi->list_lock);
 681
 682         return 0;
 683
 684 err_release_rx:
 685         dma_release_channel(epf_mhi->dma_chan_rx);
 686         epf_mhi->dma_chan_rx = NULL;
 687 err_release_tx:
 688         dma_release_channel(epf_mhi->dma_chan_tx);
 689         epf_mhi->dma_chan_tx = NULL;
 690
 691         return ret;
 692 }
 693
 694 static void pci_epf_mhi_dma_deinit(struct pci_epf_mhi *epf_mhi)
 695 {
 696         destroy_workqueue(epf_mhi->dma_wq);
 697         dma_release_channel(epf_mhi->dma_chan_tx);
 698         dma_release_channel(epf_mhi->dma_chan_rx);
 699         epf_mhi->dma_chan_tx = NULL;
 700         epf_mhi->dma_chan_rx = NULL;
 701 }
 702
 703 static int pci_epf_mhi_core_init(struct pci_epf *epf)
 704 {
 705         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 706         const struct pci_epf_mhi_ep_info *info = epf_mhi->info;
 707         struct pci_epf_bar *epf_bar = &epf->bar[info->bar_num];
 708         struct pci_epc *epc = epf->epc;
 709         struct device *dev = &epf->dev;
 710         int ret;
 711
 712         epf_bar->phys_addr = epf_mhi->mmio_phys;
 713         epf_bar->size = epf_mhi->mmio_size;
 714         epf_bar->barno = info->bar_num;
 715         epf_bar->flags = info->epf_flags;
 716         ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, epf_bar);
 717         if (ret) {
 718                 dev_err(dev, "Failed to set BAR: %d\n", ret);
 719                 return ret;
 720         }
 721
 722         ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
 723                               order_base_2(info->msi_count));
 724         if (ret) {
 725                 dev_err(dev, "Failed to set MSI configuration: %d\n", ret);
 726                 return ret;
 727         }
 728
 729         ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no,
 730                                    epf->header);
 731         if (ret) {
 732                 dev_err(dev, "Failed to set Configuration header: %d\n", ret);
 733                 return ret;
 734         }
 735
 736         epf_mhi->epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
 737         if (!epf_mhi->epc_features)
 738                 return -ENODATA;
 739
 740         return 0;
 741 }
 742
 743 static int pci_epf_mhi_link_up(struct pci_epf *epf)
 744 {
 745         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 746         const struct pci_epf_mhi_ep_info *info = epf_mhi->info;
 747         struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;
 748         struct pci_epc *epc = epf->epc;
 749         struct device *dev = &epf->dev;
 750         int ret;
 751
 752         if (info->flags & MHI_EPF_USE_DMA) {
 753                 ret = pci_epf_mhi_dma_init(epf_mhi);
 754                 if (ret) {
 755                         dev_err(dev, "Failed to initialize DMA: %d\n", ret);
 756                         return ret;
 757                 }
 758         }
 759
 760         mhi_cntrl->mmio = epf_mhi->mmio;
 761         mhi_cntrl->irq = epf_mhi->irq;
 762         mhi_cntrl->mru = info->mru;
 763
 764         /* Assign the struct dev of PCI EP as MHI controller device */
 765         mhi_cntrl->cntrl_dev = epc->dev.parent;
 766         mhi_cntrl->raise_irq = pci_epf_mhi_raise_irq;
 767         mhi_cntrl->alloc_map = pci_epf_mhi_alloc_map;
 768         mhi_cntrl->unmap_free = pci_epf_mhi_unmap_free;
 769         mhi_cntrl->read_sync = mhi_cntrl->read_async = pci_epf_mhi_iatu_read;
 770         mhi_cntrl->write_sync = mhi_cntrl->write_async = pci_epf_mhi_iatu_write;
 771         if (info->flags & MHI_EPF_USE_DMA) {
 772                 mhi_cntrl->read_sync = pci_epf_mhi_edma_read;
 773                 mhi_cntrl->write_sync = pci_epf_mhi_edma_write;
 774                 mhi_cntrl->read_async = pci_epf_mhi_edma_read_async;
 775                 mhi_cntrl->write_async = pci_epf_mhi_edma_write_async;
 776         }
 777
 778         /* Register the MHI EP controller */
 779         ret = mhi_ep_register_controller(mhi_cntrl, info->config);
 780         if (ret) {
 781                 dev_err(dev, "Failed to register MHI EP controller: %d\n", ret);
 782                 if (info->flags & MHI_EPF_USE_DMA)
 783                         pci_epf_mhi_dma_deinit(epf_mhi);
 784                 return ret;
 785         }
 786
 787         return 0;
 788 }
 789
 790 static int pci_epf_mhi_link_down(struct pci_epf *epf)
 791 {
 792         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 793         const struct pci_epf_mhi_ep_info *info = epf_mhi->info;
 794         struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;
 795
 796         if (mhi_cntrl->mhi_dev) {
 797                 mhi_ep_power_down(mhi_cntrl);
 798                 if (info->flags & MHI_EPF_USE_DMA)
 799                         pci_epf_mhi_dma_deinit(epf_mhi);
 800                 mhi_ep_unregister_controller(mhi_cntrl);
 801         }
 802
 803         return 0;
 804 }
 805
 806 static int pci_epf_mhi_bme(struct pci_epf *epf)
 807 {
 808         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 809         const struct pci_epf_mhi_ep_info *info = epf_mhi->info;
 810         struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;
 811         struct device *dev = &epf->dev;
 812         int ret;
 813
 814         /*
 815          * Power up the MHI EP stack if link is up and stack is in power down
 816          * state.
 817          */
 818         if (!mhi_cntrl->enabled && mhi_cntrl->mhi_dev) {
 819                 ret = mhi_ep_power_up(mhi_cntrl);
 820                 if (ret) {
 821                         dev_err(dev, "Failed to power up MHI EP: %d\n", ret);
 822                         if (info->flags & MHI_EPF_USE_DMA)
 823                                 pci_epf_mhi_dma_deinit(epf_mhi);
 824                         mhi_ep_unregister_controller(mhi_cntrl);
 825                 }
 826         }
 827
 828         return 0;
 829 }
 830
 831 static int pci_epf_mhi_bind(struct pci_epf *epf)
 832 {
 833         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 834         struct pci_epc *epc = epf->epc;
 835         struct platform_device *pdev = to_platform_device(epc->dev.parent);
 836         struct resource *res;
 837         int ret;
 838
 839         /* Get MMIO base address from Endpoint controller */
 840         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mmio");
 841         epf_mhi->mmio_phys = res->start;
 842         epf_mhi->mmio_size = resource_size(res);
 843
 844         epf_mhi->mmio = ioremap(epf_mhi->mmio_phys, epf_mhi->mmio_size);
 845         if (!epf_mhi->mmio)
 846                 return -ENOMEM;
 847
 848         ret = platform_get_irq_byname(pdev, "doorbell");
 849         if (ret < 0) {
 850                 iounmap(epf_mhi->mmio);
 851                 return ret;
 852         }
 853
 854         epf_mhi->irq = ret;
 855
 856         return 0;
 857 }
 858
 859 static void pci_epf_mhi_unbind(struct pci_epf *epf)
 860 {
 861         struct pci_epf_mhi *epf_mhi = epf_get_drvdata(epf);
 862         const struct pci_epf_mhi_ep_info *info = epf_mhi->info;
 863         struct pci_epf_bar *epf_bar = &epf->bar[info->bar_num];
 864         struct mhi_ep_cntrl *mhi_cntrl = &epf_mhi->mhi_cntrl;
 865         struct pci_epc *epc = epf->epc;
 866
 867         /*
 868          * Forcefully power down the MHI EP stack. Only way to bring the MHI EP
 869          * stack back to working state after successive bind is by getting BME
 870          * from host.
 871          */
 872         if (mhi_cntrl->mhi_dev) {
 873                 mhi_ep_power_down(mhi_cntrl);
 874                 if (info->flags & MHI_EPF_USE_DMA)
 875                         pci_epf_mhi_dma_deinit(epf_mhi);
 876                 mhi_ep_unregister_controller(mhi_cntrl);
 877         }
 878
 879         iounmap(epf_mhi->mmio);
 880         pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, epf_bar);
 881 }
 882
 883 static const struct pci_epc_event_ops pci_epf_mhi_event_ops = {
 884         .core_init = pci_epf_mhi_core_init,
 885         .link_up = pci_epf_mhi_link_up,
 886         .link_down = pci_epf_mhi_link_down,
 887         .bme = pci_epf_mhi_bme,
 888 };
 889
 890 static int pci_epf_mhi_probe(struct pci_epf *epf,
 891                              const struct pci_epf_device_id *id)
 892 {
 893         struct pci_epf_mhi_ep_info *info =
 894                         (struct pci_epf_mhi_ep_info *)id->driver_data;
 895         struct pci_epf_mhi *epf_mhi;
 896         struct device *dev = &epf->dev;
 897
 898         epf_mhi = devm_kzalloc(dev, sizeof(*epf_mhi), GFP_KERNEL);
 899         if (!epf_mhi)
 900                 return -ENOMEM;
 901
 902         epf->header = info->epf_header;
 903         epf_mhi->info = info;
 904         epf_mhi->epf = epf;
 905
 906         epf->event_ops = &pci_epf_mhi_event_ops;
 907
 908         mutex_init(&epf_mhi->lock);
 909
 910         epf_set_drvdata(epf, epf_mhi);
 911
 912         return 0;
 913 }
 914
 915 static const struct pci_epf_device_id pci_epf_mhi_ids[] = {
 916         { .name = "sdx55", .driver_data = (kernel_ulong_t)&sdx55_info },
 917         { .name = "sm8450", .driver_data = (kernel_ulong_t)&sm8450_info },
 918         {},
 919 };
 920
 921 static const struct pci_epf_ops pci_epf_mhi_ops = {
 922         .unbind = pci_epf_mhi_unbind,
 923         .bind   = pci_epf_mhi_bind,
 924 };
 925
 926 static struct pci_epf_driver pci_epf_mhi_driver = {
 927         .driver.name    = "pci_epf_mhi",
 928         .probe          = pci_epf_mhi_probe,
 929         .id_table       = pci_epf_mhi_ids,
 930         .ops            = &pci_epf_mhi_ops,
 931         .owner          = THIS_MODULE,
 932 };
 933
 934 static int __init pci_epf_mhi_init(void)
 935 {
 936         return pci_epf_register_driver(&pci_epf_mhi_driver);
 937 }
 938 module_init(pci_epf_mhi_init);
 939
 940 static void __exit pci_epf_mhi_exit(void)
 941 {
 942         pci_epf_unregister_driver(&pci_epf_mhi_driver);
 943 }
 944 module_exit(pci_epf_mhi_exit);
 945
 946 MODULE_DESCRIPTION("PCI EPF driver for MHI Endpoint devices");
 947 MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
 948 MODULE_LICENSE("GPL");