drivers/usb/cdns3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  * Work around 2:
  32  * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33  * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34  * in correct order. If the first packet in the buffer will not be handled,
  35  * then the following packets directed for other endpoints and  functions
  36  * will be blocked.
  37  * Additionally the packets directed to one endpoint can block entire on-chip
  38  * buffers. In this case transfer to other endpoints also will blocked.
  39  *
  40  * To resolve this issue after raising the descriptor missing interrupt
  41  * driver prepares internal usb_request object and use it to arm DMA transfer.
  42  *
  43  * The problematic situation was observed in case when endpoint has been enabled
  44  * but no usb_request were queued. Driver try detects such endpoints and will
  45  * use this workaround only for these endpoint.
  46  *
  47  * Driver use limited number of buffer. This number can be set by macro
  48  * CDNS3_WA2_NUM_BUFFERS.
  49  *
  50  * Such blocking situation was observed on ACM gadget. For this function
  51  * host send OUT data packet but ACM function is not prepared for this packet.
  52  * It's cause that buffer placed in on chip memory block transfer to other
  53  * endpoints.
  54  *
  55  * Issue has been fixed in DEV_VER_V2 version of controller.
  56  *
  57  */
  58
  59 #include <linux/dma-mapping.h>
  60 #include <linux/usb/gadget.h>
  61 #include <linux/module.h>
  62 #include <linux/iopoll.h>
  63
  64 #include "core.h"
  65 #include "gadget-export.h"
  66 #include "gadget.h"
  67 #include "trace.h"
  68 #include "drd.h"
  69
  70 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  71                                    struct usb_request *request,
  72                                    gfp_t gfp_flags);
  73
  74 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  75                                  struct usb_request *request);
  76
  77 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
  78                                         struct usb_request *request);
  79
  80 /**
  81  * cdns3_clear_register_bit - clear bit in given register.
  82  * @ptr: address of device controller register to be read and changed
  83  * @mask: bits requested to clar
  84  */
  85 static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
  86 {
  87         mask = readl(ptr) & ~mask;
  88         writel(mask, ptr);
  89 }
  90
  91 /**
  92  * cdns3_set_register_bit - set bit in given register.
  93  * @ptr: address of device controller register to be read and changed
  94  * @mask: bits requested to set
  95  */
  96 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  97 {
  98         mask = readl(ptr) | mask;
  99         writel(mask, ptr);
 100 }
 101
 102 /**
 103  * cdns3_ep_addr_to_index - Macro converts endpoint address to
 104  * index of endpoint object in cdns3_device.eps[] container
 105  * @ep_addr: endpoint address for which endpoint object is required
 106  *
 107  */
 108 u8 cdns3_ep_addr_to_index(u8 ep_addr)
 109 {
 110         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 111 }
 112
 113 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 114                              struct cdns3_endpoint *priv_ep)
 115 {
 116         int dma_index;
 117
 118         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 119
 120         return dma_index / TRB_SIZE;
 121 }
 122
 123 /**
 124  * cdns3_next_request - returns next request from list
 125  * @list: list containing requests
 126  *
 127  * Returns request or NULL if no requests in list
 128  */
 129 struct usb_request *cdns3_next_request(struct list_head *list)
 130 {
 131         return list_first_entry_or_null(list, struct usb_request, list);
 132 }
 133
 134 /**
 135  * cdns3_next_align_buf - returns next buffer from list
 136  * @list: list containing buffers
 137  *
 138  * Returns buffer or NULL if no buffers in list
 139  */
 140 static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 141 {
 142         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 143 }
 144
 145 /**
 146  * cdns3_next_priv_request - returns next request from list
 147  * @list: list containing requests
 148  *
 149  * Returns request or NULL if no requests in list
 150  */
 151 static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 152 {
 153         return list_first_entry_or_null(list, struct cdns3_request, list);
 154 }
 155
 156 /**
 157  * select_ep - selects endpoint
 158  * @priv_dev:  extended gadget object
 159  * @ep: endpoint address
 160  */
 161 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 162 {
 163         if (priv_dev->selected_ep == ep)
 164                 return;
 165
 166         priv_dev->selected_ep = ep;
 167         writel(ep, &priv_dev->regs->ep_sel);
 168 }
 169
 170 /**
 171  * cdns3_get_tdl - gets current tdl for selected endpoint.
 172  * @priv_dev:  extended gadget object
 173  *
 174  * Before calling this function the appropriate endpoint must
 175  * be selected by means of cdns3_select_ep function.
 176  */
 177 static int cdns3_get_tdl(struct cdns3_device *priv_dev)
 178 {
 179         if (priv_dev->dev_ver < DEV_VER_V3)
 180                 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 181         else
 182                 return readl(&priv_dev->regs->ep_tdl);
 183 }
 184
 185 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 186                                  struct cdns3_trb *trb)
 187 {
 188         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 189
 190         return priv_ep->trb_pool_dma + offset;
 191 }
 192
 193 static int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 194 {
 195         switch (priv_ep->type) {
 196         case USB_ENDPOINT_XFER_ISOC:
 197                 return TRB_ISO_RING_SIZE;
 198         case USB_ENDPOINT_XFER_CONTROL:
 199                 return TRB_CTRL_RING_SIZE;
 200         default:
 201                 if (priv_ep->use_streams)
 202                         return TRB_STREAM_RING_SIZE;
 203                 else
 204                         return TRB_RING_SIZE;
 205         }
 206 }
 207
 208 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 209 {
 210         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 211
 212         if (priv_ep->trb_pool) {
 213                 dma_free_coherent(priv_dev->sysdev,
 214                                   cdns3_ring_size(priv_ep),
 215                                   priv_ep->trb_pool, priv_ep->trb_pool_dma);
 216                 priv_ep->trb_pool = NULL;
 217         }
 218 }
 219
 220 /**
 221  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 222  * @priv_ep:  endpoint object
 223  *
 224  * Function will return 0 on success or -ENOMEM on allocation error
 225  */
 226 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 227 {
 228         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 229         int ring_size = cdns3_ring_size(priv_ep);
 230         int num_trbs = ring_size / TRB_SIZE;
 231         struct cdns3_trb *link_trb;
 232
 233         if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
 234                 cdns3_free_trb_pool(priv_ep);
 235
 236         if (!priv_ep->trb_pool) {
 237                 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
 238                                                        ring_size,
 239                                                        &priv_ep->trb_pool_dma,
 240                                                        GFP_DMA32 | GFP_ATOMIC);
 241                 if (!priv_ep->trb_pool)
 242                         return -ENOMEM;
 243
 244                 priv_ep->alloc_ring_size = ring_size;
 245         }
 246
 247         memset(priv_ep->trb_pool, 0, ring_size);
 248
 249         priv_ep->num_trbs = num_trbs;
 250
 251         if (!priv_ep->num)
 252                 return 0;
 253
 254         /* Initialize the last TRB as Link TRB */
 255         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 256
 257         if (priv_ep->use_streams) {
 258                 /*
 259                  * For stream capable endpoints driver use single correct TRB.
 260                  * The last trb has zeroed cycle bit
 261                  */
 262                 link_trb->control = 0;
 263         } else {
 264                 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
 265                 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
 266         }
 267         return 0;
 268 }
 269
 270 /**
 271  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 272  * @priv_ep: endpoint object
 273  *
 274  * Endpoint must be selected before call to this function
 275  */
 276 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 277 {
 278         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 279         int val;
 280
 281         trace_cdns3_halt(priv_ep, 1, 1);
 282
 283         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 284                &priv_dev->regs->ep_cmd);
 285
 286         /* wait for DFLUSH cleared */
 287         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 288                                   !(val & EP_CMD_DFLUSH), 1, 1000);
 289         priv_ep->flags |= EP_STALLED;
 290         priv_ep->flags &= ~EP_STALL_PENDING;
 291 }
 292
 293 /**
 294  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 295  * @priv_dev: extended gadget object
 296  */
 297 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 298 {
 299         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 300
 301         cdns3_allow_enable_l1(priv_dev, 0);
 302         priv_dev->hw_configured_flag = 0;
 303         priv_dev->onchip_used_size = 0;
 304         priv_dev->out_mem_is_allocated = 0;
 305         priv_dev->wait_for_setup = 0;
 306         priv_dev->using_streams = 0;
 307 }
 308
 309 /**
 310  * cdns3_ep_inc_trb - increment a trb index.
 311  * @index: Pointer to the TRB index to increment.
 312  * @cs: Cycle state
 313  * @trb_in_seg: number of TRBs in segment
 314  *
 315  * The index should never point to the link TRB. After incrementing,
 316  * if it is point to the link TRB, wrap around to the beginning and revert
 317  * cycle state bit The
 318  * link TRB is always at the last TRB entry.
 319  */
 320 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 321 {
 322         (*index)++;
 323         if (*index == (trb_in_seg - 1)) {
 324                 *index = 0;
 325                 *cs ^=  1;
 326         }
 327 }
 328
 329 /**
 330  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 331  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 332  */
 333 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 334 {
 335         priv_ep->free_trbs--;
 336         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 337 }
 338
 339 /**
 340  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 341  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 342  */
 343 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 344 {
 345         priv_ep->free_trbs++;
 346         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 347 }
 348
 349 static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 350 {
 351         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 352         int current_trb = priv_req->start_trb;
 353
 354         while (current_trb != priv_req->end_trb) {
 355                 cdns3_ep_inc_deq(priv_ep);
 356                 current_trb = priv_ep->dequeue;
 357         }
 358
 359         cdns3_ep_inc_deq(priv_ep);
 360 }
 361
 362 /**
 363  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 364  * @priv_dev: Extended gadget object
 365  * @enable: Enable/disable permit to transition to L1.
 366  *
 367  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 368  * then controller answer with ACK handshake.
 369  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 370  * then controller answer with NYET handshake.
 371  */
 372 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 373 {
 374         if (enable)
 375                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 376         else
 377                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 378 }
 379
 380 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 381 {
 382         u32 reg;
 383
 384         reg = readl(&priv_dev->regs->usb_sts);
 385
 386         if (DEV_SUPERSPEED(reg))
 387                 return USB_SPEED_SUPER;
 388         else if (DEV_HIGHSPEED(reg))
 389                 return USB_SPEED_HIGH;
 390         else if (DEV_FULLSPEED(reg))
 391                 return USB_SPEED_FULL;
 392         else if (DEV_LOWSPEED(reg))
 393                 return USB_SPEED_LOW;
 394         return USB_SPEED_UNKNOWN;
 395 }
 396
 397 /**
 398  * cdns3_start_all_request - add to ring all request not started
 399  * @priv_dev: Extended gadget object
 400  * @priv_ep: The endpoint for whom request will be started.
 401  *
 402  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 403  *         all requests.
 404  */
 405 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 406                                    struct cdns3_endpoint *priv_ep)
 407 {
 408         struct usb_request *request;
 409         int ret = 0;
 410         u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 411
 412         /*
 413          * If the last pending transfer is INTERNAL
 414          * OR streams are enabled for this endpoint
 415          * do NOT start new transfer till the last one is pending
 416          */
 417         if (!pending_empty) {
 418                 struct cdns3_request *priv_req;
 419
 420                 request = cdns3_next_request(&priv_ep->pending_req_list);
 421                 priv_req = to_cdns3_request(request);
 422                 if ((priv_req->flags & REQUEST_INTERNAL) ||
 423                     (priv_ep->flags & EP_TDLCHK_EN) ||
 424                         priv_ep->use_streams) {
 425                         dev_dbg(priv_dev->dev, "Blocking external request\n");
 426                         return ret;
 427                 }
 428         }
 429
 430         while (!list_empty(&priv_ep->deferred_req_list)) {
 431                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 432
 433                 if (!priv_ep->use_streams) {
 434                         ret = cdns3_ep_run_transfer(priv_ep, request);
 435                 } else {
 436                         priv_ep->stream_sg_idx = 0;
 437                         ret = cdns3_ep_run_stream_transfer(priv_ep, request);
 438                 }
 439                 if (ret)
 440                         return ret;
 441
 442                 list_del(&request->list);
 443                 list_add_tail(&request->list,
 444                               &priv_ep->pending_req_list);
 445                 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
 446                         break;
 447         }
 448
 449         priv_ep->flags &= ~EP_RING_FULL;
 450         return ret;
 451 }
 452
 453 /*
 454  * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 455  * driver try to detect whether endpoint need additional internal
 456  * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 457  * if before first DESCMISS interrupt the DMA will be armed.
 458  */
 459 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
 460         if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 461                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 462                 (reg) |= EP_STS_EN_DESCMISEN; \
 463         } } while (0)
 464
 465 static void __cdns3_descmiss_copy_data(struct usb_request *request,
 466         struct usb_request *descmiss_req)
 467 {
 468         int length = request->actual + descmiss_req->actual;
 469         struct scatterlist *s = request->sg;
 470
 471         if (!s) {
 472                 if (length <= request->length) {
 473                         memcpy(&((u8 *)request->buf)[request->actual],
 474                                descmiss_req->buf,
 475                                descmiss_req->actual);
 476                         request->actual = length;
 477                 } else {
 478                         /* It should never occures */
 479                         request->status = -ENOMEM;
 480                 }
 481         } else {
 482                 if (length <= sg_dma_len(s)) {
 483                         void *p = phys_to_virt(sg_dma_address(s));
 484
 485                         memcpy(&((u8 *)p)[request->actual],
 486                                 descmiss_req->buf,
 487                                 descmiss_req->actual);
 488                         request->actual = length;
 489                 } else {
 490                         request->status = -ENOMEM;
 491                 }
 492         }
 493 }
 494
 495 /**
 496  * cdns3_wa2_descmiss_copy_data copy data from internal requests to
 497  * request queued by class driver.
 498  * @priv_ep: extended endpoint object
 499  * @request: request object
 500  */
 501 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 502                                          struct usb_request *request)
 503 {
 504         struct usb_request *descmiss_req;
 505         struct cdns3_request *descmiss_priv_req;
 506
 507         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 508                 int chunk_end;
 509                 int length;
 510
 511                 descmiss_priv_req =
 512                         cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 513                 descmiss_req = &descmiss_priv_req->request;
 514
 515                 /* driver can't touch pending request */
 516                 if (descmiss_priv_req->flags & REQUEST_PENDING)
 517                         break;
 518
 519                 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 520                 length = request->actual + descmiss_req->actual;
 521                 request->status = descmiss_req->status;
 522                 __cdns3_descmiss_copy_data(request, descmiss_req);
 523                 list_del_init(&descmiss_priv_req->list);
 524                 kfree(descmiss_req->buf);
 525                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 526                 --priv_ep->wa2_counter;
 527
 528                 if (!chunk_end)
 529                         break;
 530         }
 531 }
 532
 533 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 534                                                      struct cdns3_endpoint *priv_ep,
 535                                                      struct cdns3_request *priv_req)
 536 {
 537         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 538             priv_req->flags & REQUEST_INTERNAL) {
 539                 struct usb_request *req;
 540
 541                 req = cdns3_next_request(&priv_ep->deferred_req_list);
 542
 543                 priv_ep->descmis_req = NULL;
 544
 545                 if (!req)
 546                         return NULL;
 547
 548                 /* unmap the gadget request before copying data */
 549                 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
 550                                                 priv_ep->dir);
 551
 552                 cdns3_wa2_descmiss_copy_data(priv_ep, req);
 553                 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 554                     req->length != req->actual) {
 555                         /* wait for next part of transfer */
 556                         /* re-map the gadget request buffer*/
 557                         usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
 558                                 usb_endpoint_dir_in(priv_ep->endpoint.desc));
 559                         return NULL;
 560                 }
 561
 562                 if (req->status == -EINPROGRESS)
 563                         req->status = 0;
 564
 565                 list_del_init(&req->list);
 566                 cdns3_start_all_request(priv_dev, priv_ep);
 567                 return req;
 568         }
 569
 570         return &priv_req->request;
 571 }
 572
 573 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 574                                      struct cdns3_endpoint *priv_ep,
 575                                      struct cdns3_request *priv_req)
 576 {
 577         int deferred = 0;
 578
 579         /*
 580          * If transfer was queued before DESCMISS appear than we
 581          * can disable handling of DESCMISS interrupt. Driver assumes that it
 582          * can disable special treatment for this endpoint.
 583          */
 584         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 585                 u32 reg;
 586
 587                 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 588                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 589                 reg = readl(&priv_dev->regs->ep_sts_en);
 590                 reg &= ~EP_STS_EN_DESCMISEN;
 591                 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 592                 writel(reg, &priv_dev->regs->ep_sts_en);
 593         }
 594
 595         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 596                 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 597                 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 598
 599                 /*
 600                  *  DESCMISS transfer has been finished, so data will be
 601                  *  directly copied from internal allocated usb_request
 602                  *  objects.
 603                  */
 604                 if (pending_empty && !descmiss_empty &&
 605                     !(priv_req->flags & REQUEST_INTERNAL)) {
 606                         cdns3_wa2_descmiss_copy_data(priv_ep,
 607                                                      &priv_req->request);
 608
 609                         trace_cdns3_wa2(priv_ep, "get internal stored data");
 610
 611                         list_add_tail(&priv_req->request.list,
 612                                       &priv_ep->pending_req_list);
 613                         cdns3_gadget_giveback(priv_ep, priv_req,
 614                                               priv_req->request.status);
 615
 616                         /*
 617                          * Intentionally driver returns positive value as
 618                          * correct value. It informs that transfer has
 619                          * been finished.
 620                          */
 621                         return EINPROGRESS;
 622                 }
 623
 624                 /*
 625                  * Driver will wait for completion DESCMISS transfer,
 626                  * before starts new, not DESCMISS transfer.
 627                  */
 628                 if (!pending_empty && !descmiss_empty) {
 629                         trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 630                         deferred = 1;
 631                 }
 632
 633                 if (priv_req->flags & REQUEST_INTERNAL)
 634                         list_add_tail(&priv_req->list,
 635                                       &priv_ep->wa2_descmiss_req_list);
 636         }
 637
 638         return deferred;
 639 }
 640
 641 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 642 {
 643         struct cdns3_request *priv_req;
 644
 645         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 646                 u8 chain;
 647
 648                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 649                 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 650
 651                 trace_cdns3_wa2(priv_ep, "removes eldest request");
 652
 653                 kfree(priv_req->request.buf);
 654                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 655                                              &priv_req->request);
 656                 list_del_init(&priv_req->list);
 657                 --priv_ep->wa2_counter;
 658
 659                 if (!chain)
 660                         break;
 661         }
 662 }
 663
 664 /**
 665  * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 666  * @priv_ep: extended gadget object
 667  *
 668  * This function is used only for WA2. For more information see Work around 2
 669  * description.
 670  */
 671 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 672 {
 673         struct cdns3_request *priv_req;
 674         struct usb_request *request;
 675         u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 676
 677         /* check for pending transfer */
 678         if (!pending_empty) {
 679                 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
 680                 return;
 681         }
 682
 683         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 684                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 685                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 686         }
 687
 688         trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 689
 690         if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
 691                 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
 692                 cdns3_wa2_remove_old_request(priv_ep);
 693         }
 694
 695         request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 696                                                 GFP_ATOMIC);
 697         if (!request)
 698                 goto err;
 699
 700         priv_req = to_cdns3_request(request);
 701         priv_req->flags |= REQUEST_INTERNAL;
 702
 703         /* if this field is still assigned it indicate that transfer related
 704          * with this request has not been finished yet. Driver in this
 705          * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 706          * flag to previous one. It will indicate that current request is
 707          * part of the previous one.
 708          */
 709         if (priv_ep->descmis_req)
 710                 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 711
 712         priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 713                                         GFP_ATOMIC);
 714         priv_ep->wa2_counter++;
 715
 716         if (!priv_req->request.buf) {
 717                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 718                 goto err;
 719         }
 720
 721         priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 722         priv_ep->descmis_req = priv_req;
 723
 724         __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 725                                 &priv_ep->descmis_req->request,
 726                                 GFP_ATOMIC);
 727
 728         return;
 729
 730 err:
 731         dev_err(priv_ep->cdns3_dev->dev,
 732                 "Failed: No sufficient memory for DESCMIS\n");
 733 }
 734
 735 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
 736 {
 737         u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 738
 739         if (tdl) {
 740                 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
 741
 742                 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
 743                        &priv_dev->regs->ep_cmd);
 744         }
 745 }
 746
 747 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
 748 {
 749         u32 ep_sts_reg;
 750
 751         /* select EP0-out */
 752         cdns3_select_ep(priv_dev, 0);
 753
 754         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 755
 756         if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
 757                 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
 758                 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
 759
 760                 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
 761                     outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
 762                         u8 pending_empty = list_empty(&outq_ep->pending_req_list);
 763
 764                         if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
 765                             (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
 766                             !pending_empty) {
 767                         } else {
 768                                 u32 ep_sts_en_reg;
 769                                 u32 ep_cmd_reg;
 770
 771                                 cdns3_select_ep(priv_dev, outq_ep->num |
 772                                                 outq_ep->dir);
 773                                 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
 774                                 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
 775
 776                                 outq_ep->flags |= EP_TDLCHK_EN;
 777                                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 778                                                        EP_CFG_TDL_CHK);
 779
 780                                 cdns3_wa2_enable_detection(priv_dev, outq_ep,
 781                                                            ep_sts_en_reg);
 782                                 writel(ep_sts_en_reg,
 783                                        &priv_dev->regs->ep_sts_en);
 784                                 /* reset tdl value to zero */
 785                                 cdns3_wa2_reset_tdl(priv_dev);
 786                                 /*
 787                                  * Memory barrier - Reset tdl before ringing the
 788                                  * doorbell.
 789                                  */
 790                                 wmb();
 791                                 if (EP_CMD_DRDY & ep_cmd_reg) {
 792                                         trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
 793
 794                                 } else {
 795                                         trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
 796                                         /*
 797                                          * ring doorbell to generate DESCMIS irq
 798                                          */
 799                                         writel(EP_CMD_DRDY,
 800                                                &priv_dev->regs->ep_cmd);
 801                                 }
 802                         }
 803                 }
 804         }
 805 }
 806
 807 /**
 808  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 809  * @priv_ep: The endpoint to whom the request belongs to
 810  * @priv_req: The request we're giving back
 811  * @status: completion code for the request
 812  *
 813  * Must be called with controller's lock held and interrupts disabled. This
 814  * function will unmap @req and call its ->complete() callback to notify upper
 815  * layers that it has completed.
 816  */
 817 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 818                            struct cdns3_request *priv_req,
 819                            int status)
 820 {
 821         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 822         struct usb_request *request = &priv_req->request;
 823
 824         list_del_init(&request->list);
 825
 826         if (request->status == -EINPROGRESS)
 827                 request->status = status;
 828
 829         usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 830                                         priv_ep->dir);
 831
 832         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 833             priv_ep->dir == USB_DIR_OUT && !request->status)
 834                 memcpy(request->buf, priv_req->aligned_buf->buf,
 835                        request->length);
 836
 837         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 838         /* All TRBs have finished, clear the counter */
 839         priv_req->finished_trb = 0;
 840         trace_cdns3_gadget_giveback(priv_req);
 841
 842         if (priv_dev->dev_ver < DEV_VER_V2) {
 843                 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 844                                                     priv_req);
 845                 if (!request)
 846                         return;
 847         }
 848
 849         if (request->complete) {
 850                 spin_unlock(&priv_dev->lock);
 851                 usb_gadget_giveback_request(&priv_ep->endpoint,
 852                                             request);
 853                 spin_lock(&priv_dev->lock);
 854         }
 855
 856         if (request->buf == priv_dev->zlp_buf)
 857                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 858 }
 859
 860 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 861 {
 862         /* Work around for stale data address in TRB*/
 863         if (priv_ep->wa1_set) {
 864                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 865
 866                 priv_ep->wa1_set = 0;
 867                 priv_ep->wa1_trb_index = 0xFFFF;
 868                 if (priv_ep->wa1_cycle_bit) {
 869                         priv_ep->wa1_trb->control =
 870                                 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
 871                 } else {
 872                         priv_ep->wa1_trb->control =
 873                                 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
 874                 }
 875         }
 876 }
 877
 878 static void cdns3_free_aligned_request_buf(struct work_struct *work)
 879 {
 880         struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 881                                         aligned_buf_wq);
 882         struct cdns3_aligned_buf *buf, *tmp;
 883         unsigned long flags;
 884
 885         spin_lock_irqsave(&priv_dev->lock, flags);
 886
 887         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 888                 if (!buf->in_use) {
 889                         list_del(&buf->list);
 890
 891                         /*
 892                          * Re-enable interrupts to free DMA capable memory.
 893                          * Driver can't free this memory with disabled
 894                          * interrupts.
 895                          */
 896                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 897                         dma_free_coherent(priv_dev->sysdev, buf->size,
 898                                           buf->buf, buf->dma);
 899                         kfree(buf);
 900                         spin_lock_irqsave(&priv_dev->lock, flags);
 901                 }
 902         }
 903
 904         spin_unlock_irqrestore(&priv_dev->lock, flags);
 905 }
 906
 907 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 908 {
 909         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 910         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 911         struct cdns3_aligned_buf *buf;
 912
 913         /* check if buffer is aligned to 8. */
 914         if (!((uintptr_t)priv_req->request.buf & 0x7))
 915                 return 0;
 916
 917         buf = priv_req->aligned_buf;
 918
 919         if (!buf || priv_req->request.length > buf->size) {
 920                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 921                 if (!buf)
 922                         return -ENOMEM;
 923
 924                 buf->size = priv_req->request.length;
 925
 926                 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
 927                                               buf->size,
 928                                               &buf->dma,
 929                                               GFP_ATOMIC);
 930                 if (!buf->buf) {
 931                         kfree(buf);
 932                         return -ENOMEM;
 933                 }
 934
 935                 if (priv_req->aligned_buf) {
 936                         trace_cdns3_free_aligned_request(priv_req);
 937                         priv_req->aligned_buf->in_use = 0;
 938                         queue_work(system_freezable_wq,
 939                                    &priv_dev->aligned_buf_wq);
 940                 }
 941
 942                 buf->in_use = 1;
 943                 priv_req->aligned_buf = buf;
 944
 945                 list_add_tail(&buf->list,
 946                               &priv_dev->aligned_buf_list);
 947         }
 948
 949         if (priv_ep->dir == USB_DIR_IN) {
 950                 memcpy(buf->buf, priv_req->request.buf,
 951                        priv_req->request.length);
 952         }
 953
 954         priv_req->flags |= REQUEST_UNALIGNED;
 955         trace_cdns3_prepare_aligned_request(priv_req);
 956
 957         return 0;
 958 }
 959
 960 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 961                                   struct cdns3_trb *trb)
 962 {
 963         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 964
 965         if (!priv_ep->wa1_set) {
 966                 u32 doorbell;
 967
 968                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 969
 970                 if (doorbell) {
 971                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 972                         priv_ep->wa1_set = 1;
 973                         priv_ep->wa1_trb = trb;
 974                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 975                         trace_cdns3_wa1(priv_ep, "set guard");
 976                         return 0;
 977                 }
 978         }
 979         return 1;
 980 }
 981
 982 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 983                                              struct cdns3_endpoint *priv_ep)
 984 {
 985         int dma_index;
 986         u32 doorbell;
 987
 988         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 989         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 990
 991         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 992                 cdns3_wa1_restore_cycle_bit(priv_ep);
 993 }
 994
 995 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
 996                                         struct usb_request *request)
 997 {
 998         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 999         struct cdns3_request *priv_req;
1000         struct cdns3_trb *trb;
1001         dma_addr_t trb_dma;
1002         int address;
1003         u32 control;
1004         u32 length;
1005         u32 tdl;
1006         unsigned int sg_idx = priv_ep->stream_sg_idx;
1007
1008         priv_req = to_cdns3_request(request);
1009         address = priv_ep->endpoint.desc->bEndpointAddress;
1010
1011         priv_ep->flags |= EP_PENDING_REQUEST;
1012
1013         /* must allocate buffer aligned to 8 */
1014         if (priv_req->flags & REQUEST_UNALIGNED)
1015                 trb_dma = priv_req->aligned_buf->dma;
1016         else
1017                 trb_dma = request->dma;
1018
1019         /*  For stream capable endpoints driver use only single TD. */
1020         trb = priv_ep->trb_pool + priv_ep->enqueue;
1021         priv_req->start_trb = priv_ep->enqueue;
1022         priv_req->end_trb = priv_req->start_trb;
1023         priv_req->trb = trb;
1024
1025         cdns3_select_ep(priv_ep->cdns3_dev, address);
1026
1027         control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1028                   TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1029
1030         if (!request->num_sgs) {
1031                 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1032                 length = request->length;
1033         } else {
1034                 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1035                 length = request->sg[sg_idx].length;
1036         }
1037
1038         tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1039
1040         trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1041
1042         /*
1043          * For DEV_VER_V2 controller version we have enabled
1044          * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1045          * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1046          */
1047         if (priv_dev->dev_ver >= DEV_VER_V2) {
1048                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1049                         trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1050         }
1051         priv_req->flags |= REQUEST_PENDING;
1052
1053         trb->control = cpu_to_le32(control);
1054
1055         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1056
1057         /*
1058          * Memory barrier - Cycle Bit must be set before trb->length  and
1059          * trb->buffer fields.
1060          */
1061         wmb();
1062
1063         /* always first element */
1064         writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1065                &priv_dev->regs->ep_traddr);
1066
1067         if (!(priv_ep->flags & EP_STALLED)) {
1068                 trace_cdns3_ring(priv_ep);
1069                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1070                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1071
1072                 priv_ep->prime_flag = false;
1073
1074                 /*
1075                  * Controller version DEV_VER_V2 tdl calculation
1076                  * is based on TRB
1077                  */
1078
1079                 if (priv_dev->dev_ver < DEV_VER_V2)
1080                         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1081                                &priv_dev->regs->ep_cmd);
1082                 else if (priv_dev->dev_ver > DEV_VER_V2)
1083                         writel(tdl, &priv_dev->regs->ep_tdl);
1084
1085                 priv_ep->last_stream_id = priv_req->request.stream_id;
1086                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1087                 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1088                        EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1089
1090                 trace_cdns3_doorbell_epx(priv_ep->name,
1091                                          readl(&priv_dev->regs->ep_traddr));
1092         }
1093
1094         /* WORKAROUND for transition to L0 */
1095         __cdns3_gadget_wakeup(priv_dev);
1096
1097         return 0;
1098 }
1099
1100 /**
1101  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1102  * @priv_ep: endpoint object
1103  * @request: request object
1104  *
1105  * Returns zero on success or negative value on failure
1106  */
1107 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1108                                  struct usb_request *request)
1109 {
1110         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1111         struct cdns3_request *priv_req;
1112         struct cdns3_trb *trb;
1113         struct cdns3_trb *link_trb;
1114         dma_addr_t trb_dma;
1115         u32 togle_pcs = 1;
1116         int sg_iter = 0;
1117         int num_trb;
1118         int address;
1119         u32 control;
1120         int pcs;
1121         u16 total_tdl = 0;
1122         struct scatterlist *s = NULL;
1123         bool sg_supported = !!(request->num_mapped_sgs);
1124
1125         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1126                 num_trb = priv_ep->interval;
1127         else
1128                 num_trb = sg_supported ? request->num_mapped_sgs : 1;
1129
1130         if (num_trb > priv_ep->free_trbs) {
1131                 priv_ep->flags |= EP_RING_FULL;
1132                 return -ENOBUFS;
1133         }
1134
1135         priv_req = to_cdns3_request(request);
1136         address = priv_ep->endpoint.desc->bEndpointAddress;
1137
1138         priv_ep->flags |= EP_PENDING_REQUEST;
1139
1140         /* must allocate buffer aligned to 8 */
1141         if (priv_req->flags & REQUEST_UNALIGNED)
1142                 trb_dma = priv_req->aligned_buf->dma;
1143         else
1144                 trb_dma = request->dma;
1145
1146         trb = priv_ep->trb_pool + priv_ep->enqueue;
1147         priv_req->start_trb = priv_ep->enqueue;
1148         priv_req->trb = trb;
1149
1150         cdns3_select_ep(priv_ep->cdns3_dev, address);
1151
1152         /* prepare ring */
1153         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
1154                 int doorbell, dma_index;
1155                 u32 ch_bit = 0;
1156
1157                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1158                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1159
1160                 /* Driver can't update LINK TRB if it is current processed. */
1161                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1162                         priv_ep->flags |= EP_DEFERRED_DRDY;
1163                         return -ENOBUFS;
1164                 }
1165
1166                 /*updating C bt in  Link TRB before starting DMA*/
1167                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1168                 /*
1169                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1170                  * that DMA stuck at the LINK TRB.
1171                  * On the other hand, removing TRB_CHAIN for longer TRs for
1172                  * epXout cause that DMA stuck after handling LINK TRB.
1173                  * To eliminate this strange behavioral driver set TRB_CHAIN
1174                  * bit only for TR size > 2.
1175                  */
1176                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1177                     TRBS_PER_SEGMENT > 2)
1178                         ch_bit = TRB_CHAIN;
1179
1180                 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1181                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1182         }
1183
1184         if (priv_dev->dev_ver <= DEV_VER_V2)
1185                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1186
1187         if (sg_supported)
1188                 s = request->sg;
1189
1190         /* set incorrect Cycle Bit for first trb*/
1191         control = priv_ep->pcs ? 0 : TRB_CYCLE;
1192
1193         do {
1194                 u32 length;
1195                 u16 td_size = 0;
1196
1197                 /* fill TRB */
1198                 control |= TRB_TYPE(TRB_NORMAL);
1199                 if (sg_supported) {
1200                         trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1201                         length = sg_dma_len(s);
1202                 } else {
1203                         trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1204                         length = request->length;
1205                 }
1206
1207                 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
1208                         td_size = DIV_ROUND_UP(length,
1209                                                priv_ep->endpoint.maxpacket);
1210                 else if (priv_ep->flags & EP_TDLCHK_EN)
1211                         total_tdl += DIV_ROUND_UP(length,
1212                                                priv_ep->endpoint.maxpacket);
1213
1214                 trb->length = cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1215                                         TRB_LEN(length));
1216                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1217                         trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1218                 else
1219                         control |= TRB_TDL_HS_SIZE(td_size);
1220
1221                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1222
1223                 /*
1224                  * first trb should be prepared as last to avoid processing
1225                  *  transfer to early
1226                  */
1227                 if (sg_iter != 0)
1228                         control |= pcs;
1229
1230                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
1231                         control |= TRB_IOC | TRB_ISP;
1232                 } else {
1233                         /* for last element in TD or in SG list */
1234                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
1235                                 control |= pcs | TRB_IOC | TRB_ISP;
1236                 }
1237
1238                 if (sg_iter)
1239                         trb->control = cpu_to_le32(control);
1240                 else
1241                         priv_req->trb->control = cpu_to_le32(control);
1242
1243                 if (sg_supported) {
1244                         trb->control |= TRB_ISP;
1245                         /* Don't set chain bit for last TRB */
1246                         if (sg_iter < num_trb - 1)
1247                                 trb->control |= TRB_CHAIN;
1248
1249                         s = sg_next(s);
1250                 }
1251
1252                 control = 0;
1253                 ++sg_iter;
1254                 priv_req->end_trb = priv_ep->enqueue;
1255                 cdns3_ep_inc_enq(priv_ep);
1256                 trb = priv_ep->trb_pool + priv_ep->enqueue;
1257         } while (sg_iter < num_trb);
1258
1259         trb = priv_req->trb;
1260
1261         priv_req->flags |= REQUEST_PENDING;
1262         priv_req->num_of_trb = num_trb;
1263
1264         if (sg_iter == 1)
1265                 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1266
1267         if (priv_dev->dev_ver < DEV_VER_V2 &&
1268             (priv_ep->flags & EP_TDLCHK_EN)) {
1269                 u16 tdl = total_tdl;
1270                 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1271
1272                 if (tdl > EP_CMD_TDL_MAX) {
1273                         tdl = EP_CMD_TDL_MAX;
1274                         priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1275                 }
1276
1277                 if (old_tdl < tdl) {
1278                         tdl -= old_tdl;
1279                         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1280                                &priv_dev->regs->ep_cmd);
1281                 }
1282         }
1283
1284         /*
1285          * Memory barrier - cycle bit must be set before other filds in trb.
1286          */
1287         wmb();
1288
1289         /* give the TD to the consumer*/
1290         if (togle_pcs)
1291                 trb->control = trb->control ^ cpu_to_le32(1);
1292
1293         if (priv_dev->dev_ver <= DEV_VER_V2)
1294                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1295
1296         if (num_trb > 1) {
1297                 int i = 0;
1298
1299                 while (i < num_trb) {
1300                         trace_cdns3_prepare_trb(priv_ep, trb + i);
1301                         if (trb + i == link_trb) {
1302                                 trb = priv_ep->trb_pool;
1303                                 num_trb = num_trb - i;
1304                                 i = 0;
1305                         } else {
1306                                 i++;
1307                         }
1308                 }
1309         } else {
1310                 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1311         }
1312
1313         /*
1314          * Memory barrier - Cycle Bit must be set before trb->length  and
1315          * trb->buffer fields.
1316          */
1317         wmb();
1318
1319         /*
1320          * For DMULT mode we can set address to transfer ring only once after
1321          * enabling endpoint.
1322          */
1323         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1324                 /*
1325                  * Until SW is not ready to handle the OUT transfer the ISO OUT
1326                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1327                  * EP_CFG_ENABLE must be set before updating ep_traddr.
1328                  */
1329                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
1330                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1331                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1332                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1333                                                EP_CFG_ENABLE);
1334                 }
1335
1336                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1337                                         priv_req->start_trb * TRB_SIZE),
1338                                         &priv_dev->regs->ep_traddr);
1339
1340                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1341         }
1342
1343         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1344                 trace_cdns3_ring(priv_ep);
1345                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1346                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1347                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1348                 trace_cdns3_doorbell_epx(priv_ep->name,
1349                                          readl(&priv_dev->regs->ep_traddr));
1350         }
1351
1352         /* WORKAROUND for transition to L0 */
1353         __cdns3_gadget_wakeup(priv_dev);
1354
1355         return 0;
1356 }
1357
1358 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1359 {
1360         struct cdns3_endpoint *priv_ep;
1361         struct usb_ep *ep;
1362
1363         if (priv_dev->hw_configured_flag)
1364                 return;
1365
1366         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1367
1368         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1369                                USB_CONF_U1EN | USB_CONF_U2EN);
1370
1371         priv_dev->hw_configured_flag = 1;
1372
1373         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1374                 if (ep->enabled) {
1375                         priv_ep = ep_to_cdns3_ep(ep);
1376                         cdns3_start_all_request(priv_dev, priv_ep);
1377                 }
1378         }
1379
1380         cdns3_allow_enable_l1(priv_dev, 1);
1381 }
1382
1383 /**
1384  * cdns3_trb_handled - check whether trb has been handled by DMA
1385  *
1386  * @priv_ep: extended endpoint object.
1387  * @priv_req: request object for checking
1388  *
1389  * Endpoint must be selected before invoking this function.
1390  *
1391  * Returns false if request has not been handled by DMA, else returns true.
1392  *
1393  * SR - start ring
1394  * ER -  end ring
1395  * DQ = priv_ep->dequeue - dequeue position
1396  * EQ = priv_ep->enqueue -  enqueue position
1397  * ST = priv_req->start_trb - index of first TRB in transfer ring
1398  * ET = priv_req->end_trb - index of last TRB in transfer ring
1399  * CI = current_index - index of processed TRB by DMA.
1400  *
1401  * As first step, we check if the TRB between the ST and ET.
1402  * Then, we check if cycle bit for index priv_ep->dequeue
1403  * is correct.
1404  *
1405  * some rules:
1406  * 1. priv_ep->dequeue never equals to current_index.
1407  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1408  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1409  *    and priv_ep->free_trbs is zero.
1410  *    This case indicate that TR is full.
1411  *
1412  * At below two cases, the request have been handled.
1413  * Case 1 - priv_ep->dequeue < current_index
1414  *      SR ... EQ ... DQ ... CI ... ER
1415  *      SR ... DQ ... CI ... EQ ... ER
1416  *
1417  * Case 2 - priv_ep->dequeue > current_index
1418  * This situation takes place when CI go through the LINK TRB at the end of
1419  * transfer ring.
1420  *      SR ... CI ... EQ ... DQ ... ER
1421  */
1422 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1423                                   struct cdns3_request *priv_req)
1424 {
1425         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1426         struct cdns3_trb *trb;
1427         int current_index = 0;
1428         int handled = 0;
1429         int doorbell;
1430
1431         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1432         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1433
1434         /* current trb doesn't belong to this request */
1435         if (priv_req->start_trb < priv_req->end_trb) {
1436                 if (priv_ep->dequeue > priv_req->end_trb)
1437                         goto finish;
1438
1439                 if (priv_ep->dequeue < priv_req->start_trb)
1440                         goto finish;
1441         }
1442
1443         if ((priv_req->start_trb > priv_req->end_trb) &&
1444                 (priv_ep->dequeue > priv_req->end_trb) &&
1445                 (priv_ep->dequeue < priv_req->start_trb))
1446                 goto finish;
1447
1448         if ((priv_req->start_trb == priv_req->end_trb) &&
1449                 (priv_ep->dequeue != priv_req->end_trb))
1450                 goto finish;
1451
1452         trb = &priv_ep->trb_pool[priv_ep->dequeue];
1453
1454         if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1455                 goto finish;
1456
1457         if (doorbell == 1 && current_index == priv_ep->dequeue)
1458                 goto finish;
1459
1460         /* The corner case for TRBS_PER_SEGMENT equal 2). */
1461         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1462                 handled = 1;
1463                 goto finish;
1464         }
1465
1466         if (priv_ep->enqueue == priv_ep->dequeue &&
1467             priv_ep->free_trbs == 0) {
1468                 handled = 1;
1469         } else if (priv_ep->dequeue < current_index) {
1470                 if ((current_index == (priv_ep->num_trbs - 1)) &&
1471                     !priv_ep->dequeue)
1472                         goto finish;
1473
1474                 handled = 1;
1475         } else if (priv_ep->dequeue  > current_index) {
1476                         handled = 1;
1477         }
1478
1479 finish:
1480         trace_cdns3_request_handled(priv_req, current_index, handled);
1481
1482         return handled;
1483 }
1484
1485 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1486                                      struct cdns3_endpoint *priv_ep)
1487 {
1488         struct cdns3_request *priv_req;
1489         struct usb_request *request;
1490         struct cdns3_trb *trb;
1491         bool request_handled = false;
1492         bool transfer_end = false;
1493
1494         while (!list_empty(&priv_ep->pending_req_list)) {
1495                 request = cdns3_next_request(&priv_ep->pending_req_list);
1496                 priv_req = to_cdns3_request(request);
1497
1498                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1499
1500                 /* Request was dequeued and TRB was changed to TRB_LINK. */
1501                 if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1502                         trace_cdns3_complete_trb(priv_ep, trb);
1503                         cdns3_move_deq_to_next_trb(priv_req);
1504                 }
1505
1506                 if (!request->stream_id) {
1507                         /* Re-select endpoint. It could be changed by other CPU
1508                          * during handling usb_gadget_giveback_request.
1509                          */
1510                         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1511
1512                         while (cdns3_trb_handled(priv_ep, priv_req)) {
1513                                 priv_req->finished_trb++;
1514                                 if (priv_req->finished_trb >= priv_req->num_of_trb)
1515                                         request_handled = true;
1516
1517                                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1518                                 trace_cdns3_complete_trb(priv_ep, trb);
1519
1520                                 if (!transfer_end)
1521                                         request->actual +=
1522                                                 TRB_LEN(le32_to_cpu(trb->length));
1523
1524                                 if (priv_req->num_of_trb > 1 &&
1525                                         le32_to_cpu(trb->control) & TRB_SMM)
1526                                         transfer_end = true;
1527
1528                                 cdns3_ep_inc_deq(priv_ep);
1529                         }
1530
1531                         if (request_handled) {
1532                                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1533                                 request_handled = false;
1534                                 transfer_end = false;
1535                         } else {
1536                                 goto prepare_next_td;
1537                         }
1538
1539                         if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1540                             TRBS_PER_SEGMENT == 2)
1541                                 break;
1542                 } else {
1543                         /* Re-select endpoint. It could be changed by other CPU
1544                          * during handling usb_gadget_giveback_request.
1545                          */
1546                         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1547
1548                         trb = priv_ep->trb_pool;
1549                         trace_cdns3_complete_trb(priv_ep, trb);
1550
1551                         if (trb != priv_req->trb)
1552                                 dev_warn(priv_dev->dev,
1553                                          "request_trb=0x%p, queue_trb=0x%p\n",
1554                                          priv_req->trb, trb);
1555
1556                         request->actual += TRB_LEN(le32_to_cpu(trb->length));
1557
1558                         if (!request->num_sgs ||
1559                             (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1560                                 priv_ep->stream_sg_idx = 0;
1561                                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1562                         } else {
1563                                 priv_ep->stream_sg_idx++;
1564                                 cdns3_ep_run_stream_transfer(priv_ep, request);
1565                         }
1566                         break;
1567                 }
1568         }
1569         priv_ep->flags &= ~EP_PENDING_REQUEST;
1570
1571 prepare_next_td:
1572         if (!(priv_ep->flags & EP_STALLED) &&
1573             !(priv_ep->flags & EP_STALL_PENDING))
1574                 cdns3_start_all_request(priv_dev, priv_ep);
1575 }
1576
1577 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1578 {
1579         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1580
1581         cdns3_wa1_restore_cycle_bit(priv_ep);
1582
1583         if (rearm) {
1584                 trace_cdns3_ring(priv_ep);
1585
1586                 /* Cycle Bit must be updated before arming DMA. */
1587                 wmb();
1588                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1589
1590                 __cdns3_gadget_wakeup(priv_dev);
1591
1592                 trace_cdns3_doorbell_epx(priv_ep->name,
1593                                          readl(&priv_dev->regs->ep_traddr));
1594         }
1595 }
1596
1597 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1598 {
1599         u16 tdl = priv_ep->pending_tdl;
1600         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1601
1602         if (tdl > EP_CMD_TDL_MAX) {
1603                 tdl = EP_CMD_TDL_MAX;
1604                 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1605         } else {
1606                 priv_ep->pending_tdl = 0;
1607         }
1608
1609         writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1610 }
1611
1612 /**
1613  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1614  * @priv_ep: endpoint object
1615  *
1616  * Returns 0
1617  */
1618 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1619 {
1620         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1621         u32 ep_sts_reg;
1622         struct usb_request *deferred_request;
1623         struct usb_request *pending_request;
1624         u32 tdl = 0;
1625
1626         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1627
1628         trace_cdns3_epx_irq(priv_dev, priv_ep);
1629
1630         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1631         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1632
1633         if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1634                 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1635
1636                 tdl = cdns3_get_tdl(priv_dev);
1637
1638                 /*
1639                  * Continue the previous transfer:
1640                  * There is some racing between ERDY and PRIME. The device send
1641                  * ERDY and almost in the same time Host send PRIME. It cause
1642                  * that host ignore the ERDY packet and driver has to send it
1643                  * again.
1644                  */
1645                 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1646                     EP_STS_HOSTPP(ep_sts_reg))) {
1647                         writel(EP_CMD_ERDY |
1648                                EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1649                                &priv_dev->regs->ep_cmd);
1650                         ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1651                 } else {
1652                         priv_ep->prime_flag = true;
1653
1654                         pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1655                         deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1656
1657                         if (deferred_request && !pending_request) {
1658                                 cdns3_start_all_request(priv_dev, priv_ep);
1659                         }
1660                 }
1661         }
1662
1663         if (ep_sts_reg & EP_STS_TRBERR) {
1664                 if (priv_ep->flags & EP_STALL_PENDING &&
1665                     !(ep_sts_reg & EP_STS_DESCMIS &&
1666                     priv_dev->dev_ver < DEV_VER_V2)) {
1667                         cdns3_ep_stall_flush(priv_ep);
1668                 }
1669
1670                 /*
1671                  * For isochronous transfer driver completes request on
1672                  * IOC or on TRBERR. IOC appears only when device receive
1673                  * OUT data packet. If host disable stream or lost some packet
1674                  * then the only way to finish all queued transfer is to do it
1675                  * on TRBERR event.
1676                  */
1677                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1678                     !priv_ep->wa1_set) {
1679                         if (!priv_ep->dir) {
1680                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1681
1682                                 ep_cfg &= ~EP_CFG_ENABLE;
1683                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1684                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1685                         }
1686                         cdns3_transfer_completed(priv_dev, priv_ep);
1687                 } else if (!(priv_ep->flags & EP_STALLED) &&
1688                           !(priv_ep->flags & EP_STALL_PENDING)) {
1689                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
1690                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1691                                 cdns3_start_all_request(priv_dev, priv_ep);
1692                         } else {
1693                                 cdns3_rearm_transfer(priv_ep,
1694                                                      priv_ep->wa1_set);
1695                         }
1696                 }
1697         }
1698
1699         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1700             (ep_sts_reg & EP_STS_IOT)) {
1701                 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1702                         if (ep_sts_reg & EP_STS_ISP)
1703                                 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1704                         else
1705                                 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1706                 }
1707
1708                 if (!priv_ep->use_streams) {
1709                         if ((ep_sts_reg & EP_STS_IOC) ||
1710                             (ep_sts_reg & EP_STS_ISP)) {
1711                                 cdns3_transfer_completed(priv_dev, priv_ep);
1712                         } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1713                                    priv_ep->pending_tdl) {
1714                                 /* handle IOT with pending tdl */
1715                                 cdns3_reprogram_tdl(priv_ep);
1716                         }
1717                 } else if (priv_ep->dir == USB_DIR_OUT) {
1718                         priv_ep->ep_sts_pending |= ep_sts_reg;
1719                 } else if (ep_sts_reg & EP_STS_IOT) {
1720                         cdns3_transfer_completed(priv_dev, priv_ep);
1721                 }
1722         }
1723
1724         /*
1725          * MD_EXIT interrupt sets when stream capable endpoint exits
1726          * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1727          */
1728         if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1729             (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1730                 priv_ep->ep_sts_pending = 0;
1731                 cdns3_transfer_completed(priv_dev, priv_ep);
1732         }
1733
1734         /*
1735          * WA2: this condition should only be meet when
1736          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1737          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1738          * In other cases this interrupt will be disabled.
1739          */
1740         if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1741             !(priv_ep->flags & EP_STALLED))
1742                 cdns3_wa2_descmissing_packet(priv_ep);
1743
1744         return 0;
1745 }
1746
1747 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1748 {
1749         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1750                 spin_unlock(&priv_dev->lock);
1751                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1752                 spin_lock(&priv_dev->lock);
1753         }
1754 }
1755
1756 /**
1757  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1758  * @priv_dev: extended gadget object
1759  * @usb_ists: bitmap representation of device's reported interrupts
1760  * (usb_ists register value)
1761  */
1762 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1763                                               u32 usb_ists)
1764 {
1765         int speed = 0;
1766
1767         trace_cdns3_usb_irq(priv_dev, usb_ists);
1768         if (usb_ists & USB_ISTS_L1ENTI) {
1769                 /*
1770                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
1771                  * from L1. To fix it, if any DMA transfer is pending driver
1772                  * must starts driving resume signal immediately.
1773                  */
1774                 if (readl(&priv_dev->regs->drbl))
1775                         __cdns3_gadget_wakeup(priv_dev);
1776         }
1777
1778         /* Connection detected */
1779         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1780                 speed = cdns3_get_speed(priv_dev);
1781                 priv_dev->gadget.speed = speed;
1782                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1783                 cdns3_ep0_config(priv_dev);
1784         }
1785
1786         /* Disconnection detected */
1787         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1788                 cdns3_disconnect_gadget(priv_dev);
1789                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1790                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1791                 cdns3_hw_reset_eps_config(priv_dev);
1792         }
1793
1794         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1795                 if (priv_dev->gadget_driver &&
1796                     priv_dev->gadget_driver->suspend) {
1797                         spin_unlock(&priv_dev->lock);
1798                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1799                         spin_lock(&priv_dev->lock);
1800                 }
1801         }
1802
1803         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1804                 if (priv_dev->gadget_driver &&
1805                     priv_dev->gadget_driver->resume) {
1806                         spin_unlock(&priv_dev->lock);
1807                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1808                         spin_lock(&priv_dev->lock);
1809                 }
1810         }
1811
1812         /* reset*/
1813         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1814                 if (priv_dev->gadget_driver) {
1815                         spin_unlock(&priv_dev->lock);
1816                         usb_gadget_udc_reset(&priv_dev->gadget,
1817                                              priv_dev->gadget_driver);
1818                         spin_lock(&priv_dev->lock);
1819
1820                         /*read again to check the actual speed*/
1821                         speed = cdns3_get_speed(priv_dev);
1822                         priv_dev->gadget.speed = speed;
1823                         cdns3_hw_reset_eps_config(priv_dev);
1824                         cdns3_ep0_config(priv_dev);
1825                 }
1826         }
1827 }
1828
1829 /**
1830  * cdns3_device_irq_handler- interrupt handler for device part of controller
1831  *
1832  * @irq: irq number for cdns3 core device
1833  * @data: structure of cdns3
1834  *
1835  * Returns IRQ_HANDLED or IRQ_NONE
1836  */
1837 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1838 {
1839         struct cdns3_device *priv_dev = data;
1840         struct cdns3 *cdns = dev_get_drvdata(priv_dev->dev);
1841         irqreturn_t ret = IRQ_NONE;
1842         u32 reg;
1843
1844         if (cdns->in_lpm)
1845                 return ret;
1846
1847         /* check USB device interrupt */
1848         reg = readl(&priv_dev->regs->usb_ists);
1849         if (reg) {
1850                 /* After masking interrupts the new interrupts won't be
1851                  * reported in usb_ists/ep_ists. In order to not lose some
1852                  * of them driver disables only detected interrupts.
1853                  * They will be enabled ASAP after clearing source of
1854                  * interrupt. This an unusual behavior only applies to
1855                  * usb_ists register.
1856                  */
1857                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1858                 /* mask deferred interrupt. */
1859                 writel(reg, &priv_dev->regs->usb_ien);
1860                 ret = IRQ_WAKE_THREAD;
1861         }
1862
1863         /* check endpoint interrupt */
1864         reg = readl(&priv_dev->regs->ep_ists);
1865         if (reg) {
1866                 writel(0, &priv_dev->regs->ep_ien);
1867                 ret = IRQ_WAKE_THREAD;
1868         }
1869
1870         return ret;
1871 }
1872
1873 /**
1874  * cdns3_device_thread_irq_handler- interrupt handler for device part
1875  * of controller
1876  *
1877  * @irq: irq number for cdns3 core device
1878  * @data: structure of cdns3
1879  *
1880  * Returns IRQ_HANDLED or IRQ_NONE
1881  */
1882 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1883 {
1884         struct cdns3_device *priv_dev = data;
1885         irqreturn_t ret = IRQ_NONE;
1886         unsigned long flags;
1887         unsigned int bit;
1888         unsigned long reg;
1889
1890         spin_lock_irqsave(&priv_dev->lock, flags);
1891
1892         reg = readl(&priv_dev->regs->usb_ists);
1893         if (reg) {
1894                 writel(reg, &priv_dev->regs->usb_ists);
1895                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1896                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1897                 ret = IRQ_HANDLED;
1898         }
1899
1900         reg = readl(&priv_dev->regs->ep_ists);
1901
1902         /* handle default endpoint OUT */
1903         if (reg & EP_ISTS_EP_OUT0) {
1904                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1905                 ret = IRQ_HANDLED;
1906         }
1907
1908         /* handle default endpoint IN */
1909         if (reg & EP_ISTS_EP_IN0) {
1910                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1911                 ret = IRQ_HANDLED;
1912         }
1913
1914         /* check if interrupt from non default endpoint, if no exit */
1915         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1916         if (!reg)
1917                 goto irqend;
1918
1919         for_each_set_bit(bit, &reg,
1920                          sizeof(u32) * BITS_PER_BYTE) {
1921                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1922                 ret = IRQ_HANDLED;
1923         }
1924
1925         if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1926                 cdns3_wa2_check_outq_status(priv_dev);
1927
1928 irqend:
1929         writel(~0, &priv_dev->regs->ep_ien);
1930         spin_unlock_irqrestore(&priv_dev->lock, flags);
1931
1932         return ret;
1933 }
1934
1935 /**
1936  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1937  *
1938  * The real reservation will occur during write to EP_CFG register,
1939  * this function is used to check if the 'size' reservation is allowed.
1940  *
1941  * @priv_dev: extended gadget object
1942  * @size: the size (KB) for EP would like to allocate
1943  * @is_in: endpoint direction
1944  *
1945  * Return 0 if the required size can met or negative value on failure
1946  */
1947 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1948                                           int size, int is_in)
1949 {
1950         int remained;
1951
1952         /* 2KB are reserved for EP0*/
1953         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1954
1955         if (is_in) {
1956                 if (remained < size)
1957                         return -EPERM;
1958
1959                 priv_dev->onchip_used_size += size;
1960         } else {
1961                 int required;
1962
1963                 /**
1964                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1965                  * driver checks if it already has assigned enough buffers
1966                  */
1967                 if (priv_dev->out_mem_is_allocated >= size)
1968                         return 0;
1969
1970                 required = size - priv_dev->out_mem_is_allocated;
1971
1972                 if (required > remained)
1973                         return -EPERM;
1974
1975                 priv_dev->out_mem_is_allocated += required;
1976                 priv_dev->onchip_used_size += required;
1977         }
1978
1979         return 0;
1980 }
1981
1982 static void cdns3_stream_ep_reconfig(struct cdns3_device *priv_dev,
1983                                      struct cdns3_endpoint *priv_ep)
1984 {
1985         if (!priv_ep->use_streams || priv_dev->gadget.speed < USB_SPEED_SUPER)
1986                 return;
1987
1988         if (priv_dev->dev_ver >= DEV_VER_V3) {
1989                 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
1990
1991                 /*
1992                  * Stream capable endpoints are handled by using ep_tdl
1993                  * register. Other endpoints use TDL from TRB feature.
1994                  */
1995                 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, mask);
1996         }
1997
1998         /*  Enable Stream Bit TDL chk and SID chk */
1999         cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_STREAM_EN |
2000                                EP_CFG_TDL_CHK | EP_CFG_SID_CHK);
2001 }
2002
2003 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
2004                                   struct cdns3_endpoint *priv_ep)
2005 {
2006         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2007
2008         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
2009         if (priv_dev->dev_ver <= DEV_VER_V2)
2010                 writel(USB_CONF_DMULT, &regs->usb_conf);
2011
2012         if (priv_dev->dev_ver == DEV_VER_V2)
2013                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
2014
2015         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2016                 u32 mask;
2017
2018                 if (priv_ep->dir)
2019                         mask = BIT(priv_ep->num + 16);
2020                 else
2021                         mask = BIT(priv_ep->num);
2022
2023                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2024                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2025                         cdns3_set_register_bit(&regs->tdl_beh, mask);
2026                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
2027                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
2028                 }
2029
2030                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2031                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2032
2033                 cdns3_set_register_bit(&regs->dtrans, mask);
2034         }
2035 }
2036
2037 /**
2038  * cdns3_ep_config Configure hardware endpoint
2039  * @priv_ep: extended endpoint object
2040  */
2041 void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
2042 {
2043         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2044         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2045         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2046         u32 max_packet_size = 0;
2047         u8 maxburst = 0;
2048         u32 ep_cfg = 0;
2049         u8 buffering;
2050         u8 mult = 0;
2051         int ret;
2052
2053         buffering = CDNS3_EP_BUF_SIZE - 1;
2054
2055         cdns3_configure_dmult(priv_dev, priv_ep);
2056
2057         switch (priv_ep->type) {
2058         case USB_ENDPOINT_XFER_INT:
2059                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2060
2061                 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
2062                     priv_dev->dev_ver > DEV_VER_V2)
2063                         ep_cfg |= EP_CFG_TDL_CHK;
2064                 break;
2065         case USB_ENDPOINT_XFER_BULK:
2066                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2067
2068                 if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
2069                     priv_dev->dev_ver > DEV_VER_V2)
2070                         ep_cfg |= EP_CFG_TDL_CHK;
2071                 break;
2072         default:
2073                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2074                 mult = CDNS3_EP_ISO_HS_MULT - 1;
2075                 buffering = mult + 1;
2076         }
2077
2078         switch (priv_dev->gadget.speed) {
2079         case USB_SPEED_FULL:
2080                 max_packet_size = is_iso_ep ? 1023 : 64;
2081                 break;
2082         case USB_SPEED_HIGH:
2083                 max_packet_size = is_iso_ep ? 1024 : 512;
2084                 break;
2085         case USB_SPEED_SUPER:
2086                 /* It's limitation that driver assumes in driver. */
2087                 mult = 0;
2088                 max_packet_size = 1024;
2089                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2090                         maxburst = CDNS3_EP_ISO_SS_BURST - 1;
2091                         buffering = (mult + 1) *
2092                                     (maxburst + 1);
2093
2094                         if (priv_ep->interval > 1)
2095                                 buffering++;
2096                 } else {
2097                         maxburst = CDNS3_EP_BUF_SIZE - 1;
2098                 }
2099                 break;
2100         default:
2101                 /* all other speed are not supported */
2102                 return;
2103         }
2104
2105         if (max_packet_size == 1024)
2106                 priv_ep->trb_burst_size = 128;
2107         else if (max_packet_size >= 512)
2108                 priv_ep->trb_burst_size = 64;
2109         else
2110                 priv_ep->trb_burst_size = 16;
2111
2112         ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2113                                              !!priv_ep->dir);
2114         if (ret) {
2115                 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2116                 return;
2117         }
2118
2119         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2120                   EP_CFG_MULT(mult) |
2121                   EP_CFG_BUFFERING(buffering) |
2122                   EP_CFG_MAXBURST(maxburst);
2123
2124         cdns3_select_ep(priv_dev, bEndpointAddress);
2125         writel(ep_cfg, &priv_dev->regs->ep_cfg);
2126
2127         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2128                 priv_ep->name, ep_cfg);
2129 }
2130
2131 /* Find correct direction for HW endpoint according to description */
2132 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2133                                    struct cdns3_endpoint *priv_ep)
2134 {
2135         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2136                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2137 }
2138
2139 static struct
2140 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2141                                         struct usb_endpoint_descriptor *desc)
2142 {
2143         struct usb_ep *ep;
2144         struct cdns3_endpoint *priv_ep;
2145
2146         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2147                 unsigned long num;
2148                 int ret;
2149                 /* ep name pattern likes epXin or epXout */
2150                 char c[2] = {ep->name[2], '\0'};
2151
2152                 ret = kstrtoul(c, 10, &num);
2153                 if (ret)
2154                         return ERR_PTR(ret);
2155
2156                 priv_ep = ep_to_cdns3_ep(ep);
2157                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2158                         if (!(priv_ep->flags & EP_CLAIMED)) {
2159                                 priv_ep->num  = num;
2160                                 return priv_ep;
2161                         }
2162                 }
2163         }
2164
2165         return ERR_PTR(-ENOENT);
2166 }
2167
2168 /*
2169  *  Cadence IP has one limitation that all endpoints must be configured
2170  * (Type & MaxPacketSize) before setting configuration through hardware
2171  * register, it means we can't change endpoints configuration after
2172  * set_configuration.
2173  *
2174  * This function set EP_CLAIMED flag which is added when the gadget driver
2175  * uses usb_ep_autoconfig to configure specific endpoint;
2176  * When the udc driver receives set_configurion request,
2177  * it goes through all claimed endpoints, and configure all endpoints
2178  * accordingly.
2179  *
2180  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2181  * ep_cfg register which can be changed after set_configuration, and do
2182  * some software operation accordingly.
2183  */
2184 static struct
2185 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2186                               struct usb_endpoint_descriptor *desc,
2187                               struct usb_ss_ep_comp_descriptor *comp_desc)
2188 {
2189         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2190         struct cdns3_endpoint *priv_ep;
2191         unsigned long flags;
2192
2193         priv_ep = cdns3_find_available_ep(priv_dev, desc);
2194         if (IS_ERR(priv_ep)) {
2195                 dev_err(priv_dev->dev, "no available ep\n");
2196                 return NULL;
2197         }
2198
2199         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2200
2201         spin_lock_irqsave(&priv_dev->lock, flags);
2202         priv_ep->endpoint.desc = desc;
2203         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2204         priv_ep->type = usb_endpoint_type(desc);
2205         priv_ep->flags |= EP_CLAIMED;
2206         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2207
2208         spin_unlock_irqrestore(&priv_dev->lock, flags);
2209         return &priv_ep->endpoint;
2210 }
2211
2212 /**
2213  * cdns3_gadget_ep_alloc_request Allocates request
2214  * @ep: endpoint object associated with request
2215  * @gfp_flags: gfp flags
2216  *
2217  * Returns allocated request address, NULL on allocation error
2218  */
2219 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2220                                                   gfp_t gfp_flags)
2221 {
2222         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2223         struct cdns3_request *priv_req;
2224
2225         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2226         if (!priv_req)
2227                 return NULL;
2228
2229         priv_req->priv_ep = priv_ep;
2230
2231         trace_cdns3_alloc_request(priv_req);
2232         return &priv_req->request;
2233 }
2234
2235 /**
2236  * cdns3_gadget_ep_free_request Free memory occupied by request
2237  * @ep: endpoint object associated with request
2238  * @request: request to free memory
2239  */
2240 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2241                                   struct usb_request *request)
2242 {
2243         struct cdns3_request *priv_req = to_cdns3_request(request);
2244
2245         if (priv_req->aligned_buf)
2246                 priv_req->aligned_buf->in_use = 0;
2247
2248         trace_cdns3_free_request(priv_req);
2249         kfree(priv_req);
2250 }
2251
2252 /**
2253  * cdns3_gadget_ep_enable Enable endpoint
2254  * @ep: endpoint object
2255  * @desc: endpoint descriptor
2256  *
2257  * Returns 0 on success, error code elsewhere
2258  */
2259 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2260                                   const struct usb_endpoint_descriptor *desc)
2261 {
2262         struct cdns3_endpoint *priv_ep;
2263         struct cdns3_device *priv_dev;
2264         const struct usb_ss_ep_comp_descriptor *comp_desc;
2265         u32 reg = EP_STS_EN_TRBERREN;
2266         u32 bEndpointAddress;
2267         unsigned long flags;
2268         int enable = 1;
2269         int ret;
2270         int val;
2271
2272         priv_ep = ep_to_cdns3_ep(ep);
2273         priv_dev = priv_ep->cdns3_dev;
2274         comp_desc = priv_ep->endpoint.comp_desc;
2275
2276         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2277                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2278                 return -EINVAL;
2279         }
2280
2281         if (!desc->wMaxPacketSize) {
2282                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2283                 return -EINVAL;
2284         }
2285
2286         if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2287                           "%s is already enabled\n", priv_ep->name))
2288                 return 0;
2289
2290         spin_lock_irqsave(&priv_dev->lock, flags);
2291
2292         priv_ep->endpoint.desc = desc;
2293         priv_ep->type = usb_endpoint_type(desc);
2294         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2295
2296         if (priv_ep->interval > ISO_MAX_INTERVAL &&
2297             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2298                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2299                         ISO_MAX_INTERVAL);
2300
2301                 ret =  -EINVAL;
2302                 goto exit;
2303         }
2304
2305         bEndpointAddress = priv_ep->num | priv_ep->dir;
2306         cdns3_select_ep(priv_dev, bEndpointAddress);
2307
2308         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2309                 /*
2310                  * Enable stream support (SS mode) related interrupts
2311                  * in EP_STS_EN Register
2312                  */
2313                 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2314                         reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2315                                 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2316                                 EP_STS_EN_STREAMREN;
2317                         priv_ep->use_streams = true;
2318                         cdns3_stream_ep_reconfig(priv_dev, priv_ep);
2319                         priv_dev->using_streams |= true;
2320                 }
2321         }
2322
2323         ret = cdns3_allocate_trb_pool(priv_ep);
2324
2325         if (ret)
2326                 goto exit;
2327
2328         bEndpointAddress = priv_ep->num | priv_ep->dir;
2329         cdns3_select_ep(priv_dev, bEndpointAddress);
2330
2331         trace_cdns3_gadget_ep_enable(priv_ep);
2332
2333         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2334
2335         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2336                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2337                                         1, 1000);
2338
2339         if (unlikely(ret)) {
2340                 cdns3_free_trb_pool(priv_ep);
2341                 ret =  -EINVAL;
2342                 goto exit;
2343         }
2344
2345         /* enable interrupt for selected endpoint */
2346         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2347                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2348
2349         if (priv_dev->dev_ver < DEV_VER_V2)
2350                 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2351
2352         writel(reg, &priv_dev->regs->ep_sts_en);
2353
2354         /*
2355          * For some versions of controller at some point during ISO OUT traffic
2356          * DMA reads Transfer Ring for the EP which has never got doorbell.
2357          * This issue was detected only on simulation, but to avoid this issue
2358          * driver add protection against it. To fix it driver enable ISO OUT
2359          * endpoint before setting DRBL. This special treatment of ISO OUT
2360          * endpoints are recommended by controller specification.
2361          */
2362         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
2363                 enable = 0;
2364
2365         if (enable)
2366                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
2367
2368         ep->desc = desc;
2369         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2370                             EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2371         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2372         priv_ep->wa1_set = 0;
2373         priv_ep->enqueue = 0;
2374         priv_ep->dequeue = 0;
2375         reg = readl(&priv_dev->regs->ep_sts);
2376         priv_ep->pcs = !!EP_STS_CCS(reg);
2377         priv_ep->ccs = !!EP_STS_CCS(reg);
2378         /* one TRB is reserved for link TRB used in DMULT mode*/
2379         priv_ep->free_trbs = priv_ep->num_trbs - 1;
2380 exit:
2381         spin_unlock_irqrestore(&priv_dev->lock, flags);
2382
2383         return ret;
2384 }
2385
2386 /**
2387  * cdns3_gadget_ep_disable Disable endpoint
2388  * @ep: endpoint object
2389  *
2390  * Returns 0 on success, error code elsewhere
2391  */
2392 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2393 {
2394         struct cdns3_endpoint *priv_ep;
2395         struct cdns3_request *priv_req;
2396         struct cdns3_device *priv_dev;
2397         struct usb_request *request;
2398         unsigned long flags;
2399         int ret = 0;
2400         u32 ep_cfg;
2401         int val;
2402
2403         if (!ep) {
2404                 pr_err("usbss: invalid parameters\n");
2405                 return -EINVAL;
2406         }
2407
2408         priv_ep = ep_to_cdns3_ep(ep);
2409         priv_dev = priv_ep->cdns3_dev;
2410
2411         if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2412                           "%s is already disabled\n", priv_ep->name))
2413                 return 0;
2414
2415         spin_lock_irqsave(&priv_dev->lock, flags);
2416
2417         trace_cdns3_gadget_ep_disable(priv_ep);
2418
2419         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2420
2421         ep_cfg = readl(&priv_dev->regs->ep_cfg);
2422         ep_cfg &= ~EP_CFG_ENABLE;
2423         writel(ep_cfg, &priv_dev->regs->ep_cfg);
2424
2425         /**
2426          * Driver needs some time before resetting endpoint.
2427          * It need waits for clearing DBUSY bit or for timeout expired.
2428          * 10us is enough time for controller to stop transfer.
2429          */
2430         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2431                                   !(val & EP_STS_DBUSY), 1, 10);
2432         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2433
2434         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2435                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2436                                   1, 1000);
2437         if (unlikely(ret))
2438                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2439                         priv_ep->name);
2440
2441         while (!list_empty(&priv_ep->pending_req_list)) {
2442                 request = cdns3_next_request(&priv_ep->pending_req_list);
2443
2444                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2445                                       -ESHUTDOWN);
2446         }
2447
2448         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2449                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2450
2451                 kfree(priv_req->request.buf);
2452                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2453                                              &priv_req->request);
2454                 list_del_init(&priv_req->list);
2455                 --priv_ep->wa2_counter;
2456         }
2457
2458         while (!list_empty(&priv_ep->deferred_req_list)) {
2459                 request = cdns3_next_request(&priv_ep->deferred_req_list);
2460
2461                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2462                                       -ESHUTDOWN);
2463         }
2464
2465         priv_ep->descmis_req = NULL;
2466
2467         ep->desc = NULL;
2468         priv_ep->flags &= ~EP_ENABLED;
2469         priv_ep->use_streams = false;
2470
2471         spin_unlock_irqrestore(&priv_dev->lock, flags);
2472
2473         return ret;
2474 }
2475
2476 /**
2477  * cdns3_gadget_ep_queue Transfer data on endpoint
2478  * @ep: endpoint object
2479  * @request: request object
2480  * @gfp_flags: gfp flags
2481  *
2482  * Returns 0 on success, error code elsewhere
2483  */
2484 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2485                                    struct usb_request *request,
2486                                    gfp_t gfp_flags)
2487 {
2488         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2489         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2490         struct cdns3_request *priv_req;
2491         int ret = 0;
2492
2493         request->actual = 0;
2494         request->status = -EINPROGRESS;
2495         priv_req = to_cdns3_request(request);
2496         trace_cdns3_ep_queue(priv_req);
2497
2498         if (priv_dev->dev_ver < DEV_VER_V2) {
2499                 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2500                                                 priv_req);
2501
2502                 if (ret == EINPROGRESS)
2503                         return 0;
2504         }
2505
2506         ret = cdns3_prepare_aligned_request_buf(priv_req);
2507         if (ret < 0)
2508                 return ret;
2509
2510         ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2511                                             usb_endpoint_dir_in(ep->desc));
2512         if (ret)
2513                 return ret;
2514
2515         list_add_tail(&request->list, &priv_ep->deferred_req_list);
2516
2517         /*
2518          * For stream capable endpoint if prime irq flag is set then only start
2519          * request.
2520          * If hardware endpoint configuration has not been set yet then
2521          * just queue request in deferred list. Transfer will be started in
2522          * cdns3_set_hw_configuration.
2523          */
2524         if (!request->stream_id) {
2525                 if (priv_dev->hw_configured_flag &&
2526                     !(priv_ep->flags & EP_STALLED) &&
2527                     !(priv_ep->flags & EP_STALL_PENDING))
2528                         cdns3_start_all_request(priv_dev, priv_ep);
2529         } else {
2530                 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2531                         cdns3_start_all_request(priv_dev, priv_ep);
2532         }
2533
2534         return 0;
2535 }
2536
2537 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2538                                  gfp_t gfp_flags)
2539 {
2540         struct usb_request *zlp_request;
2541         struct cdns3_endpoint *priv_ep;
2542         struct cdns3_device *priv_dev;
2543         unsigned long flags;
2544         int ret;
2545
2546         if (!request || !ep)
2547                 return -EINVAL;
2548
2549         priv_ep = ep_to_cdns3_ep(ep);
2550         priv_dev = priv_ep->cdns3_dev;
2551
2552         spin_lock_irqsave(&priv_dev->lock, flags);
2553
2554         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2555
2556         if (ret == 0 && request->zero && request->length &&
2557             (request->length % ep->maxpacket == 0)) {
2558                 struct cdns3_request *priv_req;
2559
2560                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2561                 zlp_request->buf = priv_dev->zlp_buf;
2562                 zlp_request->length = 0;
2563
2564                 priv_req = to_cdns3_request(zlp_request);
2565                 priv_req->flags |= REQUEST_ZLP;
2566
2567                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2568                         priv_ep->name);
2569                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2570         }
2571
2572         spin_unlock_irqrestore(&priv_dev->lock, flags);
2573         return ret;
2574 }
2575
2576 /**
2577  * cdns3_gadget_ep_dequeue Remove request from transfer queue
2578  * @ep: endpoint object associated with request
2579  * @request: request object
2580  *
2581  * Returns 0 on success, error code elsewhere
2582  */
2583 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2584                             struct usb_request *request)
2585 {
2586         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2587         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2588         struct usb_request *req, *req_temp;
2589         struct cdns3_request *priv_req;
2590         struct cdns3_trb *link_trb;
2591         u8 req_on_hw_ring = 0;
2592         unsigned long flags;
2593         int ret = 0;
2594
2595         if (!ep || !request || !ep->desc)
2596                 return -EINVAL;
2597
2598         spin_lock_irqsave(&priv_dev->lock, flags);
2599
2600         priv_req = to_cdns3_request(request);
2601
2602         trace_cdns3_ep_dequeue(priv_req);
2603
2604         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2605
2606         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2607                                  list) {
2608                 if (request == req) {
2609                         req_on_hw_ring = 1;
2610                         goto found;
2611                 }
2612         }
2613
2614         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2615                                  list) {
2616                 if (request == req)
2617                         goto found;
2618         }
2619
2620         goto not_found;
2621
2622 found:
2623         link_trb = priv_req->trb;
2624
2625         /* Update ring only if removed request is on pending_req_list list */
2626         if (req_on_hw_ring && link_trb) {
2627                 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2628                         ((priv_req->end_trb + 1) * TRB_SIZE)));
2629                 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2630                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2631
2632                 if (priv_ep->wa1_trb == priv_req->trb)
2633                         cdns3_wa1_restore_cycle_bit(priv_ep);
2634         }
2635
2636         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2637
2638 not_found:
2639         spin_unlock_irqrestore(&priv_dev->lock, flags);
2640         return ret;
2641 }
2642
2643 /**
2644  * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2645  * Should be called after acquiring spin_lock and selecting ep
2646  * @priv_ep: endpoint object to set stall on.
2647  */
2648 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2649 {
2650         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2651
2652         trace_cdns3_halt(priv_ep, 1, 0);
2653
2654         if (!(priv_ep->flags & EP_STALLED)) {
2655                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2656
2657                 if (!(ep_sts_reg & EP_STS_DBUSY))
2658                         cdns3_ep_stall_flush(priv_ep);
2659                 else
2660                         priv_ep->flags |= EP_STALL_PENDING;
2661         }
2662 }
2663
2664 /**
2665  * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2666  * Should be called after acquiring spin_lock and selecting ep
2667  * @priv_ep: endpoint object to clear stall on
2668  */
2669 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2670 {
2671         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2672         struct usb_request *request;
2673         struct cdns3_request *priv_req;
2674         struct cdns3_trb *trb = NULL;
2675         int ret;
2676         int val;
2677
2678         trace_cdns3_halt(priv_ep, 0, 0);
2679
2680         request = cdns3_next_request(&priv_ep->pending_req_list);
2681         if (request) {
2682                 priv_req = to_cdns3_request(request);
2683                 trb = priv_req->trb;
2684                 if (trb)
2685                         trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2686         }
2687
2688         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2689
2690         /* wait for EPRST cleared */
2691         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2692                                         !(val & EP_CMD_EPRST), 1, 100);
2693         if (ret)
2694                 return -EINVAL;
2695
2696         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2697
2698         if (request) {
2699                 if (trb)
2700                         trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2701
2702                 cdns3_rearm_transfer(priv_ep, 1);
2703         }
2704
2705         cdns3_start_all_request(priv_dev, priv_ep);
2706         return ret;
2707 }
2708
2709 /**
2710  * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2711  * @ep: endpoint object to set/clear stall on
2712  * @value: 1 for set stall, 0 for clear stall
2713  *
2714  * Returns 0 on success, error code elsewhere
2715  */
2716 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2717 {
2718         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2719         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2720         unsigned long flags;
2721         int ret = 0;
2722
2723         if (!(priv_ep->flags & EP_ENABLED))
2724                 return -EPERM;
2725
2726         spin_lock_irqsave(&priv_dev->lock, flags);
2727
2728         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2729
2730         if (!value) {
2731                 priv_ep->flags &= ~EP_WEDGE;
2732                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2733         } else {
2734                 __cdns3_gadget_ep_set_halt(priv_ep);
2735         }
2736
2737         spin_unlock_irqrestore(&priv_dev->lock, flags);
2738
2739         return ret;
2740 }
2741
2742 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2743
2744 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2745         .enable = cdns3_gadget_ep_enable,
2746         .disable = cdns3_gadget_ep_disable,
2747         .alloc_request = cdns3_gadget_ep_alloc_request,
2748         .free_request = cdns3_gadget_ep_free_request,
2749         .queue = cdns3_gadget_ep_queue,
2750         .dequeue = cdns3_gadget_ep_dequeue,
2751         .set_halt = cdns3_gadget_ep_set_halt,
2752         .set_wedge = cdns3_gadget_ep_set_wedge,
2753 };
2754
2755 /**
2756  * cdns3_gadget_get_frame Returns number of actual ITP frame
2757  * @gadget: gadget object
2758  *
2759  * Returns number of actual ITP frame
2760  */
2761 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2762 {
2763         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2764
2765         return readl(&priv_dev->regs->usb_itpn);
2766 }
2767
2768 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2769 {
2770         enum usb_device_speed speed;
2771
2772         speed = cdns3_get_speed(priv_dev);
2773
2774         if (speed >= USB_SPEED_SUPER)
2775                 return 0;
2776
2777         /* Start driving resume signaling to indicate remote wakeup. */
2778         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2779
2780         return 0;
2781 }
2782
2783 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2784 {
2785         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2786         unsigned long flags;
2787         int ret = 0;
2788
2789         spin_lock_irqsave(&priv_dev->lock, flags);
2790         ret = __cdns3_gadget_wakeup(priv_dev);
2791         spin_unlock_irqrestore(&priv_dev->lock, flags);
2792         return ret;
2793 }
2794
2795 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2796                                         int is_selfpowered)
2797 {
2798         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2799         unsigned long flags;
2800
2801         spin_lock_irqsave(&priv_dev->lock, flags);
2802         priv_dev->is_selfpowered = !!is_selfpowered;
2803         spin_unlock_irqrestore(&priv_dev->lock, flags);
2804         return 0;
2805 }
2806
2807 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2808 {
2809         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2810
2811         if (is_on) {
2812                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2813         } else {
2814                 writel(~0, &priv_dev->regs->ep_ists);
2815                 writel(~0, &priv_dev->regs->usb_ists);
2816                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2817         }
2818
2819         return 0;
2820 }
2821
2822 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2823 {
2824         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2825         u32 reg;
2826
2827         cdns3_ep0_config(priv_dev);
2828
2829         /* enable interrupts for endpoint 0 (in and out) */
2830         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2831
2832         /*
2833          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2834          * revision of controller.
2835          */
2836         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2837                 reg = readl(&regs->dbg_link1);
2838
2839                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2840                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2841                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2842                 writel(reg, &regs->dbg_link1);
2843         }
2844
2845         /*
2846          * By default some platforms has set protected access to memory.
2847          * This cause problem with cache, so driver restore non-secure
2848          * access to memory.
2849          */
2850         reg = readl(&regs->dma_axi_ctrl);
2851         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2852                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2853         writel(reg, &regs->dma_axi_ctrl);
2854
2855         /* enable generic interrupt*/
2856         writel(USB_IEN_INIT, &regs->usb_ien);
2857         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2858         /*  keep Fast Access bit */
2859         writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2860
2861         cdns3_configure_dmult(priv_dev, NULL);
2862 }
2863
2864 /**
2865  * cdns3_gadget_udc_start Gadget start
2866  * @gadget: gadget object
2867  * @driver: driver which operates on this gadget
2868  *
2869  * Returns 0 on success, error code elsewhere
2870  */
2871 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2872                                   struct usb_gadget_driver *driver)
2873 {
2874         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2875         unsigned long flags;
2876         enum usb_device_speed max_speed = driver->max_speed;
2877
2878         spin_lock_irqsave(&priv_dev->lock, flags);
2879         priv_dev->gadget_driver = driver;
2880
2881         /* limit speed if necessary */
2882         max_speed = min(driver->max_speed, gadget->max_speed);
2883
2884         switch (max_speed) {
2885         case USB_SPEED_FULL:
2886                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2887                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2888                 break;
2889         case USB_SPEED_HIGH:
2890                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2891                 break;
2892         case USB_SPEED_SUPER:
2893                 break;
2894         default:
2895                 dev_err(priv_dev->dev,
2896                         "invalid maximum_speed parameter %d\n",
2897                         max_speed);
2898                 fallthrough;
2899         case USB_SPEED_UNKNOWN:
2900                 /* default to superspeed */
2901                 max_speed = USB_SPEED_SUPER;
2902                 break;
2903         }
2904
2905         cdns3_gadget_config(priv_dev);
2906         spin_unlock_irqrestore(&priv_dev->lock, flags);
2907         return 0;
2908 }
2909
2910 /**
2911  * cdns3_gadget_udc_stop Stops gadget
2912  * @gadget: gadget object
2913  *
2914  * Returns 0
2915  */
2916 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2917 {
2918         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2919         struct cdns3_endpoint *priv_ep;
2920         u32 bEndpointAddress;
2921         struct usb_ep *ep;
2922         int val;
2923
2924         priv_dev->gadget_driver = NULL;
2925
2926         priv_dev->onchip_used_size = 0;
2927         priv_dev->out_mem_is_allocated = 0;
2928         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2929
2930         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2931                 priv_ep = ep_to_cdns3_ep(ep);
2932                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2933                 cdns3_select_ep(priv_dev, bEndpointAddress);
2934                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2935                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2936                                           !(val & EP_CMD_EPRST), 1, 100);
2937
2938                 priv_ep->flags &= ~EP_CLAIMED;
2939         }
2940
2941         /* disable interrupt for device */
2942         writel(0, &priv_dev->regs->usb_ien);
2943         writel(0, &priv_dev->regs->usb_pwr);
2944         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2945
2946         return 0;
2947 }
2948
2949 static const struct usb_gadget_ops cdns3_gadget_ops = {
2950         .get_frame = cdns3_gadget_get_frame,
2951         .wakeup = cdns3_gadget_wakeup,
2952         .set_selfpowered = cdns3_gadget_set_selfpowered,
2953         .pullup = cdns3_gadget_pullup,
2954         .udc_start = cdns3_gadget_udc_start,
2955         .udc_stop = cdns3_gadget_udc_stop,
2956         .match_ep = cdns3_gadget_match_ep,
2957 };
2958
2959 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2960 {
2961         int i;
2962
2963         /* ep0 OUT point to ep0 IN. */
2964         priv_dev->eps[16] = NULL;
2965
2966         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2967                 if (priv_dev->eps[i]) {
2968                         cdns3_free_trb_pool(priv_dev->eps[i]);
2969                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2970                 }
2971 }
2972
2973 /**
2974  * cdns3_init_eps Initializes software endpoints of gadget
2975  * @priv_dev: extended gadget object
2976  *
2977  * Returns 0 on success, error code elsewhere
2978  */
2979 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2980 {
2981         u32 ep_enabled_reg, iso_ep_reg;
2982         struct cdns3_endpoint *priv_ep;
2983         int ep_dir, ep_number;
2984         u32 ep_mask;
2985         int ret = 0;
2986         int i;
2987
2988         /* Read it from USB_CAP3 to USB_CAP5 */
2989         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2990         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2991
2992         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2993
2994         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2995                 ep_dir = i >> 4;        /* i div 16 */
2996                 ep_number = i & 0xF;    /* i % 16 */
2997                 ep_mask = BIT(i);
2998
2999                 if (!(ep_enabled_reg & ep_mask))
3000                         continue;
3001
3002                 if (ep_dir && !ep_number) {
3003                         priv_dev->eps[i] = priv_dev->eps[0];
3004                         continue;
3005                 }
3006
3007                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3008                                        GFP_KERNEL);
3009                 if (!priv_ep)
3010                         goto err;
3011
3012                 /* set parent of endpoint object */
3013                 priv_ep->cdns3_dev = priv_dev;
3014                 priv_dev->eps[i] = priv_ep;
3015                 priv_ep->num = ep_number;
3016                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3017
3018                 if (!ep_number) {
3019                         ret = cdns3_init_ep0(priv_dev, priv_ep);
3020                         if (ret) {
3021                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
3022                                 goto err;
3023                         }
3024                 } else {
3025                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3026                                  ep_number, !!ep_dir ? "in" : "out");
3027                         priv_ep->endpoint.name = priv_ep->name;
3028
3029                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3030                                                    CDNS3_EP_MAX_PACKET_LIMIT);
3031                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3032                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3033                         if (ep_dir)
3034                                 priv_ep->endpoint.caps.dir_in = 1;
3035                         else
3036                                 priv_ep->endpoint.caps.dir_out = 1;
3037
3038                         if (iso_ep_reg & ep_mask)
3039                                 priv_ep->endpoint.caps.type_iso = 1;
3040
3041                         priv_ep->endpoint.caps.type_bulk = 1;
3042                         priv_ep->endpoint.caps.type_int = 1;
3043
3044                         list_add_tail(&priv_ep->endpoint.ep_list,
3045                                       &priv_dev->gadget.ep_list);
3046                 }
3047
3048                 priv_ep->flags = 0;
3049
3050                 dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
3051                          priv_ep->name,
3052                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3053                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3054
3055                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3056                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3057                 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3058         }
3059
3060         return 0;
3061 err:
3062         cdns3_free_all_eps(priv_dev);
3063         return -ENOMEM;
3064 }
3065
3066 static void cdns3_gadget_release(struct device *dev)
3067 {
3068         struct cdns3_device *priv_dev = container_of(dev,
3069                         struct cdns3_device, gadget.dev);
3070
3071         kfree(priv_dev);
3072 }
3073
3074 void cdns3_gadget_exit(struct cdns3 *cdns)
3075 {
3076         struct cdns3_device *priv_dev;
3077
3078         priv_dev = cdns->gadget_dev;
3079
3080
3081         pm_runtime_mark_last_busy(cdns->dev);
3082         pm_runtime_put_autosuspend(cdns->dev);
3083
3084         usb_del_gadget(&priv_dev->gadget);
3085         devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3086
3087         cdns3_free_all_eps(priv_dev);
3088
3089         while (!list_empty(&priv_dev->aligned_buf_list)) {
3090                 struct cdns3_aligned_buf *buf;
3091
3092                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3093                 dma_free_coherent(priv_dev->sysdev, buf->size,
3094                                   buf->buf,
3095                                   buf->dma);
3096
3097                 list_del(&buf->list);
3098                 kfree(buf);
3099         }
3100
3101         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3102                           priv_dev->setup_dma);
3103
3104         kfree(priv_dev->zlp_buf);
3105         usb_put_gadget(&priv_dev->gadget);
3106         cdns->gadget_dev = NULL;
3107         cdns3_drd_gadget_off(cdns);
3108 }
3109
3110 static int cdns3_gadget_start(struct cdns3 *cdns)
3111 {
3112         struct cdns3_device *priv_dev;
3113         u32 max_speed;
3114         int ret;
3115
3116         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3117         if (!priv_dev)
3118                 return -ENOMEM;
3119
3120         usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3121                         cdns3_gadget_release);
3122         cdns->gadget_dev = priv_dev;
3123         priv_dev->sysdev = cdns->dev;
3124         priv_dev->dev = cdns->dev;
3125         priv_dev->regs = cdns->dev_regs;
3126
3127         device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3128                                  &priv_dev->onchip_buffers);
3129
3130         if (priv_dev->onchip_buffers <=  0) {
3131                 u32 reg = readl(&priv_dev->regs->usb_cap2);
3132
3133                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3134         }
3135
3136         if (!priv_dev->onchip_buffers)
3137                 priv_dev->onchip_buffers = 256;
3138
3139         max_speed = usb_get_maximum_speed(cdns->dev);
3140
3141         /* Check the maximum_speed parameter */
3142         switch (max_speed) {
3143         case USB_SPEED_FULL:
3144         case USB_SPEED_HIGH:
3145         case USB_SPEED_SUPER:
3146                 break;
3147         default:
3148                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3149                         max_speed);
3150                 fallthrough;
3151         case USB_SPEED_UNKNOWN:
3152                 /* default to superspeed */
3153                 max_speed = USB_SPEED_SUPER;
3154                 break;
3155         }
3156
3157         /* fill gadget fields */
3158         priv_dev->gadget.max_speed = max_speed;
3159         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3160         priv_dev->gadget.ops = &cdns3_gadget_ops;
3161         priv_dev->gadget.name = "usb-ss-gadget";
3162         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3163         priv_dev->gadget.irq = cdns->dev_irq;
3164
3165         spin_lock_init(&priv_dev->lock);
3166         INIT_WORK(&priv_dev->pending_status_wq,
3167                   cdns3_pending_setup_status_handler);
3168
3169         INIT_WORK(&priv_dev->aligned_buf_wq,
3170                   cdns3_free_aligned_request_buf);
3171
3172         /* initialize endpoint container */
3173         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3174         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3175
3176         ret = cdns3_init_eps(priv_dev);
3177         if (ret) {
3178                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3179                 goto err1;
3180         }
3181
3182         /* allocate memory for setup packet buffer */
3183         priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3184                                                  &priv_dev->setup_dma, GFP_DMA);
3185         if (!priv_dev->setup_buf) {
3186                 ret = -ENOMEM;
3187                 goto err2;
3188         }
3189
3190         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3191
3192         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3193                 readl(&priv_dev->regs->usb_cap6));
3194         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3195                 readl(&priv_dev->regs->usb_cap1));
3196         dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3197                 readl(&priv_dev->regs->usb_cap2));
3198
3199         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3200         if (priv_dev->dev_ver >= DEV_VER_V2)
3201                 priv_dev->gadget.sg_supported = 1;
3202
3203         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3204         if (!priv_dev->zlp_buf) {
3205                 ret = -ENOMEM;
3206                 goto err3;
3207         }
3208
3209         /* add USB gadget device */
3210         ret = usb_add_gadget(&priv_dev->gadget);
3211         if (ret < 0) {
3212                 dev_err(priv_dev->dev, "Failed to add gadget\n");
3213                 goto err4;
3214         }
3215
3216         return 0;
3217 err4:
3218         kfree(priv_dev->zlp_buf);
3219 err3:
3220         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3221                           priv_dev->setup_dma);
3222 err2:
3223         cdns3_free_all_eps(priv_dev);
3224 err1:
3225         usb_put_gadget(&priv_dev->gadget);
3226         cdns->gadget_dev = NULL;
3227         return ret;
3228 }
3229
3230 static int __cdns3_gadget_init(struct cdns3 *cdns)
3231 {
3232         int ret = 0;
3233
3234         /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3235         ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3236         if (ret) {
3237                 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3238                 return ret;
3239         }
3240
3241         cdns3_drd_gadget_on(cdns);
3242         pm_runtime_get_sync(cdns->dev);
3243
3244         ret = cdns3_gadget_start(cdns);
3245         if (ret)
3246                 return ret;
3247
3248         /*
3249          * Because interrupt line can be shared with other components in
3250          * driver it can't use IRQF_ONESHOT flag here.
3251          */
3252         ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3253                                         cdns3_device_irq_handler,
3254                                         cdns3_device_thread_irq_handler,
3255                                         IRQF_SHARED, dev_name(cdns->dev),
3256                                         cdns->gadget_dev);
3257
3258         if (ret)
3259                 goto err0;
3260
3261         return 0;
3262 err0:
3263         cdns3_gadget_exit(cdns);
3264         return ret;
3265 }
3266
3267 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
3268 {
3269         struct cdns3_device *priv_dev = cdns->gadget_dev;
3270
3271         cdns3_disconnect_gadget(priv_dev);
3272
3273         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3274         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3275         cdns3_hw_reset_eps_config(priv_dev);
3276
3277         /* disable interrupt for device */
3278         writel(0, &priv_dev->regs->usb_ien);
3279
3280         return 0;
3281 }
3282
3283 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
3284 {
3285         struct cdns3_device *priv_dev = cdns->gadget_dev;
3286
3287         if (!priv_dev->gadget_driver)
3288                 return 0;
3289
3290         cdns3_gadget_config(priv_dev);
3291
3292         return 0;
3293 }
3294
3295 /**
3296  * cdns3_gadget_init - initialize device structure
3297  *
3298  * @cdns: cdns3 instance
3299  *
3300  * This function initializes the gadget.
3301  */
3302 int cdns3_gadget_init(struct cdns3 *cdns)
3303 {
3304         struct cdns3_role_driver *rdrv;
3305
3306         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3307         if (!rdrv)
3308                 return -ENOMEM;
3309
3310         rdrv->start     = __cdns3_gadget_init;
3311         rdrv->stop      = cdns3_gadget_exit;
3312         rdrv->suspend   = cdns3_gadget_suspend;
3313         rdrv->resume    = cdns3_gadget_resume;
3314         rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
3315         rdrv->name      = "gadget";
3316         cdns->roles[USB_ROLE_DEVICE] = rdrv;
3317
3318         return 0;
3319 }