1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBSS DRD Driver - gadget side.
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
15 * At some situations, the controller may get stale data address in TRB
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
29 * Issue has been fixed in DEV_VER_V3 version of controller.
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
55 * Issue has been fixed in DEV_VER_V2 version of controller.
59 #include <linux/dma-mapping.h>
60 #include <linux/usb/gadget.h>
61 #include <linux/module.h>
62 #include <linux/dmapool.h>
63 #include <linux/iopoll.h>
64 #include <linux/property.h>
67 #include "gadget-export.h"
68 #include "cdns3-gadget.h"
69 #include "cdns3-trace.h"
72 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
73 struct usb_request *request,
76 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
77 struct usb_request *request);
79 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
80 struct usb_request *request);
83 * cdns3_clear_register_bit - clear bit in given register.
84 * @ptr: address of device controller register to be read and changed
85 * @mask: bits requested to clar
87 static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
89 mask = readl(ptr) & ~mask;
94 * cdns3_set_register_bit - set bit in given register.
95 * @ptr: address of device controller register to be read and changed
96 * @mask: bits requested to set
98 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
100 mask = readl(ptr) | mask;
105 * cdns3_ep_addr_to_index - Macro converts endpoint address to
106 * index of endpoint object in cdns3_device.eps[] container
107 * @ep_addr: endpoint address for which endpoint object is required
110 u8 cdns3_ep_addr_to_index(u8 ep_addr)
112 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
115 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
116 struct cdns3_endpoint *priv_ep)
120 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
122 return dma_index / TRB_SIZE;
126 * cdns3_next_request - returns next request from list
127 * @list: list containing requests
129 * Returns request or NULL if no requests in list
131 struct usb_request *cdns3_next_request(struct list_head *list)
133 return list_first_entry_or_null(list, struct usb_request, list);
137 * cdns3_next_align_buf - returns next buffer from list
138 * @list: list containing buffers
140 * Returns buffer or NULL if no buffers in list
142 static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
144 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
148 * cdns3_next_priv_request - returns next request from list
149 * @list: list containing requests
151 * Returns request or NULL if no requests in list
153 static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
155 return list_first_entry_or_null(list, struct cdns3_request, list);
159 * cdns3_select_ep - selects endpoint
160 * @priv_dev: extended gadget object
161 * @ep: endpoint address
163 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
165 if (priv_dev->selected_ep == ep)
168 priv_dev->selected_ep = ep;
169 writel(ep, &priv_dev->regs->ep_sel);
173 * cdns3_get_tdl - gets current tdl for selected endpoint.
174 * @priv_dev: extended gadget object
176 * Before calling this function the appropriate endpoint must
177 * be selected by means of cdns3_select_ep function.
179 static int cdns3_get_tdl(struct cdns3_device *priv_dev)
181 if (priv_dev->dev_ver < DEV_VER_V3)
182 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
184 return readl(&priv_dev->regs->ep_tdl);
187 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
188 struct cdns3_trb *trb)
190 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
192 return priv_ep->trb_pool_dma + offset;
195 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
197 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
199 if (priv_ep->trb_pool) {
200 dma_pool_free(priv_dev->eps_dma_pool,
201 priv_ep->trb_pool, priv_ep->trb_pool_dma);
202 priv_ep->trb_pool = NULL;
207 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
208 * @priv_ep: endpoint object
210 * Function will return 0 on success or -ENOMEM on allocation error
212 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
214 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
215 int ring_size = TRB_RING_SIZE;
216 int num_trbs = ring_size / TRB_SIZE;
217 struct cdns3_trb *link_trb;
219 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
220 cdns3_free_trb_pool(priv_ep);
222 if (!priv_ep->trb_pool) {
223 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
225 &priv_ep->trb_pool_dma);
227 if (!priv_ep->trb_pool)
230 priv_ep->alloc_ring_size = ring_size;
233 memset(priv_ep->trb_pool, 0, ring_size);
235 priv_ep->num_trbs = num_trbs;
240 /* Initialize the last TRB as Link TRB */
241 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
243 if (priv_ep->use_streams) {
245 * For stream capable endpoints driver use single correct TRB.
246 * The last trb has zeroed cycle bit
248 link_trb->control = 0;
250 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
251 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
257 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
258 * @priv_ep: endpoint object
260 * Endpoint must be selected before call to this function
262 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
264 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
267 trace_cdns3_halt(priv_ep, 1, 1);
269 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
270 &priv_dev->regs->ep_cmd);
272 /* wait for DFLUSH cleared */
273 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
274 !(val & EP_CMD_DFLUSH), 1, 1000);
275 priv_ep->flags |= EP_STALLED;
276 priv_ep->flags &= ~EP_STALL_PENDING;
280 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
281 * @priv_dev: extended gadget object
283 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
287 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
289 cdns3_allow_enable_l1(priv_dev, 0);
290 priv_dev->hw_configured_flag = 0;
291 priv_dev->onchip_used_size = 0;
292 priv_dev->out_mem_is_allocated = 0;
293 priv_dev->wait_for_setup = 0;
294 priv_dev->using_streams = 0;
296 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
297 if (priv_dev->eps[i])
298 priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
302 * cdns3_ep_inc_trb - increment a trb index.
303 * @index: Pointer to the TRB index to increment.
305 * @trb_in_seg: number of TRBs in segment
307 * The index should never point to the link TRB. After incrementing,
308 * if it is point to the link TRB, wrap around to the beginning and revert
309 * cycle state bit The
310 * link TRB is always at the last TRB entry.
312 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
315 if (*index == (trb_in_seg - 1)) {
322 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
323 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
325 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
327 priv_ep->free_trbs--;
328 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
332 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
333 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
335 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
337 priv_ep->free_trbs++;
338 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
342 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
343 * @priv_dev: Extended gadget object
344 * @enable: Enable/disable permit to transition to L1.
346 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
347 * then controller answer with ACK handshake.
348 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
349 * then controller answer with NYET handshake.
351 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
354 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
356 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
359 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
363 reg = readl(&priv_dev->regs->usb_sts);
365 if (DEV_SUPERSPEED(reg))
366 return USB_SPEED_SUPER;
367 else if (DEV_HIGHSPEED(reg))
368 return USB_SPEED_HIGH;
369 else if (DEV_FULLSPEED(reg))
370 return USB_SPEED_FULL;
371 else if (DEV_LOWSPEED(reg))
372 return USB_SPEED_LOW;
373 return USB_SPEED_UNKNOWN;
377 * cdns3_start_all_request - add to ring all request not started
378 * @priv_dev: Extended gadget object
379 * @priv_ep: The endpoint for whom request will be started.
381 * Returns return ENOMEM if transfer ring i not enough TRBs to start
384 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
385 struct cdns3_endpoint *priv_ep)
387 struct usb_request *request;
389 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
392 * If the last pending transfer is INTERNAL
393 * OR streams are enabled for this endpoint
394 * do NOT start new transfer till the last one is pending
396 if (!pending_empty) {
397 struct cdns3_request *priv_req;
399 request = cdns3_next_request(&priv_ep->pending_req_list);
400 priv_req = to_cdns3_request(request);
401 if ((priv_req->flags & REQUEST_INTERNAL) ||
402 (priv_ep->flags & EP_TDLCHK_EN) ||
403 priv_ep->use_streams) {
404 dev_dbg(priv_dev->dev, "Blocking external request\n");
409 while (!list_empty(&priv_ep->deferred_req_list)) {
410 request = cdns3_next_request(&priv_ep->deferred_req_list);
412 if (!priv_ep->use_streams) {
413 ret = cdns3_ep_run_transfer(priv_ep, request);
415 priv_ep->stream_sg_idx = 0;
416 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
421 list_move_tail(&request->list, &priv_ep->pending_req_list);
422 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
426 priv_ep->flags &= ~EP_RING_FULL;
431 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
432 * driver try to detect whether endpoint need additional internal
433 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
434 * if before first DESCMISS interrupt the DMA will be armed.
436 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
437 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
438 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
439 (reg) |= EP_STS_EN_DESCMISEN; \
442 static void __cdns3_descmiss_copy_data(struct usb_request *request,
443 struct usb_request *descmiss_req)
445 int length = request->actual + descmiss_req->actual;
446 struct scatterlist *s = request->sg;
449 if (length <= request->length) {
450 memcpy(&((u8 *)request->buf)[request->actual],
452 descmiss_req->actual);
453 request->actual = length;
455 /* It should never occures */
456 request->status = -ENOMEM;
459 if (length <= sg_dma_len(s)) {
460 void *p = phys_to_virt(sg_dma_address(s));
462 memcpy(&((u8 *)p)[request->actual],
464 descmiss_req->actual);
465 request->actual = length;
467 request->status = -ENOMEM;
473 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
474 * request queued by class driver.
475 * @priv_ep: extended endpoint object
476 * @request: request object
478 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
479 struct usb_request *request)
481 struct usb_request *descmiss_req;
482 struct cdns3_request *descmiss_priv_req;
484 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
488 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
489 descmiss_req = &descmiss_priv_req->request;
491 /* driver can't touch pending request */
492 if (descmiss_priv_req->flags & REQUEST_PENDING)
495 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
496 request->status = descmiss_req->status;
497 __cdns3_descmiss_copy_data(request, descmiss_req);
498 list_del_init(&descmiss_priv_req->list);
499 kfree(descmiss_req->buf);
500 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
501 --priv_ep->wa2_counter;
508 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
509 struct cdns3_endpoint *priv_ep,
510 struct cdns3_request *priv_req)
512 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
513 priv_req->flags & REQUEST_INTERNAL) {
514 struct usb_request *req;
516 req = cdns3_next_request(&priv_ep->deferred_req_list);
518 priv_ep->descmis_req = NULL;
523 /* unmap the gadget request before copying data */
524 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
527 cdns3_wa2_descmiss_copy_data(priv_ep, req);
528 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
529 req->length != req->actual) {
530 /* wait for next part of transfer */
531 /* re-map the gadget request buffer*/
532 usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
533 usb_endpoint_dir_in(priv_ep->endpoint.desc));
537 if (req->status == -EINPROGRESS)
540 list_del_init(&req->list);
541 cdns3_start_all_request(priv_dev, priv_ep);
545 return &priv_req->request;
548 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
549 struct cdns3_endpoint *priv_ep,
550 struct cdns3_request *priv_req)
555 * If transfer was queued before DESCMISS appear than we
556 * can disable handling of DESCMISS interrupt. Driver assumes that it
557 * can disable special treatment for this endpoint.
559 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
562 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
563 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
564 reg = readl(&priv_dev->regs->ep_sts_en);
565 reg &= ~EP_STS_EN_DESCMISEN;
566 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
567 writel(reg, &priv_dev->regs->ep_sts_en);
570 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
571 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
572 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
575 * DESCMISS transfer has been finished, so data will be
576 * directly copied from internal allocated usb_request
579 if (pending_empty && !descmiss_empty &&
580 !(priv_req->flags & REQUEST_INTERNAL)) {
581 cdns3_wa2_descmiss_copy_data(priv_ep,
584 trace_cdns3_wa2(priv_ep, "get internal stored data");
586 list_add_tail(&priv_req->request.list,
587 &priv_ep->pending_req_list);
588 cdns3_gadget_giveback(priv_ep, priv_req,
589 priv_req->request.status);
592 * Intentionally driver returns positive value as
593 * correct value. It informs that transfer has
600 * Driver will wait for completion DESCMISS transfer,
601 * before starts new, not DESCMISS transfer.
603 if (!pending_empty && !descmiss_empty) {
604 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
608 if (priv_req->flags & REQUEST_INTERNAL)
609 list_add_tail(&priv_req->list,
610 &priv_ep->wa2_descmiss_req_list);
616 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
618 struct cdns3_request *priv_req;
620 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
623 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
624 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
626 trace_cdns3_wa2(priv_ep, "removes eldest request");
628 kfree(priv_req->request.buf);
629 list_del_init(&priv_req->list);
630 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
632 --priv_ep->wa2_counter;
640 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
641 * @priv_ep: extended gadget object
643 * This function is used only for WA2. For more information see Work around 2
646 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
648 struct cdns3_request *priv_req;
649 struct usb_request *request;
650 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
652 /* check for pending transfer */
653 if (!pending_empty) {
654 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
658 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
659 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
660 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
663 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
665 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
666 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
667 cdns3_wa2_remove_old_request(priv_ep);
670 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
675 priv_req = to_cdns3_request(request);
676 priv_req->flags |= REQUEST_INTERNAL;
678 /* if this field is still assigned it indicate that transfer related
679 * with this request has not been finished yet. Driver in this
680 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
681 * flag to previous one. It will indicate that current request is
682 * part of the previous one.
684 if (priv_ep->descmis_req)
685 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
687 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
689 priv_ep->wa2_counter++;
691 if (!priv_req->request.buf) {
692 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
696 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
697 priv_ep->descmis_req = priv_req;
699 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
700 &priv_ep->descmis_req->request,
706 dev_err(priv_ep->cdns3_dev->dev,
707 "Failed: No sufficient memory for DESCMIS\n");
710 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
712 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
715 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
717 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
718 &priv_dev->regs->ep_cmd);
722 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
727 cdns3_select_ep(priv_dev, 0);
729 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
731 if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
732 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
733 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
735 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
736 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
737 u8 pending_empty = list_empty(&outq_ep->pending_req_list);
739 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
740 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
746 cdns3_select_ep(priv_dev, outq_ep->num |
748 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
749 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
751 outq_ep->flags |= EP_TDLCHK_EN;
752 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
755 cdns3_wa2_enable_detection(priv_dev, outq_ep,
757 writel(ep_sts_en_reg,
758 &priv_dev->regs->ep_sts_en);
759 /* reset tdl value to zero */
760 cdns3_wa2_reset_tdl(priv_dev);
762 * Memory barrier - Reset tdl before ringing the
766 if (EP_CMD_DRDY & ep_cmd_reg) {
767 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
770 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
772 * ring doorbell to generate DESCMIS irq
775 &priv_dev->regs->ep_cmd);
783 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
784 * @priv_ep: The endpoint to whom the request belongs to
785 * @priv_req: The request we're giving back
786 * @status: completion code for the request
788 * Must be called with controller's lock held and interrupts disabled. This
789 * function will unmap @req and call its ->complete() callback to notify upper
790 * layers that it has completed.
792 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
793 struct cdns3_request *priv_req,
796 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
797 struct usb_request *request = &priv_req->request;
799 list_del_init(&request->list);
801 if (request->status == -EINPROGRESS)
802 request->status = status;
804 if (likely(!(priv_req->flags & REQUEST_UNALIGNED)))
805 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
808 if ((priv_req->flags & REQUEST_UNALIGNED) &&
809 priv_ep->dir == USB_DIR_OUT && !request->status) {
810 /* Make DMA buffer CPU accessible */
811 dma_sync_single_for_cpu(priv_dev->sysdev,
812 priv_req->aligned_buf->dma,
814 priv_req->aligned_buf->dir);
815 memcpy(request->buf, priv_req->aligned_buf->buf,
819 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
820 /* All TRBs have finished, clear the counter */
821 priv_req->finished_trb = 0;
822 trace_cdns3_gadget_giveback(priv_req);
824 if (priv_dev->dev_ver < DEV_VER_V2) {
825 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
831 if (request->complete) {
832 spin_unlock(&priv_dev->lock);
833 usb_gadget_giveback_request(&priv_ep->endpoint,
835 spin_lock(&priv_dev->lock);
838 if (request->buf == priv_dev->zlp_buf)
839 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
842 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
844 /* Work around for stale data address in TRB*/
845 if (priv_ep->wa1_set) {
846 trace_cdns3_wa1(priv_ep, "restore cycle bit");
848 priv_ep->wa1_set = 0;
849 priv_ep->wa1_trb_index = 0xFFFF;
850 if (priv_ep->wa1_cycle_bit) {
851 priv_ep->wa1_trb->control =
852 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
854 priv_ep->wa1_trb->control =
855 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
860 static void cdns3_free_aligned_request_buf(struct work_struct *work)
862 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
864 struct cdns3_aligned_buf *buf, *tmp;
867 spin_lock_irqsave(&priv_dev->lock, flags);
869 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
871 list_del(&buf->list);
874 * Re-enable interrupts to free DMA capable memory.
875 * Driver can't free this memory with disabled
878 spin_unlock_irqrestore(&priv_dev->lock, flags);
879 dma_free_noncoherent(priv_dev->sysdev, buf->size,
880 buf->buf, buf->dma, buf->dir);
882 spin_lock_irqsave(&priv_dev->lock, flags);
886 spin_unlock_irqrestore(&priv_dev->lock, flags);
889 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
891 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
892 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
893 struct cdns3_aligned_buf *buf;
895 /* check if buffer is aligned to 8. */
896 if (!((uintptr_t)priv_req->request.buf & 0x7))
899 buf = priv_req->aligned_buf;
901 if (!buf || priv_req->request.length > buf->size) {
902 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
906 buf->size = priv_req->request.length;
907 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
908 DMA_TO_DEVICE : DMA_FROM_DEVICE;
910 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
920 if (priv_req->aligned_buf) {
921 trace_cdns3_free_aligned_request(priv_req);
922 priv_req->aligned_buf->in_use = 0;
923 queue_work(system_freezable_wq,
924 &priv_dev->aligned_buf_wq);
928 priv_req->aligned_buf = buf;
930 list_add_tail(&buf->list,
931 &priv_dev->aligned_buf_list);
934 if (priv_ep->dir == USB_DIR_IN) {
935 /* Make DMA buffer CPU accessible */
936 dma_sync_single_for_cpu(priv_dev->sysdev,
937 buf->dma, buf->size, buf->dir);
938 memcpy(buf->buf, priv_req->request.buf,
939 priv_req->request.length);
942 /* Transfer DMA buffer ownership back to device */
943 dma_sync_single_for_device(priv_dev->sysdev,
944 buf->dma, buf->size, buf->dir);
946 priv_req->flags |= REQUEST_UNALIGNED;
947 trace_cdns3_prepare_aligned_request(priv_req);
952 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
953 struct cdns3_trb *trb)
955 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
957 if (!priv_ep->wa1_set) {
960 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
963 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
964 priv_ep->wa1_set = 1;
965 priv_ep->wa1_trb = trb;
966 priv_ep->wa1_trb_index = priv_ep->enqueue;
967 trace_cdns3_wa1(priv_ep, "set guard");
974 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
975 struct cdns3_endpoint *priv_ep)
980 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
981 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
983 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
984 cdns3_wa1_restore_cycle_bit(priv_ep);
987 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
988 struct usb_request *request)
990 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
991 struct cdns3_request *priv_req;
992 struct cdns3_trb *trb;
998 unsigned int sg_idx = priv_ep->stream_sg_idx;
1000 priv_req = to_cdns3_request(request);
1001 address = priv_ep->endpoint.desc->bEndpointAddress;
1003 priv_ep->flags |= EP_PENDING_REQUEST;
1005 /* must allocate buffer aligned to 8 */
1006 if (priv_req->flags & REQUEST_UNALIGNED)
1007 trb_dma = priv_req->aligned_buf->dma;
1009 trb_dma = request->dma;
1011 /* For stream capable endpoints driver use only single TD. */
1012 trb = priv_ep->trb_pool + priv_ep->enqueue;
1013 priv_req->start_trb = priv_ep->enqueue;
1014 priv_req->end_trb = priv_req->start_trb;
1015 priv_req->trb = trb;
1017 cdns3_select_ep(priv_ep->cdns3_dev, address);
1019 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1020 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1022 if (!request->num_sgs) {
1023 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1024 length = request->length;
1026 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1027 length = request->sg[sg_idx].length;
1030 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1032 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1035 * For DEV_VER_V2 controller version we have enabled
1036 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1037 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1039 if (priv_dev->dev_ver >= DEV_VER_V2) {
1040 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1041 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1043 priv_req->flags |= REQUEST_PENDING;
1045 trb->control = cpu_to_le32(control);
1047 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1050 * Memory barrier - Cycle Bit must be set before trb->length and
1051 * trb->buffer fields.
1055 /* always first element */
1056 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1057 &priv_dev->regs->ep_traddr);
1059 if (!(priv_ep->flags & EP_STALLED)) {
1060 trace_cdns3_ring(priv_ep);
1061 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1062 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1064 priv_ep->prime_flag = false;
1067 * Controller version DEV_VER_V2 tdl calculation
1071 if (priv_dev->dev_ver < DEV_VER_V2)
1072 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1073 &priv_dev->regs->ep_cmd);
1074 else if (priv_dev->dev_ver > DEV_VER_V2)
1075 writel(tdl, &priv_dev->regs->ep_tdl);
1077 priv_ep->last_stream_id = priv_req->request.stream_id;
1078 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1079 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1080 EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1082 trace_cdns3_doorbell_epx(priv_ep->name,
1083 readl(&priv_dev->regs->ep_traddr));
1086 /* WORKAROUND for transition to L0 */
1087 __cdns3_gadget_wakeup(priv_dev);
1092 static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1094 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1096 if (priv_dev->dev_ver < DEV_VER_V3)
1099 if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {
1100 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);
1101 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1106 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1107 * @priv_ep: endpoint object
1108 * @request: request object
1110 * Returns zero on success or negative value on failure
1112 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1113 struct usb_request *request)
1115 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1116 struct cdns3_request *priv_req;
1117 struct cdns3_trb *trb;
1118 struct cdns3_trb *link_trb = NULL;
1129 struct scatterlist *s = NULL;
1130 bool sg_supported = !!(request->num_mapped_sgs);
1131 u32 ioc = request->no_interrupt ? 0 : TRB_IOC;
1133 num_trb_req = sg_supported ? request->num_mapped_sgs : 1;
1135 /* ISO transfer require each SOF have a TD, each TD include some TRBs */
1136 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1137 num_trb = priv_ep->interval * num_trb_req;
1139 num_trb = num_trb_req;
1141 priv_req = to_cdns3_request(request);
1142 address = priv_ep->endpoint.desc->bEndpointAddress;
1144 priv_ep->flags |= EP_PENDING_REQUEST;
1146 /* must allocate buffer aligned to 8 */
1147 if (priv_req->flags & REQUEST_UNALIGNED)
1148 trb_dma = priv_req->aligned_buf->dma;
1150 trb_dma = request->dma;
1152 trb = priv_ep->trb_pool + priv_ep->enqueue;
1153 priv_req->start_trb = priv_ep->enqueue;
1154 priv_req->trb = trb;
1156 cdns3_select_ep(priv_ep->cdns3_dev, address);
1159 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1160 int doorbell, dma_index;
1163 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1164 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1166 /* Driver can't update LINK TRB if it is current processed. */
1167 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1168 priv_ep->flags |= EP_DEFERRED_DRDY;
1172 /*updating C bt in Link TRB before starting DMA*/
1173 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1175 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1176 * that DMA stuck at the LINK TRB.
1177 * On the other hand, removing TRB_CHAIN for longer TRs for
1178 * epXout cause that DMA stuck after handling LINK TRB.
1179 * To eliminate this strange behavioral driver set TRB_CHAIN
1180 * bit only for TR size > 2.
1182 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1183 TRBS_PER_SEGMENT > 2)
1186 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1187 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1189 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1191 * ISO require LINK TRB must be first one of TD.
1192 * Fill LINK TRBs for left trb space to simply software process logic.
1194 while (priv_ep->enqueue) {
1196 trace_cdns3_prepare_trb(priv_ep, trb);
1198 cdns3_ep_inc_enq(priv_ep);
1199 trb = priv_ep->trb_pool + priv_ep->enqueue;
1200 priv_req->trb = trb;
1205 if (num_trb > priv_ep->free_trbs) {
1206 priv_ep->flags |= EP_RING_FULL;
1210 if (priv_dev->dev_ver <= DEV_VER_V2)
1211 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1213 /* set incorrect Cycle Bit for first trb*/
1214 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1216 if (priv_dev->dev_ver >= DEV_VER_V2) {
1219 td_size = DIV_ROUND_UP(request->length,
1220 priv_ep->endpoint.maxpacket);
1221 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1222 trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1224 control |= TRB_TDL_HS_SIZE(td_size);
1230 if (!(sg_iter % num_trb_req) && sg_supported)
1234 control |= TRB_TYPE(TRB_NORMAL);
1236 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1237 length = sg_dma_len(s);
1239 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1240 length = request->length;
1243 if (priv_ep->flags & EP_TDLCHK_EN)
1244 total_tdl += DIV_ROUND_UP(length,
1245 priv_ep->endpoint.maxpacket);
1247 trb_burst = priv_ep->trb_burst_size;
1250 * Supposed DMA cross 4k bounder problem should be fixed at DEV_VER_V2, but still
1251 * met problem when do ISO transfer if sg enabled.
1253 * Data pattern likes below when sg enabled, package size is 1k and mult is 2
1254 * [UVC Header(8B) ] [data(3k - 8)] ...
1256 * The received data at offset 0xd000 will get 0xc000 data, len 0x70. Error happen
1266 * But it is still unclear about why error have not happen below 0xd000, it should
1267 * cross 4k bounder. But anyway, the below code can fix this problem.
1269 * To avoid DMA cross 4k bounder at ISO transfer, reduce burst len according to 16.
1271 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2)
1272 if (ALIGN_DOWN(trb->buffer, SZ_4K) !=
1273 ALIGN_DOWN(trb->buffer + length, SZ_4K))
1276 trb->length |= cpu_to_le32(TRB_BURST_LEN(trb_burst) |
1278 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1281 * first trb should be prepared as last to avoid processing
1287 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1288 control |= ioc | TRB_ISP;
1290 /* for last element in TD or in SG list */
1291 if (sg_iter == (num_trb - 1) && sg_iter != 0)
1292 control |= pcs | ioc | TRB_ISP;
1296 trb->control = cpu_to_le32(control);
1298 priv_req->trb->control = cpu_to_le32(control);
1301 trb->control |= cpu_to_le32(TRB_ISP);
1302 /* Don't set chain bit for last TRB */
1303 if ((sg_iter % num_trb_req) < num_trb_req - 1)
1304 trb->control |= cpu_to_le32(TRB_CHAIN);
1311 priv_req->end_trb = priv_ep->enqueue;
1312 cdns3_ep_inc_enq(priv_ep);
1313 trb = priv_ep->trb_pool + priv_ep->enqueue;
1315 } while (sg_iter < num_trb);
1317 trb = priv_req->trb;
1319 priv_req->flags |= REQUEST_PENDING;
1320 priv_req->num_of_trb = num_trb;
1323 trb->control |= cpu_to_le32(ioc | TRB_ISP);
1325 if (priv_dev->dev_ver < DEV_VER_V2 &&
1326 (priv_ep->flags & EP_TDLCHK_EN)) {
1327 u16 tdl = total_tdl;
1328 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1330 if (tdl > EP_CMD_TDL_MAX) {
1331 tdl = EP_CMD_TDL_MAX;
1332 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1335 if (old_tdl < tdl) {
1337 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1338 &priv_dev->regs->ep_cmd);
1343 * Memory barrier - cycle bit must be set before other filds in trb.
1347 /* give the TD to the consumer*/
1349 trb->control = trb->control ^ cpu_to_le32(1);
1351 if (priv_dev->dev_ver <= DEV_VER_V2)
1352 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1357 while (i < num_trb) {
1358 trace_cdns3_prepare_trb(priv_ep, trb + i);
1359 if (trb + i == link_trb) {
1360 trb = priv_ep->trb_pool;
1361 num_trb = num_trb - i;
1368 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1372 * Memory barrier - Cycle Bit must be set before trb->length and
1373 * trb->buffer fields.
1378 * For DMULT mode we can set address to transfer ring only once after
1379 * enabling endpoint.
1381 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1383 * Until SW is not ready to handle the OUT transfer the ISO OUT
1384 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1385 * EP_CFG_ENABLE must be set before updating ep_traddr.
1387 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1388 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1389 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1390 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1394 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1395 priv_req->start_trb * TRB_SIZE),
1396 &priv_dev->regs->ep_traddr);
1398 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1401 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1402 trace_cdns3_ring(priv_ep);
1403 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1404 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1405 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1406 cdns3_rearm_drdy_if_needed(priv_ep);
1407 trace_cdns3_doorbell_epx(priv_ep->name,
1408 readl(&priv_dev->regs->ep_traddr));
1411 /* WORKAROUND for transition to L0 */
1412 __cdns3_gadget_wakeup(priv_dev);
1417 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1419 struct cdns3_endpoint *priv_ep;
1422 if (priv_dev->hw_configured_flag)
1425 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1427 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1428 USB_CONF_U1EN | USB_CONF_U2EN);
1430 priv_dev->hw_configured_flag = 1;
1432 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1434 priv_ep = ep_to_cdns3_ep(ep);
1435 cdns3_start_all_request(priv_dev, priv_ep);
1439 cdns3_allow_enable_l1(priv_dev, 1);
1443 * cdns3_trb_handled - check whether trb has been handled by DMA
1445 * @priv_ep: extended endpoint object.
1446 * @priv_req: request object for checking
1448 * Endpoint must be selected before invoking this function.
1450 * Returns false if request has not been handled by DMA, else returns true.
1454 * DQ = priv_ep->dequeue - dequeue position
1455 * EQ = priv_ep->enqueue - enqueue position
1456 * ST = priv_req->start_trb - index of first TRB in transfer ring
1457 * ET = priv_req->end_trb - index of last TRB in transfer ring
1458 * CI = current_index - index of processed TRB by DMA.
1460 * As first step, we check if the TRB between the ST and ET.
1461 * Then, we check if cycle bit for index priv_ep->dequeue
1465 * 1. priv_ep->dequeue never equals to current_index.
1466 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1467 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1468 * and priv_ep->free_trbs is zero.
1469 * This case indicate that TR is full.
1471 * At below two cases, the request have been handled.
1472 * Case 1 - priv_ep->dequeue < current_index
1473 * SR ... EQ ... DQ ... CI ... ER
1474 * SR ... DQ ... CI ... EQ ... ER
1476 * Case 2 - priv_ep->dequeue > current_index
1477 * This situation takes place when CI go through the LINK TRB at the end of
1479 * SR ... CI ... EQ ... DQ ... ER
1481 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1482 struct cdns3_request *priv_req)
1484 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1485 struct cdns3_trb *trb;
1486 int current_index = 0;
1490 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1491 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1493 /* current trb doesn't belong to this request */
1494 if (priv_req->start_trb < priv_req->end_trb) {
1495 if (priv_ep->dequeue > priv_req->end_trb)
1498 if (priv_ep->dequeue < priv_req->start_trb)
1502 if ((priv_req->start_trb > priv_req->end_trb) &&
1503 (priv_ep->dequeue > priv_req->end_trb) &&
1504 (priv_ep->dequeue < priv_req->start_trb))
1507 if ((priv_req->start_trb == priv_req->end_trb) &&
1508 (priv_ep->dequeue != priv_req->end_trb))
1511 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1513 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1516 if (doorbell == 1 && current_index == priv_ep->dequeue)
1519 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1520 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1525 if (priv_ep->enqueue == priv_ep->dequeue &&
1526 priv_ep->free_trbs == 0) {
1528 } else if (priv_ep->dequeue < current_index) {
1529 if ((current_index == (priv_ep->num_trbs - 1)) &&
1534 } else if (priv_ep->dequeue > current_index) {
1539 trace_cdns3_request_handled(priv_req, current_index, handled);
1544 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1545 struct cdns3_endpoint *priv_ep)
1547 struct cdns3_request *priv_req;
1548 struct usb_request *request;
1549 struct cdns3_trb *trb;
1550 bool request_handled = false;
1551 bool transfer_end = false;
1553 while (!list_empty(&priv_ep->pending_req_list)) {
1554 request = cdns3_next_request(&priv_ep->pending_req_list);
1555 priv_req = to_cdns3_request(request);
1557 trb = priv_ep->trb_pool + priv_ep->dequeue;
1559 /* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
1560 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1562 /* ISO ep_traddr may stop at LINK TRB */
1563 if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) &&
1564 priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1567 trace_cdns3_complete_trb(priv_ep, trb);
1568 cdns3_ep_inc_deq(priv_ep);
1569 trb = priv_ep->trb_pool + priv_ep->dequeue;
1572 if (!request->stream_id) {
1573 /* Re-select endpoint. It could be changed by other CPU
1574 * during handling usb_gadget_giveback_request.
1576 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1578 while (cdns3_trb_handled(priv_ep, priv_req)) {
1579 priv_req->finished_trb++;
1580 if (priv_req->finished_trb >= priv_req->num_of_trb)
1581 request_handled = true;
1583 trb = priv_ep->trb_pool + priv_ep->dequeue;
1584 trace_cdns3_complete_trb(priv_ep, trb);
1588 TRB_LEN(le32_to_cpu(trb->length));
1590 if (priv_req->num_of_trb > 1 &&
1591 le32_to_cpu(trb->control) & TRB_SMM &&
1592 le32_to_cpu(trb->control) & TRB_CHAIN)
1593 transfer_end = true;
1595 cdns3_ep_inc_deq(priv_ep);
1598 if (request_handled) {
1599 /* TRBs are duplicated by priv_ep->interval time for ISO IN */
1600 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir)
1601 request->actual /= priv_ep->interval;
1603 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1604 request_handled = false;
1605 transfer_end = false;
1607 goto prepare_next_td;
1610 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1611 TRBS_PER_SEGMENT == 2)
1614 /* Re-select endpoint. It could be changed by other CPU
1615 * during handling usb_gadget_giveback_request.
1617 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1619 trb = priv_ep->trb_pool;
1620 trace_cdns3_complete_trb(priv_ep, trb);
1622 if (trb != priv_req->trb)
1623 dev_warn(priv_dev->dev,
1624 "request_trb=0x%p, queue_trb=0x%p\n",
1625 priv_req->trb, trb);
1627 request->actual += TRB_LEN(le32_to_cpu(trb->length));
1629 if (!request->num_sgs ||
1630 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1631 priv_ep->stream_sg_idx = 0;
1632 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1634 priv_ep->stream_sg_idx++;
1635 cdns3_ep_run_stream_transfer(priv_ep, request);
1640 priv_ep->flags &= ~EP_PENDING_REQUEST;
1643 if (!(priv_ep->flags & EP_STALLED) &&
1644 !(priv_ep->flags & EP_STALL_PENDING))
1645 cdns3_start_all_request(priv_dev, priv_ep);
1648 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1650 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1652 cdns3_wa1_restore_cycle_bit(priv_ep);
1655 trace_cdns3_ring(priv_ep);
1657 /* Cycle Bit must be updated before arming DMA. */
1659 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1661 __cdns3_gadget_wakeup(priv_dev);
1663 trace_cdns3_doorbell_epx(priv_ep->name,
1664 readl(&priv_dev->regs->ep_traddr));
1668 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1670 u16 tdl = priv_ep->pending_tdl;
1671 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1673 if (tdl > EP_CMD_TDL_MAX) {
1674 tdl = EP_CMD_TDL_MAX;
1675 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1677 priv_ep->pending_tdl = 0;
1680 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1684 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1685 * @priv_ep: endpoint object
1689 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1691 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1693 struct usb_request *deferred_request;
1694 struct usb_request *pending_request;
1697 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1699 trace_cdns3_epx_irq(priv_dev, priv_ep);
1701 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1702 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1704 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1705 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1707 tdl = cdns3_get_tdl(priv_dev);
1710 * Continue the previous transfer:
1711 * There is some racing between ERDY and PRIME. The device send
1712 * ERDY and almost in the same time Host send PRIME. It cause
1713 * that host ignore the ERDY packet and driver has to send it
1716 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1717 EP_STS_HOSTPP(ep_sts_reg))) {
1718 writel(EP_CMD_ERDY |
1719 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1720 &priv_dev->regs->ep_cmd);
1721 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1723 priv_ep->prime_flag = true;
1725 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1726 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1728 if (deferred_request && !pending_request) {
1729 cdns3_start_all_request(priv_dev, priv_ep);
1734 if (ep_sts_reg & EP_STS_TRBERR) {
1735 if (priv_ep->flags & EP_STALL_PENDING &&
1736 !(ep_sts_reg & EP_STS_DESCMIS &&
1737 priv_dev->dev_ver < DEV_VER_V2)) {
1738 cdns3_ep_stall_flush(priv_ep);
1742 * For isochronous transfer driver completes request on
1743 * IOC or on TRBERR. IOC appears only when device receive
1744 * OUT data packet. If host disable stream or lost some packet
1745 * then the only way to finish all queued transfer is to do it
1748 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1749 !priv_ep->wa1_set) {
1750 if (!priv_ep->dir) {
1751 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1753 ep_cfg &= ~EP_CFG_ENABLE;
1754 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1755 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1756 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1758 cdns3_transfer_completed(priv_dev, priv_ep);
1759 } else if (!(priv_ep->flags & EP_STALLED) &&
1760 !(priv_ep->flags & EP_STALL_PENDING)) {
1761 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1762 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1763 cdns3_start_all_request(priv_dev, priv_ep);
1765 cdns3_rearm_transfer(priv_ep,
1771 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1772 (ep_sts_reg & EP_STS_IOT)) {
1773 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1774 if (ep_sts_reg & EP_STS_ISP)
1775 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1777 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1780 if (!priv_ep->use_streams) {
1781 if ((ep_sts_reg & EP_STS_IOC) ||
1782 (ep_sts_reg & EP_STS_ISP)) {
1783 cdns3_transfer_completed(priv_dev, priv_ep);
1784 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1785 priv_ep->pending_tdl) {
1786 /* handle IOT with pending tdl */
1787 cdns3_reprogram_tdl(priv_ep);
1789 } else if (priv_ep->dir == USB_DIR_OUT) {
1790 priv_ep->ep_sts_pending |= ep_sts_reg;
1791 } else if (ep_sts_reg & EP_STS_IOT) {
1792 cdns3_transfer_completed(priv_dev, priv_ep);
1797 * MD_EXIT interrupt sets when stream capable endpoint exits
1798 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1800 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1801 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1802 priv_ep->ep_sts_pending = 0;
1803 cdns3_transfer_completed(priv_dev, priv_ep);
1807 * WA2: this condition should only be meet when
1808 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1809 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1810 * In other cases this interrupt will be disabled.
1812 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1813 !(priv_ep->flags & EP_STALLED))
1814 cdns3_wa2_descmissing_packet(priv_ep);
1819 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1821 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1822 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1826 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1827 * @priv_dev: extended gadget object
1828 * @usb_ists: bitmap representation of device's reported interrupts
1829 * (usb_ists register value)
1831 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1833 __must_hold(&priv_dev->lock)
1837 trace_cdns3_usb_irq(priv_dev, usb_ists);
1838 if (usb_ists & USB_ISTS_L1ENTI) {
1840 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1841 * from L1. To fix it, if any DMA transfer is pending driver
1842 * must starts driving resume signal immediately.
1844 if (readl(&priv_dev->regs->drbl))
1845 __cdns3_gadget_wakeup(priv_dev);
1848 /* Connection detected */
1849 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1850 speed = cdns3_get_speed(priv_dev);
1851 priv_dev->gadget.speed = speed;
1852 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1853 cdns3_ep0_config(priv_dev);
1856 /* Disconnection detected */
1857 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1858 spin_unlock(&priv_dev->lock);
1859 cdns3_disconnect_gadget(priv_dev);
1860 spin_lock(&priv_dev->lock);
1861 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1862 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1863 cdns3_hw_reset_eps_config(priv_dev);
1866 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1867 if (priv_dev->gadget_driver &&
1868 priv_dev->gadget_driver->suspend) {
1869 spin_unlock(&priv_dev->lock);
1870 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1871 spin_lock(&priv_dev->lock);
1875 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1876 if (priv_dev->gadget_driver &&
1877 priv_dev->gadget_driver->resume) {
1878 spin_unlock(&priv_dev->lock);
1879 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1880 spin_lock(&priv_dev->lock);
1885 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1886 if (priv_dev->gadget_driver) {
1887 spin_unlock(&priv_dev->lock);
1888 usb_gadget_udc_reset(&priv_dev->gadget,
1889 priv_dev->gadget_driver);
1890 spin_lock(&priv_dev->lock);
1892 /*read again to check the actual speed*/
1893 speed = cdns3_get_speed(priv_dev);
1894 priv_dev->gadget.speed = speed;
1895 cdns3_hw_reset_eps_config(priv_dev);
1896 cdns3_ep0_config(priv_dev);
1902 * cdns3_device_irq_handler - interrupt handler for device part of controller
1904 * @irq: irq number for cdns3 core device
1905 * @data: structure of cdns3
1907 * Returns IRQ_HANDLED or IRQ_NONE
1909 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1911 struct cdns3_device *priv_dev = data;
1912 struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1913 irqreturn_t ret = IRQ_NONE;
1919 /* check USB device interrupt */
1920 reg = readl(&priv_dev->regs->usb_ists);
1922 /* After masking interrupts the new interrupts won't be
1923 * reported in usb_ists/ep_ists. In order to not lose some
1924 * of them driver disables only detected interrupts.
1925 * They will be enabled ASAP after clearing source of
1926 * interrupt. This an unusual behavior only applies to
1927 * usb_ists register.
1929 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1930 /* mask deferred interrupt. */
1931 writel(reg, &priv_dev->regs->usb_ien);
1932 ret = IRQ_WAKE_THREAD;
1935 /* check endpoint interrupt */
1936 reg = readl(&priv_dev->regs->ep_ists);
1938 writel(0, &priv_dev->regs->ep_ien);
1939 ret = IRQ_WAKE_THREAD;
1946 * cdns3_device_thread_irq_handler - interrupt handler for device part
1949 * @irq: irq number for cdns3 core device
1950 * @data: structure of cdns3
1952 * Returns IRQ_HANDLED or IRQ_NONE
1954 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1956 struct cdns3_device *priv_dev = data;
1957 irqreturn_t ret = IRQ_NONE;
1958 unsigned long flags;
1962 spin_lock_irqsave(&priv_dev->lock, flags);
1964 reg = readl(&priv_dev->regs->usb_ists);
1966 writel(reg, &priv_dev->regs->usb_ists);
1967 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1968 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1972 reg = readl(&priv_dev->regs->ep_ists);
1974 /* handle default endpoint OUT */
1975 if (reg & EP_ISTS_EP_OUT0) {
1976 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1980 /* handle default endpoint IN */
1981 if (reg & EP_ISTS_EP_IN0) {
1982 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1986 /* check if interrupt from non default endpoint, if no exit */
1987 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1991 for_each_set_bit(bit, ®,
1992 sizeof(u32) * BITS_PER_BYTE) {
1993 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1997 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1998 cdns3_wa2_check_outq_status(priv_dev);
2001 writel(~0, &priv_dev->regs->ep_ien);
2002 spin_unlock_irqrestore(&priv_dev->lock, flags);
2008 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
2010 * The real reservation will occur during write to EP_CFG register,
2011 * this function is used to check if the 'size' reservation is allowed.
2013 * @priv_dev: extended gadget object
2014 * @size: the size (KB) for EP would like to allocate
2015 * @is_in: endpoint direction
2017 * Return 0 if the required size can met or negative value on failure
2019 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
2020 int size, int is_in)
2024 /* 2KB are reserved for EP0*/
2025 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
2028 if (remained < size)
2031 priv_dev->onchip_used_size += size;
2036 * ALL OUT EPs are shared the same chunk onchip memory, so
2037 * driver checks if it already has assigned enough buffers
2039 if (priv_dev->out_mem_is_allocated >= size)
2042 required = size - priv_dev->out_mem_is_allocated;
2044 if (required > remained)
2047 priv_dev->out_mem_is_allocated += required;
2048 priv_dev->onchip_used_size += required;
2054 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
2055 struct cdns3_endpoint *priv_ep)
2057 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2059 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
2060 if (priv_dev->dev_ver <= DEV_VER_V2)
2061 writel(USB_CONF_DMULT, ®s->usb_conf);
2063 if (priv_dev->dev_ver == DEV_VER_V2)
2064 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
2066 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2070 mask = BIT(priv_ep->num + 16);
2072 mask = BIT(priv_ep->num);
2074 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
2075 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2076 cdns3_set_register_bit(®s->tdl_beh, mask);
2077 cdns3_set_register_bit(®s->tdl_beh2, mask);
2078 cdns3_set_register_bit(®s->dma_adv_td, mask);
2081 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2082 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2084 cdns3_set_register_bit(®s->dtrans, mask);
2089 * cdns3_ep_config - Configure hardware endpoint
2090 * @priv_ep: extended endpoint object
2091 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2093 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2095 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2096 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2097 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2098 u32 max_packet_size = priv_ep->wMaxPacketSize;
2099 u8 maxburst = priv_ep->bMaxBurst;
2104 buffering = priv_dev->ep_buf_size - 1;
2106 cdns3_configure_dmult(priv_dev, priv_ep);
2108 switch (priv_ep->type) {
2109 case USB_ENDPOINT_XFER_INT:
2110 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2112 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2113 ep_cfg |= EP_CFG_TDL_CHK;
2115 case USB_ENDPOINT_XFER_BULK:
2116 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2118 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2119 ep_cfg |= EP_CFG_TDL_CHK;
2122 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2123 buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1;
2126 switch (priv_dev->gadget.speed) {
2127 case USB_SPEED_FULL:
2128 max_packet_size = is_iso_ep ? 1023 : 64;
2130 case USB_SPEED_HIGH:
2131 max_packet_size = is_iso_ep ? 1024 : 512;
2133 case USB_SPEED_SUPER:
2134 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2135 max_packet_size = 1024;
2136 maxburst = priv_dev->ep_buf_size - 1;
2140 /* all other speed are not supported */
2144 if (max_packet_size == 1024)
2145 priv_ep->trb_burst_size = 128;
2146 else if (max_packet_size >= 512)
2147 priv_ep->trb_burst_size = 64;
2149 priv_ep->trb_burst_size = 16;
2152 * In versions preceding DEV_VER_V2, for example, iMX8QM, there exit the bugs
2153 * in the DMA. These bugs occur when the trb_burst_size exceeds 16 and the
2154 * address is not aligned to 128 Bytes (which is a product of the 64-bit AXI
2155 * and AXI maximum burst length of 16 or 0xF+1, dma_axi_ctrl0[3:0]). This
2156 * results in data corruption when it crosses the 4K border. The corruption
2157 * specifically occurs from the position (4K - (address & 0x7F)) to 4K.
2159 * So force trb_burst_size to 16 at such platform.
2161 if (priv_dev->dev_ver < DEV_VER_V2)
2162 priv_ep->trb_burst_size = 16;
2164 buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX);
2165 maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX);
2167 /* onchip buffer is only allocated before configuration */
2168 if (!priv_dev->hw_configured_flag) {
2169 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2172 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2178 ep_cfg |= EP_CFG_ENABLE;
2180 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2181 if (priv_dev->dev_ver >= DEV_VER_V3) {
2182 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2185 * Stream capable endpoints are handled by using ep_tdl
2186 * register. Other endpoints use TDL from TRB feature.
2188 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2192 /* Enable Stream Bit TDL chk and SID chk */
2193 ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2196 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2197 EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */
2198 EP_CFG_BUFFERING(buffering) |
2199 EP_CFG_MAXBURST(maxburst);
2201 cdns3_select_ep(priv_dev, bEndpointAddress);
2202 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2203 priv_ep->flags |= EP_CONFIGURED;
2205 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2206 priv_ep->name, ep_cfg);
2211 /* Find correct direction for HW endpoint according to description */
2212 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2213 struct cdns3_endpoint *priv_ep)
2215 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2216 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2220 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2221 struct usb_endpoint_descriptor *desc)
2224 struct cdns3_endpoint *priv_ep;
2226 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2229 /* ep name pattern likes epXin or epXout */
2230 char c[2] = {ep->name[2], '\0'};
2232 ret = kstrtoul(c, 10, &num);
2234 return ERR_PTR(ret);
2236 priv_ep = ep_to_cdns3_ep(ep);
2237 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2238 if (!(priv_ep->flags & EP_CLAIMED)) {
2245 return ERR_PTR(-ENOENT);
2249 * Cadence IP has one limitation that all endpoints must be configured
2250 * (Type & MaxPacketSize) before setting configuration through hardware
2251 * register, it means we can't change endpoints configuration after
2252 * set_configuration.
2254 * This function set EP_CLAIMED flag which is added when the gadget driver
2255 * uses usb_ep_autoconfig to configure specific endpoint;
2256 * When the udc driver receives set_configurion request,
2257 * it goes through all claimed endpoints, and configure all endpoints
2260 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2261 * ep_cfg register which can be changed after set_configuration, and do
2262 * some software operation accordingly.
2265 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2266 struct usb_endpoint_descriptor *desc,
2267 struct usb_ss_ep_comp_descriptor *comp_desc)
2269 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2270 struct cdns3_endpoint *priv_ep;
2271 unsigned long flags;
2273 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2274 if (IS_ERR(priv_ep)) {
2275 dev_err(priv_dev->dev, "no available ep\n");
2279 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2281 spin_lock_irqsave(&priv_dev->lock, flags);
2282 priv_ep->endpoint.desc = desc;
2283 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2284 priv_ep->type = usb_endpoint_type(desc);
2285 priv_ep->flags |= EP_CLAIMED;
2286 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2287 priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc);
2288 priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize);
2289 priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK;
2290 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) {
2291 priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1;
2292 priv_ep->bMaxBurst = comp_desc->bMaxBurst;
2295 spin_unlock_irqrestore(&priv_dev->lock, flags);
2296 return &priv_ep->endpoint;
2300 * cdns3_gadget_ep_alloc_request - Allocates request
2301 * @ep: endpoint object associated with request
2302 * @gfp_flags: gfp flags
2304 * Returns allocated request address, NULL on allocation error
2306 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2309 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2310 struct cdns3_request *priv_req;
2312 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2316 priv_req->priv_ep = priv_ep;
2318 trace_cdns3_alloc_request(priv_req);
2319 return &priv_req->request;
2323 * cdns3_gadget_ep_free_request - Free memory occupied by request
2324 * @ep: endpoint object associated with request
2325 * @request: request to free memory
2327 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2328 struct usb_request *request)
2330 struct cdns3_request *priv_req = to_cdns3_request(request);
2332 if (priv_req->aligned_buf)
2333 priv_req->aligned_buf->in_use = 0;
2335 trace_cdns3_free_request(priv_req);
2340 * cdns3_gadget_ep_enable - Enable endpoint
2341 * @ep: endpoint object
2342 * @desc: endpoint descriptor
2344 * Returns 0 on success, error code elsewhere
2346 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2347 const struct usb_endpoint_descriptor *desc)
2349 struct cdns3_endpoint *priv_ep;
2350 struct cdns3_device *priv_dev;
2351 const struct usb_ss_ep_comp_descriptor *comp_desc;
2352 u32 reg = EP_STS_EN_TRBERREN;
2353 u32 bEndpointAddress;
2354 unsigned long flags;
2360 pr_debug("usbss: ep not configured?\n");
2364 priv_ep = ep_to_cdns3_ep(ep);
2365 priv_dev = priv_ep->cdns3_dev;
2366 comp_desc = priv_ep->endpoint.comp_desc;
2368 if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2369 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2373 if (!desc->wMaxPacketSize) {
2374 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2378 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2379 "%s is already enabled\n", priv_ep->name))
2382 spin_lock_irqsave(&priv_dev->lock, flags);
2384 priv_ep->endpoint.desc = desc;
2385 priv_ep->type = usb_endpoint_type(desc);
2386 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2388 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2389 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2390 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2397 bEndpointAddress = priv_ep->num | priv_ep->dir;
2398 cdns3_select_ep(priv_dev, bEndpointAddress);
2401 * For some versions of controller at some point during ISO OUT traffic
2402 * DMA reads Transfer Ring for the EP which has never got doorbell.
2403 * This issue was detected only on simulation, but to avoid this issue
2404 * driver add protection against it. To fix it driver enable ISO OUT
2405 * endpoint before setting DRBL. This special treatment of ISO OUT
2406 * endpoints are recommended by controller specification.
2408 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2411 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2413 * Enable stream support (SS mode) related interrupts
2414 * in EP_STS_EN Register
2416 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2417 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2418 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2419 EP_STS_EN_STREAMREN;
2420 priv_ep->use_streams = true;
2421 ret = cdns3_ep_config(priv_ep, enable);
2422 priv_dev->using_streams |= true;
2425 ret = cdns3_ep_config(priv_ep, enable);
2431 ret = cdns3_allocate_trb_pool(priv_ep);
2435 bEndpointAddress = priv_ep->num | priv_ep->dir;
2436 cdns3_select_ep(priv_dev, bEndpointAddress);
2438 trace_cdns3_gadget_ep_enable(priv_ep);
2440 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2442 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2443 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2446 if (unlikely(ret)) {
2447 cdns3_free_trb_pool(priv_ep);
2452 /* enable interrupt for selected endpoint */
2453 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2454 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2456 if (priv_dev->dev_ver < DEV_VER_V2)
2457 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2459 writel(reg, &priv_dev->regs->ep_sts_en);
2462 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2463 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2464 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2465 priv_ep->wa1_set = 0;
2466 priv_ep->enqueue = 0;
2467 priv_ep->dequeue = 0;
2468 reg = readl(&priv_dev->regs->ep_sts);
2469 priv_ep->pcs = !!EP_STS_CCS(reg);
2470 priv_ep->ccs = !!EP_STS_CCS(reg);
2471 /* one TRB is reserved for link TRB used in DMULT mode*/
2472 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2474 spin_unlock_irqrestore(&priv_dev->lock, flags);
2480 * cdns3_gadget_ep_disable - Disable endpoint
2481 * @ep: endpoint object
2483 * Returns 0 on success, error code elsewhere
2485 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2487 struct cdns3_endpoint *priv_ep;
2488 struct cdns3_request *priv_req;
2489 struct cdns3_device *priv_dev;
2490 struct usb_request *request;
2491 unsigned long flags;
2497 pr_err("usbss: invalid parameters\n");
2501 priv_ep = ep_to_cdns3_ep(ep);
2502 priv_dev = priv_ep->cdns3_dev;
2504 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2505 "%s is already disabled\n", priv_ep->name))
2508 spin_lock_irqsave(&priv_dev->lock, flags);
2510 trace_cdns3_gadget_ep_disable(priv_ep);
2512 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2514 ep_cfg = readl(&priv_dev->regs->ep_cfg);
2515 ep_cfg &= ~EP_CFG_ENABLE;
2516 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2519 * Driver needs some time before resetting endpoint.
2520 * It need waits for clearing DBUSY bit or for timeout expired.
2521 * 10us is enough time for controller to stop transfer.
2523 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2524 !(val & EP_STS_DBUSY), 1, 10);
2525 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2527 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2528 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2531 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2534 while (!list_empty(&priv_ep->pending_req_list)) {
2535 request = cdns3_next_request(&priv_ep->pending_req_list);
2537 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2541 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2542 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2544 kfree(priv_req->request.buf);
2545 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2546 &priv_req->request);
2547 list_del_init(&priv_req->list);
2548 --priv_ep->wa2_counter;
2551 while (!list_empty(&priv_ep->deferred_req_list)) {
2552 request = cdns3_next_request(&priv_ep->deferred_req_list);
2554 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2558 priv_ep->descmis_req = NULL;
2561 priv_ep->flags &= ~EP_ENABLED;
2562 priv_ep->use_streams = false;
2564 spin_unlock_irqrestore(&priv_dev->lock, flags);
2570 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2571 * @ep: endpoint object
2572 * @request: request object
2573 * @gfp_flags: gfp flags
2575 * Returns 0 on success, error code elsewhere
2577 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2578 struct usb_request *request,
2581 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2582 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2583 struct cdns3_request *priv_req;
2586 request->actual = 0;
2587 request->status = -EINPROGRESS;
2588 priv_req = to_cdns3_request(request);
2589 trace_cdns3_ep_queue(priv_req);
2591 if (priv_dev->dev_ver < DEV_VER_V2) {
2592 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2595 if (ret == EINPROGRESS)
2599 ret = cdns3_prepare_aligned_request_buf(priv_req);
2603 if (likely(!(priv_req->flags & REQUEST_UNALIGNED))) {
2604 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2605 usb_endpoint_dir_in(ep->desc));
2610 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2613 * For stream capable endpoint if prime irq flag is set then only start
2615 * If hardware endpoint configuration has not been set yet then
2616 * just queue request in deferred list. Transfer will be started in
2617 * cdns3_set_hw_configuration.
2619 if (!request->stream_id) {
2620 if (priv_dev->hw_configured_flag &&
2621 !(priv_ep->flags & EP_STALLED) &&
2622 !(priv_ep->flags & EP_STALL_PENDING))
2623 cdns3_start_all_request(priv_dev, priv_ep);
2625 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2626 cdns3_start_all_request(priv_dev, priv_ep);
2632 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2635 struct usb_request *zlp_request;
2636 struct cdns3_endpoint *priv_ep;
2637 struct cdns3_device *priv_dev;
2638 unsigned long flags;
2641 if (!request || !ep)
2644 priv_ep = ep_to_cdns3_ep(ep);
2645 priv_dev = priv_ep->cdns3_dev;
2647 spin_lock_irqsave(&priv_dev->lock, flags);
2649 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2651 if (ret == 0 && request->zero && request->length &&
2652 (request->length % ep->maxpacket == 0)) {
2653 struct cdns3_request *priv_req;
2655 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2656 zlp_request->buf = priv_dev->zlp_buf;
2657 zlp_request->length = 0;
2659 priv_req = to_cdns3_request(zlp_request);
2660 priv_req->flags |= REQUEST_ZLP;
2662 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2664 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2667 spin_unlock_irqrestore(&priv_dev->lock, flags);
2672 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2673 * @ep: endpoint object associated with request
2674 * @request: request object
2676 * Returns 0 on success, error code elsewhere
2678 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2679 struct usb_request *request)
2681 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2682 struct cdns3_device *priv_dev;
2683 struct usb_request *req, *req_temp;
2684 struct cdns3_request *priv_req;
2685 struct cdns3_trb *link_trb;
2686 u8 req_on_hw_ring = 0;
2687 unsigned long flags;
2691 if (!ep || !request || !ep->desc)
2694 priv_dev = priv_ep->cdns3_dev;
2696 spin_lock_irqsave(&priv_dev->lock, flags);
2698 priv_req = to_cdns3_request(request);
2700 trace_cdns3_ep_dequeue(priv_req);
2702 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2704 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2706 if (request == req) {
2712 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2721 link_trb = priv_req->trb;
2723 /* Update ring only if removed request is on pending_req_list list */
2724 if (req_on_hw_ring && link_trb) {
2726 writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
2728 /* wait for DFLUSH cleared */
2729 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2730 !(val & EP_CMD_DFLUSH), 1, 1000);
2732 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2733 ((priv_req->end_trb + 1) * TRB_SIZE)));
2734 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2735 TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2737 if (priv_ep->wa1_trb == priv_req->trb)
2738 cdns3_wa1_restore_cycle_bit(priv_ep);
2741 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2743 req = cdns3_next_request(&priv_ep->pending_req_list);
2745 cdns3_rearm_transfer(priv_ep, 1);
2748 spin_unlock_irqrestore(&priv_dev->lock, flags);
2753 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2754 * Should be called after acquiring spin_lock and selecting ep
2755 * @priv_ep: endpoint object to set stall on.
2757 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2759 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2761 trace_cdns3_halt(priv_ep, 1, 0);
2763 if (!(priv_ep->flags & EP_STALLED)) {
2764 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2766 if (!(ep_sts_reg & EP_STS_DBUSY))
2767 cdns3_ep_stall_flush(priv_ep);
2769 priv_ep->flags |= EP_STALL_PENDING;
2774 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2775 * Should be called after acquiring spin_lock and selecting ep
2776 * @priv_ep: endpoint object to clear stall on
2778 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2780 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2781 struct usb_request *request;
2782 struct cdns3_request *priv_req;
2783 struct cdns3_trb *trb = NULL;
2784 struct cdns3_trb trb_tmp;
2788 trace_cdns3_halt(priv_ep, 0, 0);
2790 request = cdns3_next_request(&priv_ep->pending_req_list);
2792 priv_req = to_cdns3_request(request);
2793 trb = priv_req->trb;
2796 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2800 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2802 /* wait for EPRST cleared */
2803 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2804 !(val & EP_CMD_EPRST), 1, 100);
2808 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2814 cdns3_rearm_transfer(priv_ep, 1);
2817 cdns3_start_all_request(priv_dev, priv_ep);
2822 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2823 * @ep: endpoint object to set/clear stall on
2824 * @value: 1 for set stall, 0 for clear stall
2826 * Returns 0 on success, error code elsewhere
2828 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2830 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2831 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2832 unsigned long flags;
2835 if (!(priv_ep->flags & EP_ENABLED))
2838 spin_lock_irqsave(&priv_dev->lock, flags);
2840 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2843 priv_ep->flags &= ~EP_WEDGE;
2844 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2846 __cdns3_gadget_ep_set_halt(priv_ep);
2849 spin_unlock_irqrestore(&priv_dev->lock, flags);
2854 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2856 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2857 .enable = cdns3_gadget_ep_enable,
2858 .disable = cdns3_gadget_ep_disable,
2859 .alloc_request = cdns3_gadget_ep_alloc_request,
2860 .free_request = cdns3_gadget_ep_free_request,
2861 .queue = cdns3_gadget_ep_queue,
2862 .dequeue = cdns3_gadget_ep_dequeue,
2863 .set_halt = cdns3_gadget_ep_set_halt,
2864 .set_wedge = cdns3_gadget_ep_set_wedge,
2868 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2869 * @gadget: gadget object
2871 * Returns number of actual ITP frame
2873 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2875 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2877 return readl(&priv_dev->regs->usb_itpn);
2880 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2882 enum usb_device_speed speed;
2884 speed = cdns3_get_speed(priv_dev);
2886 if (speed >= USB_SPEED_SUPER)
2889 /* Start driving resume signaling to indicate remote wakeup. */
2890 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2895 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2897 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2898 unsigned long flags;
2901 spin_lock_irqsave(&priv_dev->lock, flags);
2902 ret = __cdns3_gadget_wakeup(priv_dev);
2903 spin_unlock_irqrestore(&priv_dev->lock, flags);
2907 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2910 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2911 unsigned long flags;
2913 spin_lock_irqsave(&priv_dev->lock, flags);
2914 priv_dev->is_selfpowered = !!is_selfpowered;
2915 spin_unlock_irqrestore(&priv_dev->lock, flags);
2919 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2921 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2924 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2926 writel(~0, &priv_dev->regs->ep_ists);
2927 writel(~0, &priv_dev->regs->usb_ists);
2928 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2934 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2936 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2939 cdns3_ep0_config(priv_dev);
2941 /* enable interrupts for endpoint 0 (in and out) */
2942 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2945 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2946 * revision of controller.
2948 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2949 reg = readl(®s->dbg_link1);
2951 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2952 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2953 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2954 writel(reg, ®s->dbg_link1);
2958 * By default some platforms has set protected access to memory.
2959 * This cause problem with cache, so driver restore non-secure
2962 reg = readl(®s->dma_axi_ctrl);
2963 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2964 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2965 writel(reg, ®s->dma_axi_ctrl);
2967 /* enable generic interrupt*/
2968 writel(USB_IEN_INIT, ®s->usb_ien);
2969 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2970 /* keep Fast Access bit */
2971 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2973 cdns3_configure_dmult(priv_dev, NULL);
2977 * cdns3_gadget_udc_start - Gadget start
2978 * @gadget: gadget object
2979 * @driver: driver which operates on this gadget
2981 * Returns 0 on success, error code elsewhere
2983 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2984 struct usb_gadget_driver *driver)
2986 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2987 unsigned long flags;
2988 enum usb_device_speed max_speed = driver->max_speed;
2990 spin_lock_irqsave(&priv_dev->lock, flags);
2991 priv_dev->gadget_driver = driver;
2993 /* limit speed if necessary */
2994 max_speed = min(driver->max_speed, gadget->max_speed);
2996 switch (max_speed) {
2997 case USB_SPEED_FULL:
2998 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2999 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
3001 case USB_SPEED_HIGH:
3002 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
3004 case USB_SPEED_SUPER:
3007 dev_err(priv_dev->dev,
3008 "invalid maximum_speed parameter %d\n",
3011 case USB_SPEED_UNKNOWN:
3012 /* default to superspeed */
3013 max_speed = USB_SPEED_SUPER;
3017 cdns3_gadget_config(priv_dev);
3018 spin_unlock_irqrestore(&priv_dev->lock, flags);
3023 * cdns3_gadget_udc_stop - Stops gadget
3024 * @gadget: gadget object
3028 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
3030 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3031 struct cdns3_endpoint *priv_ep;
3032 u32 bEndpointAddress;
3036 priv_dev->gadget_driver = NULL;
3038 priv_dev->onchip_used_size = 0;
3039 priv_dev->out_mem_is_allocated = 0;
3040 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3042 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
3043 priv_ep = ep_to_cdns3_ep(ep);
3044 bEndpointAddress = priv_ep->num | priv_ep->dir;
3045 cdns3_select_ep(priv_dev, bEndpointAddress);
3046 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
3047 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
3048 !(val & EP_CMD_EPRST), 1, 100);
3050 priv_ep->flags &= ~EP_CLAIMED;
3053 /* disable interrupt for device */
3054 writel(0, &priv_dev->regs->usb_ien);
3055 writel(0, &priv_dev->regs->usb_pwr);
3056 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
3062 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
3063 * @gadget: pointer to the USB gadget
3065 * Used to record the maximum number of endpoints being used in a USB composite
3066 * device. (across all configurations) This is to be used in the calculation
3067 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3068 * It will help ensured that the resizing logic reserves enough space for at
3069 * least one max packet.
3071 static int cdns3_gadget_check_config(struct usb_gadget *gadget)
3073 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3074 struct cdns3_endpoint *priv_ep;
3082 list_for_each_entry(ep, &gadget->ep_list, ep_list) {
3083 priv_ep = ep_to_cdns3_ep(ep);
3084 if (!(priv_ep->flags & EP_CLAIMED))
3087 n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1);
3088 if (ep->address & USB_DIR_IN) {
3090 * ISO transfer: DMA start move data when get ISO, only transfer
3091 * data as min(TD size, iso). No benefit for allocate bigger
3092 * internal memory than 'iso'.
3094 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3099 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
3100 out = max_t(int, out, n);
3104 /* 2KB are reserved for EP0, 1KB for out*/
3105 total = 2 + n_in + out + iso;
3107 if (total > priv_dev->onchip_buffers)
3110 priv_dev->ep_buf_size = (priv_dev->onchip_buffers - 2 - iso) / (n_in + out);
3115 static const struct usb_gadget_ops cdns3_gadget_ops = {
3116 .get_frame = cdns3_gadget_get_frame,
3117 .wakeup = cdns3_gadget_wakeup,
3118 .set_selfpowered = cdns3_gadget_set_selfpowered,
3119 .pullup = cdns3_gadget_pullup,
3120 .udc_start = cdns3_gadget_udc_start,
3121 .udc_stop = cdns3_gadget_udc_stop,
3122 .match_ep = cdns3_gadget_match_ep,
3123 .check_config = cdns3_gadget_check_config,
3126 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
3130 /* ep0 OUT point to ep0 IN. */
3131 priv_dev->eps[16] = NULL;
3133 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
3134 if (priv_dev->eps[i]) {
3135 cdns3_free_trb_pool(priv_dev->eps[i]);
3136 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
3141 * cdns3_init_eps - Initializes software endpoints of gadget
3142 * @priv_dev: extended gadget object
3144 * Returns 0 on success, error code elsewhere
3146 static int cdns3_init_eps(struct cdns3_device *priv_dev)
3148 u32 ep_enabled_reg, iso_ep_reg;
3149 struct cdns3_endpoint *priv_ep;
3150 int ep_dir, ep_number;
3155 /* Read it from USB_CAP3 to USB_CAP5 */
3156 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3157 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3159 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3161 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3162 ep_dir = i >> 4; /* i div 16 */
3163 ep_number = i & 0xF; /* i % 16 */
3166 if (!(ep_enabled_reg & ep_mask))
3169 if (ep_dir && !ep_number) {
3170 priv_dev->eps[i] = priv_dev->eps[0];
3174 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3179 /* set parent of endpoint object */
3180 priv_ep->cdns3_dev = priv_dev;
3181 priv_dev->eps[i] = priv_ep;
3182 priv_ep->num = ep_number;
3183 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3186 ret = cdns3_init_ep0(priv_dev, priv_ep);
3188 dev_err(priv_dev->dev, "Failed to init ep0\n");
3192 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3193 ep_number, !!ep_dir ? "in" : "out");
3194 priv_ep->endpoint.name = priv_ep->name;
3196 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3197 CDNS3_EP_MAX_PACKET_LIMIT);
3198 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3199 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3201 priv_ep->endpoint.caps.dir_in = 1;
3203 priv_ep->endpoint.caps.dir_out = 1;
3205 if (iso_ep_reg & ep_mask)
3206 priv_ep->endpoint.caps.type_iso = 1;
3208 priv_ep->endpoint.caps.type_bulk = 1;
3209 priv_ep->endpoint.caps.type_int = 1;
3211 list_add_tail(&priv_ep->endpoint.ep_list,
3212 &priv_dev->gadget.ep_list);
3217 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n",
3219 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3220 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3222 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3223 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3224 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3229 cdns3_free_all_eps(priv_dev);
3233 static void cdns3_gadget_release(struct device *dev)
3235 struct cdns3_device *priv_dev = container_of(dev,
3236 struct cdns3_device, gadget.dev);
3241 static void cdns3_gadget_exit(struct cdns *cdns)
3243 struct cdns3_device *priv_dev;
3245 priv_dev = cdns->gadget_dev;
3248 pm_runtime_mark_last_busy(cdns->dev);
3249 pm_runtime_put_autosuspend(cdns->dev);
3251 usb_del_gadget(&priv_dev->gadget);
3252 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3254 cdns3_free_all_eps(priv_dev);
3256 while (!list_empty(&priv_dev->aligned_buf_list)) {
3257 struct cdns3_aligned_buf *buf;
3259 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3260 dma_free_noncoherent(priv_dev->sysdev, buf->size,
3265 list_del(&buf->list);
3269 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3270 priv_dev->setup_dma);
3271 dma_pool_destroy(priv_dev->eps_dma_pool);
3273 kfree(priv_dev->zlp_buf);
3274 usb_put_gadget(&priv_dev->gadget);
3275 cdns->gadget_dev = NULL;
3276 cdns_drd_gadget_off(cdns);
3279 static int cdns3_gadget_start(struct cdns *cdns)
3281 struct cdns3_device *priv_dev;
3285 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3289 usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3290 cdns3_gadget_release);
3291 cdns->gadget_dev = priv_dev;
3292 priv_dev->sysdev = cdns->dev;
3293 priv_dev->dev = cdns->dev;
3294 priv_dev->regs = cdns->dev_regs;
3296 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3297 &priv_dev->onchip_buffers);
3299 if (priv_dev->onchip_buffers <= 0) {
3300 u32 reg = readl(&priv_dev->regs->usb_cap2);
3302 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3305 if (!priv_dev->onchip_buffers)
3306 priv_dev->onchip_buffers = 256;
3308 max_speed = usb_get_maximum_speed(cdns->dev);
3310 /* Check the maximum_speed parameter */
3311 switch (max_speed) {
3312 case USB_SPEED_FULL:
3313 case USB_SPEED_HIGH:
3314 case USB_SPEED_SUPER:
3317 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3320 case USB_SPEED_UNKNOWN:
3321 /* default to superspeed */
3322 max_speed = USB_SPEED_SUPER;
3326 /* fill gadget fields */
3327 priv_dev->gadget.max_speed = max_speed;
3328 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3329 priv_dev->gadget.ops = &cdns3_gadget_ops;
3330 priv_dev->gadget.name = "usb-ss-gadget";
3331 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3332 priv_dev->gadget.irq = cdns->dev_irq;
3334 spin_lock_init(&priv_dev->lock);
3335 INIT_WORK(&priv_dev->pending_status_wq,
3336 cdns3_pending_setup_status_handler);
3338 INIT_WORK(&priv_dev->aligned_buf_wq,
3339 cdns3_free_aligned_request_buf);
3341 /* initialize endpoint container */
3342 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3343 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3344 priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3346 TRB_RING_SIZE, 8, 0);
3347 if (!priv_dev->eps_dma_pool) {
3348 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3353 ret = cdns3_init_eps(priv_dev);
3355 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3359 /* allocate memory for setup packet buffer */
3360 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3361 &priv_dev->setup_dma, GFP_DMA);
3362 if (!priv_dev->setup_buf) {
3367 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3369 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3370 readl(&priv_dev->regs->usb_cap6));
3371 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3372 readl(&priv_dev->regs->usb_cap1));
3373 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3374 readl(&priv_dev->regs->usb_cap2));
3376 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3377 if (priv_dev->dev_ver >= DEV_VER_V2)
3378 priv_dev->gadget.sg_supported = 1;
3380 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3381 if (!priv_dev->zlp_buf) {
3386 /* add USB gadget device */
3387 ret = usb_add_gadget(&priv_dev->gadget);
3389 dev_err(priv_dev->dev, "Failed to add gadget\n");
3395 kfree(priv_dev->zlp_buf);
3397 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3398 priv_dev->setup_dma);
3400 cdns3_free_all_eps(priv_dev);
3402 dma_pool_destroy(priv_dev->eps_dma_pool);
3404 usb_put_gadget(&priv_dev->gadget);
3405 cdns->gadget_dev = NULL;
3409 static int __cdns3_gadget_init(struct cdns *cdns)
3413 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3414 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3416 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3420 cdns_drd_gadget_on(cdns);
3421 pm_runtime_get_sync(cdns->dev);
3423 ret = cdns3_gadget_start(cdns);
3425 pm_runtime_put_sync(cdns->dev);
3430 * Because interrupt line can be shared with other components in
3431 * driver it can't use IRQF_ONESHOT flag here.
3433 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3434 cdns3_device_irq_handler,
3435 cdns3_device_thread_irq_handler,
3436 IRQF_SHARED, dev_name(cdns->dev),
3444 cdns3_gadget_exit(cdns);
3448 static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3449 __must_hold(&cdns->lock)
3451 struct cdns3_device *priv_dev = cdns->gadget_dev;
3453 spin_unlock(&cdns->lock);
3454 cdns3_disconnect_gadget(priv_dev);
3455 spin_lock(&cdns->lock);
3457 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3458 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3459 cdns3_hw_reset_eps_config(priv_dev);
3461 /* disable interrupt for device */
3462 writel(0, &priv_dev->regs->usb_ien);
3467 static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3469 struct cdns3_device *priv_dev = cdns->gadget_dev;
3471 if (!priv_dev->gadget_driver)
3474 cdns3_gadget_config(priv_dev);
3476 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3482 * cdns3_gadget_init - initialize device structure
3484 * @cdns: cdns instance
3486 * This function initializes the gadget.
3488 int cdns3_gadget_init(struct cdns *cdns)
3490 struct cdns_role_driver *rdrv;
3492 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3496 rdrv->start = __cdns3_gadget_init;
3497 rdrv->stop = cdns3_gadget_exit;
3498 rdrv->suspend = cdns3_gadget_suspend;
3499 rdrv->resume = cdns3_gadget_resume;
3500 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
3501 rdrv->name = "gadget";
3502 cdns->roles[USB_ROLE_DEVICE] = rdrv;