1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBSS DRD Driver - gadget side.
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
15 * At some situations, the controller may get stale data address in TRB
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
29 * Issue has been fixed in DEV_VER_V3 version of controller.
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
55 * Issue has been fixed in DEV_VER_V2 version of controller.
59 #include <linux/dma-mapping.h>
60 #include <linux/usb/gadget.h>
61 #include <linux/module.h>
62 #include <linux/iopoll.h>
65 #include "gadget-export.h"
70 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
71 struct usb_request *request,
74 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
75 struct usb_request *request);
77 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
78 struct usb_request *request);
81 * cdns3_clear_register_bit - clear bit in given register.
82 * @ptr: address of device controller register to be read and changed
83 * @mask: bits requested to clar
85 static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
87 mask = readl(ptr) & ~mask;
92 * cdns3_set_register_bit - set bit in given register.
93 * @ptr: address of device controller register to be read and changed
94 * @mask: bits requested to set
96 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
98 mask = readl(ptr) | mask;
103 * cdns3_ep_addr_to_index - Macro converts endpoint address to
104 * index of endpoint object in cdns3_device.eps[] container
105 * @ep_addr: endpoint address for which endpoint object is required
108 u8 cdns3_ep_addr_to_index(u8 ep_addr)
110 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
113 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
114 struct cdns3_endpoint *priv_ep)
118 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
120 return dma_index / TRB_SIZE;
124 * cdns3_next_request - returns next request from list
125 * @list: list containing requests
127 * Returns request or NULL if no requests in list
129 struct usb_request *cdns3_next_request(struct list_head *list)
131 return list_first_entry_or_null(list, struct usb_request, list);
135 * cdns3_next_align_buf - returns next buffer from list
136 * @list: list containing buffers
138 * Returns buffer or NULL if no buffers in list
140 static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
142 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
146 * cdns3_next_priv_request - returns next request from list
147 * @list: list containing requests
149 * Returns request or NULL if no requests in list
151 static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
153 return list_first_entry_or_null(list, struct cdns3_request, list);
157 * select_ep - selects endpoint
158 * @priv_dev: extended gadget object
159 * @ep: endpoint address
161 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
163 if (priv_dev->selected_ep == ep)
166 priv_dev->selected_ep = ep;
167 writel(ep, &priv_dev->regs->ep_sel);
171 * cdns3_get_tdl - gets current tdl for selected endpoint.
172 * @priv_dev: extended gadget object
174 * Before calling this function the appropriate endpoint must
175 * be selected by means of cdns3_select_ep function.
177 static int cdns3_get_tdl(struct cdns3_device *priv_dev)
179 if (priv_dev->dev_ver < DEV_VER_V3)
180 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
182 return readl(&priv_dev->regs->ep_tdl);
185 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
186 struct cdns3_trb *trb)
188 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
190 return priv_ep->trb_pool_dma + offset;
193 static int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
195 switch (priv_ep->type) {
196 case USB_ENDPOINT_XFER_ISOC:
197 return TRB_ISO_RING_SIZE;
198 case USB_ENDPOINT_XFER_CONTROL:
199 return TRB_CTRL_RING_SIZE;
201 if (priv_ep->use_streams)
202 return TRB_STREAM_RING_SIZE;
204 return TRB_RING_SIZE;
208 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
212 if (priv_ep->trb_pool) {
213 dma_free_coherent(priv_dev->sysdev,
214 cdns3_ring_size(priv_ep),
215 priv_ep->trb_pool, priv_ep->trb_pool_dma);
216 priv_ep->trb_pool = NULL;
221 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
222 * @priv_ep: endpoint object
224 * Function will return 0 on success or -ENOMEM on allocation error
226 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
228 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
229 int ring_size = cdns3_ring_size(priv_ep);
230 int num_trbs = ring_size / TRB_SIZE;
231 struct cdns3_trb *link_trb;
233 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
234 cdns3_free_trb_pool(priv_ep);
236 if (!priv_ep->trb_pool) {
237 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
239 &priv_ep->trb_pool_dma,
240 GFP_DMA32 | GFP_ATOMIC);
241 if (!priv_ep->trb_pool)
244 priv_ep->alloc_ring_size = ring_size;
247 memset(priv_ep->trb_pool, 0, ring_size);
249 priv_ep->num_trbs = num_trbs;
254 /* Initialize the last TRB as Link TRB */
255 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
257 if (priv_ep->use_streams) {
259 * For stream capable endpoints driver use single correct TRB.
260 * The last trb has zeroed cycle bit
262 link_trb->control = 0;
264 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
265 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
271 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
272 * @priv_ep: endpoint object
274 * Endpoint must be selected before call to this function
276 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
278 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
281 trace_cdns3_halt(priv_ep, 1, 1);
283 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
284 &priv_dev->regs->ep_cmd);
286 /* wait for DFLUSH cleared */
287 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
288 !(val & EP_CMD_DFLUSH), 1, 1000);
289 priv_ep->flags |= EP_STALLED;
290 priv_ep->flags &= ~EP_STALL_PENDING;
294 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
295 * @priv_dev: extended gadget object
297 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
301 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
303 cdns3_allow_enable_l1(priv_dev, 0);
304 priv_dev->hw_configured_flag = 0;
305 priv_dev->onchip_used_size = 0;
306 priv_dev->out_mem_is_allocated = 0;
307 priv_dev->wait_for_setup = 0;
308 priv_dev->using_streams = 0;
310 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
311 if (priv_dev->eps[i])
312 priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
316 * cdns3_ep_inc_trb - increment a trb index.
317 * @index: Pointer to the TRB index to increment.
319 * @trb_in_seg: number of TRBs in segment
321 * The index should never point to the link TRB. After incrementing,
322 * if it is point to the link TRB, wrap around to the beginning and revert
323 * cycle state bit The
324 * link TRB is always at the last TRB entry.
326 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
329 if (*index == (trb_in_seg - 1)) {
336 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
337 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
339 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
341 priv_ep->free_trbs--;
342 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
346 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
347 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
349 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
351 priv_ep->free_trbs++;
352 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
355 static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
357 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
358 int current_trb = priv_req->start_trb;
360 while (current_trb != priv_req->end_trb) {
361 cdns3_ep_inc_deq(priv_ep);
362 current_trb = priv_ep->dequeue;
365 cdns3_ep_inc_deq(priv_ep);
369 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
370 * @priv_dev: Extended gadget object
371 * @enable: Enable/disable permit to transition to L1.
373 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
374 * then controller answer with ACK handshake.
375 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
376 * then controller answer with NYET handshake.
378 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
381 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
383 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
386 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
390 reg = readl(&priv_dev->regs->usb_sts);
392 if (DEV_SUPERSPEED(reg))
393 return USB_SPEED_SUPER;
394 else if (DEV_HIGHSPEED(reg))
395 return USB_SPEED_HIGH;
396 else if (DEV_FULLSPEED(reg))
397 return USB_SPEED_FULL;
398 else if (DEV_LOWSPEED(reg))
399 return USB_SPEED_LOW;
400 return USB_SPEED_UNKNOWN;
404 * cdns3_start_all_request - add to ring all request not started
405 * @priv_dev: Extended gadget object
406 * @priv_ep: The endpoint for whom request will be started.
408 * Returns return ENOMEM if transfer ring i not enough TRBs to start
411 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
412 struct cdns3_endpoint *priv_ep)
414 struct usb_request *request;
416 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
419 * If the last pending transfer is INTERNAL
420 * OR streams are enabled for this endpoint
421 * do NOT start new transfer till the last one is pending
423 if (!pending_empty) {
424 struct cdns3_request *priv_req;
426 request = cdns3_next_request(&priv_ep->pending_req_list);
427 priv_req = to_cdns3_request(request);
428 if ((priv_req->flags & REQUEST_INTERNAL) ||
429 (priv_ep->flags & EP_TDLCHK_EN) ||
430 priv_ep->use_streams) {
431 dev_dbg(priv_dev->dev, "Blocking external request\n");
436 while (!list_empty(&priv_ep->deferred_req_list)) {
437 request = cdns3_next_request(&priv_ep->deferred_req_list);
439 if (!priv_ep->use_streams) {
440 ret = cdns3_ep_run_transfer(priv_ep, request);
442 priv_ep->stream_sg_idx = 0;
443 ret = cdns3_ep_run_stream_transfer(priv_ep, request);
448 list_del(&request->list);
449 list_add_tail(&request->list,
450 &priv_ep->pending_req_list);
451 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
455 priv_ep->flags &= ~EP_RING_FULL;
460 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
461 * driver try to detect whether endpoint need additional internal
462 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
463 * if before first DESCMISS interrupt the DMA will be armed.
465 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
466 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
467 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
468 (reg) |= EP_STS_EN_DESCMISEN; \
471 static void __cdns3_descmiss_copy_data(struct usb_request *request,
472 struct usb_request *descmiss_req)
474 int length = request->actual + descmiss_req->actual;
475 struct scatterlist *s = request->sg;
478 if (length <= request->length) {
479 memcpy(&((u8 *)request->buf)[request->actual],
481 descmiss_req->actual);
482 request->actual = length;
484 /* It should never occures */
485 request->status = -ENOMEM;
488 if (length <= sg_dma_len(s)) {
489 void *p = phys_to_virt(sg_dma_address(s));
491 memcpy(&((u8 *)p)[request->actual],
493 descmiss_req->actual);
494 request->actual = length;
496 request->status = -ENOMEM;
502 * cdns3_wa2_descmiss_copy_data copy data from internal requests to
503 * request queued by class driver.
504 * @priv_ep: extended endpoint object
505 * @request: request object
507 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
508 struct usb_request *request)
510 struct usb_request *descmiss_req;
511 struct cdns3_request *descmiss_priv_req;
513 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
517 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
518 descmiss_req = &descmiss_priv_req->request;
520 /* driver can't touch pending request */
521 if (descmiss_priv_req->flags & REQUEST_PENDING)
524 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
525 request->status = descmiss_req->status;
526 __cdns3_descmiss_copy_data(request, descmiss_req);
527 list_del_init(&descmiss_priv_req->list);
528 kfree(descmiss_req->buf);
529 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
530 --priv_ep->wa2_counter;
537 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
538 struct cdns3_endpoint *priv_ep,
539 struct cdns3_request *priv_req)
541 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
542 priv_req->flags & REQUEST_INTERNAL) {
543 struct usb_request *req;
545 req = cdns3_next_request(&priv_ep->deferred_req_list);
547 priv_ep->descmis_req = NULL;
552 /* unmap the gadget request before copying data */
553 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
556 cdns3_wa2_descmiss_copy_data(priv_ep, req);
557 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
558 req->length != req->actual) {
559 /* wait for next part of transfer */
560 /* re-map the gadget request buffer*/
561 usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
562 usb_endpoint_dir_in(priv_ep->endpoint.desc));
566 if (req->status == -EINPROGRESS)
569 list_del_init(&req->list);
570 cdns3_start_all_request(priv_dev, priv_ep);
574 return &priv_req->request;
577 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
578 struct cdns3_endpoint *priv_ep,
579 struct cdns3_request *priv_req)
584 * If transfer was queued before DESCMISS appear than we
585 * can disable handling of DESCMISS interrupt. Driver assumes that it
586 * can disable special treatment for this endpoint.
588 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
591 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
592 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
593 reg = readl(&priv_dev->regs->ep_sts_en);
594 reg &= ~EP_STS_EN_DESCMISEN;
595 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
596 writel(reg, &priv_dev->regs->ep_sts_en);
599 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
600 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
601 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
604 * DESCMISS transfer has been finished, so data will be
605 * directly copied from internal allocated usb_request
608 if (pending_empty && !descmiss_empty &&
609 !(priv_req->flags & REQUEST_INTERNAL)) {
610 cdns3_wa2_descmiss_copy_data(priv_ep,
613 trace_cdns3_wa2(priv_ep, "get internal stored data");
615 list_add_tail(&priv_req->request.list,
616 &priv_ep->pending_req_list);
617 cdns3_gadget_giveback(priv_ep, priv_req,
618 priv_req->request.status);
621 * Intentionally driver returns positive value as
622 * correct value. It informs that transfer has
629 * Driver will wait for completion DESCMISS transfer,
630 * before starts new, not DESCMISS transfer.
632 if (!pending_empty && !descmiss_empty) {
633 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
637 if (priv_req->flags & REQUEST_INTERNAL)
638 list_add_tail(&priv_req->list,
639 &priv_ep->wa2_descmiss_req_list);
645 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
647 struct cdns3_request *priv_req;
649 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
652 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
653 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
655 trace_cdns3_wa2(priv_ep, "removes eldest request");
657 kfree(priv_req->request.buf);
658 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
660 list_del_init(&priv_req->list);
661 --priv_ep->wa2_counter;
669 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
670 * @priv_ep: extended gadget object
672 * This function is used only for WA2. For more information see Work around 2
675 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
677 struct cdns3_request *priv_req;
678 struct usb_request *request;
679 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
681 /* check for pending transfer */
682 if (!pending_empty) {
683 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
687 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
688 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
689 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
692 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
694 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
695 trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
696 cdns3_wa2_remove_old_request(priv_ep);
699 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
704 priv_req = to_cdns3_request(request);
705 priv_req->flags |= REQUEST_INTERNAL;
707 /* if this field is still assigned it indicate that transfer related
708 * with this request has not been finished yet. Driver in this
709 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
710 * flag to previous one. It will indicate that current request is
711 * part of the previous one.
713 if (priv_ep->descmis_req)
714 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
716 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
718 priv_ep->wa2_counter++;
720 if (!priv_req->request.buf) {
721 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
725 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
726 priv_ep->descmis_req = priv_req;
728 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
729 &priv_ep->descmis_req->request,
735 dev_err(priv_ep->cdns3_dev->dev,
736 "Failed: No sufficient memory for DESCMIS\n");
739 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
741 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
744 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
746 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
747 &priv_dev->regs->ep_cmd);
751 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
756 cdns3_select_ep(priv_dev, 0);
758 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
760 if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
761 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
762 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
764 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
765 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
766 u8 pending_empty = list_empty(&outq_ep->pending_req_list);
768 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
769 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
775 cdns3_select_ep(priv_dev, outq_ep->num |
777 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
778 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
780 outq_ep->flags |= EP_TDLCHK_EN;
781 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
784 cdns3_wa2_enable_detection(priv_dev, outq_ep,
786 writel(ep_sts_en_reg,
787 &priv_dev->regs->ep_sts_en);
788 /* reset tdl value to zero */
789 cdns3_wa2_reset_tdl(priv_dev);
791 * Memory barrier - Reset tdl before ringing the
795 if (EP_CMD_DRDY & ep_cmd_reg) {
796 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
799 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
801 * ring doorbell to generate DESCMIS irq
804 &priv_dev->regs->ep_cmd);
812 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
813 * @priv_ep: The endpoint to whom the request belongs to
814 * @priv_req: The request we're giving back
815 * @status: completion code for the request
817 * Must be called with controller's lock held and interrupts disabled. This
818 * function will unmap @req and call its ->complete() callback to notify upper
819 * layers that it has completed.
821 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
822 struct cdns3_request *priv_req,
825 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
826 struct usb_request *request = &priv_req->request;
828 list_del_init(&request->list);
830 if (request->status == -EINPROGRESS)
831 request->status = status;
833 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
836 if ((priv_req->flags & REQUEST_UNALIGNED) &&
837 priv_ep->dir == USB_DIR_OUT && !request->status)
838 memcpy(request->buf, priv_req->aligned_buf->buf,
841 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
842 /* All TRBs have finished, clear the counter */
843 priv_req->finished_trb = 0;
844 trace_cdns3_gadget_giveback(priv_req);
846 if (priv_dev->dev_ver < DEV_VER_V2) {
847 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
853 if (request->complete) {
854 spin_unlock(&priv_dev->lock);
855 usb_gadget_giveback_request(&priv_ep->endpoint,
857 spin_lock(&priv_dev->lock);
860 if (request->buf == priv_dev->zlp_buf)
861 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
864 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
866 /* Work around for stale data address in TRB*/
867 if (priv_ep->wa1_set) {
868 trace_cdns3_wa1(priv_ep, "restore cycle bit");
870 priv_ep->wa1_set = 0;
871 priv_ep->wa1_trb_index = 0xFFFF;
872 if (priv_ep->wa1_cycle_bit) {
873 priv_ep->wa1_trb->control =
874 priv_ep->wa1_trb->control | cpu_to_le32(0x1);
876 priv_ep->wa1_trb->control =
877 priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
882 static void cdns3_free_aligned_request_buf(struct work_struct *work)
884 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
886 struct cdns3_aligned_buf *buf, *tmp;
889 spin_lock_irqsave(&priv_dev->lock, flags);
891 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
893 list_del(&buf->list);
896 * Re-enable interrupts to free DMA capable memory.
897 * Driver can't free this memory with disabled
900 spin_unlock_irqrestore(&priv_dev->lock, flags);
901 dma_free_coherent(priv_dev->sysdev, buf->size,
904 spin_lock_irqsave(&priv_dev->lock, flags);
908 spin_unlock_irqrestore(&priv_dev->lock, flags);
911 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
913 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
914 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
915 struct cdns3_aligned_buf *buf;
917 /* check if buffer is aligned to 8. */
918 if (!((uintptr_t)priv_req->request.buf & 0x7))
921 buf = priv_req->aligned_buf;
923 if (!buf || priv_req->request.length > buf->size) {
924 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
928 buf->size = priv_req->request.length;
930 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
939 if (priv_req->aligned_buf) {
940 trace_cdns3_free_aligned_request(priv_req);
941 priv_req->aligned_buf->in_use = 0;
942 queue_work(system_freezable_wq,
943 &priv_dev->aligned_buf_wq);
947 priv_req->aligned_buf = buf;
949 list_add_tail(&buf->list,
950 &priv_dev->aligned_buf_list);
953 if (priv_ep->dir == USB_DIR_IN) {
954 memcpy(buf->buf, priv_req->request.buf,
955 priv_req->request.length);
958 priv_req->flags |= REQUEST_UNALIGNED;
959 trace_cdns3_prepare_aligned_request(priv_req);
964 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
965 struct cdns3_trb *trb)
967 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
969 if (!priv_ep->wa1_set) {
972 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
975 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
976 priv_ep->wa1_set = 1;
977 priv_ep->wa1_trb = trb;
978 priv_ep->wa1_trb_index = priv_ep->enqueue;
979 trace_cdns3_wa1(priv_ep, "set guard");
986 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
987 struct cdns3_endpoint *priv_ep)
992 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
993 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
995 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
996 cdns3_wa1_restore_cycle_bit(priv_ep);
999 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
1000 struct usb_request *request)
1002 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1003 struct cdns3_request *priv_req;
1004 struct cdns3_trb *trb;
1010 unsigned int sg_idx = priv_ep->stream_sg_idx;
1012 priv_req = to_cdns3_request(request);
1013 address = priv_ep->endpoint.desc->bEndpointAddress;
1015 priv_ep->flags |= EP_PENDING_REQUEST;
1017 /* must allocate buffer aligned to 8 */
1018 if (priv_req->flags & REQUEST_UNALIGNED)
1019 trb_dma = priv_req->aligned_buf->dma;
1021 trb_dma = request->dma;
1023 /* For stream capable endpoints driver use only single TD. */
1024 trb = priv_ep->trb_pool + priv_ep->enqueue;
1025 priv_req->start_trb = priv_ep->enqueue;
1026 priv_req->end_trb = priv_req->start_trb;
1027 priv_req->trb = trb;
1029 cdns3_select_ep(priv_ep->cdns3_dev, address);
1031 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1032 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1034 if (!request->num_sgs) {
1035 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1036 length = request->length;
1038 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1039 length = request->sg[sg_idx].length;
1042 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1044 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1047 * For DEV_VER_V2 controller version we have enabled
1048 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1049 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1051 if (priv_dev->dev_ver >= DEV_VER_V2) {
1052 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1053 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1055 priv_req->flags |= REQUEST_PENDING;
1057 trb->control = cpu_to_le32(control);
1059 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1062 * Memory barrier - Cycle Bit must be set before trb->length and
1063 * trb->buffer fields.
1067 /* always first element */
1068 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1069 &priv_dev->regs->ep_traddr);
1071 if (!(priv_ep->flags & EP_STALLED)) {
1072 trace_cdns3_ring(priv_ep);
1073 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1074 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1076 priv_ep->prime_flag = false;
1079 * Controller version DEV_VER_V2 tdl calculation
1083 if (priv_dev->dev_ver < DEV_VER_V2)
1084 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1085 &priv_dev->regs->ep_cmd);
1086 else if (priv_dev->dev_ver > DEV_VER_V2)
1087 writel(tdl, &priv_dev->regs->ep_tdl);
1089 priv_ep->last_stream_id = priv_req->request.stream_id;
1090 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1091 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1092 EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1094 trace_cdns3_doorbell_epx(priv_ep->name,
1095 readl(&priv_dev->regs->ep_traddr));
1098 /* WORKAROUND for transition to L0 */
1099 __cdns3_gadget_wakeup(priv_dev);
1105 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1106 * @priv_ep: endpoint object
1107 * @request: request object
1109 * Returns zero on success or negative value on failure
1111 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1112 struct usb_request *request)
1114 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1115 struct cdns3_request *priv_req;
1116 struct cdns3_trb *trb;
1117 struct cdns3_trb *link_trb;
1126 struct scatterlist *s = NULL;
1127 bool sg_supported = !!(request->num_mapped_sgs);
1129 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1130 num_trb = priv_ep->interval;
1132 num_trb = sg_supported ? request->num_mapped_sgs : 1;
1134 if (num_trb > priv_ep->free_trbs) {
1135 priv_ep->flags |= EP_RING_FULL;
1139 priv_req = to_cdns3_request(request);
1140 address = priv_ep->endpoint.desc->bEndpointAddress;
1142 priv_ep->flags |= EP_PENDING_REQUEST;
1144 /* must allocate buffer aligned to 8 */
1145 if (priv_req->flags & REQUEST_UNALIGNED)
1146 trb_dma = priv_req->aligned_buf->dma;
1148 trb_dma = request->dma;
1150 trb = priv_ep->trb_pool + priv_ep->enqueue;
1151 priv_req->start_trb = priv_ep->enqueue;
1152 priv_req->trb = trb;
1154 cdns3_select_ep(priv_ep->cdns3_dev, address);
1157 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
1158 int doorbell, dma_index;
1161 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1162 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1164 /* Driver can't update LINK TRB if it is current processed. */
1165 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1166 priv_ep->flags |= EP_DEFERRED_DRDY;
1170 /*updating C bt in Link TRB before starting DMA*/
1171 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1173 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1174 * that DMA stuck at the LINK TRB.
1175 * On the other hand, removing TRB_CHAIN for longer TRs for
1176 * epXout cause that DMA stuck after handling LINK TRB.
1177 * To eliminate this strange behavioral driver set TRB_CHAIN
1178 * bit only for TR size > 2.
1180 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1181 TRBS_PER_SEGMENT > 2)
1184 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1185 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1188 if (priv_dev->dev_ver <= DEV_VER_V2)
1189 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1194 /* set incorrect Cycle Bit for first trb*/
1195 control = priv_ep->pcs ? 0 : TRB_CYCLE;
1202 control |= TRB_TYPE(TRB_NORMAL);
1204 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1205 length = sg_dma_len(s);
1207 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1208 length = request->length;
1211 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
1212 td_size = DIV_ROUND_UP(length,
1213 priv_ep->endpoint.maxpacket);
1214 else if (priv_ep->flags & EP_TDLCHK_EN)
1215 total_tdl += DIV_ROUND_UP(length,
1216 priv_ep->endpoint.maxpacket);
1218 trb->length = cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1220 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1221 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1223 control |= TRB_TDL_HS_SIZE(td_size);
1225 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1228 * first trb should be prepared as last to avoid processing
1234 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
1235 control |= TRB_IOC | TRB_ISP;
1237 /* for last element in TD or in SG list */
1238 if (sg_iter == (num_trb - 1) && sg_iter != 0)
1239 control |= pcs | TRB_IOC | TRB_ISP;
1243 trb->control = cpu_to_le32(control);
1245 priv_req->trb->control = cpu_to_le32(control);
1248 trb->control |= TRB_ISP;
1249 /* Don't set chain bit for last TRB */
1250 if (sg_iter < num_trb - 1)
1251 trb->control |= TRB_CHAIN;
1258 priv_req->end_trb = priv_ep->enqueue;
1259 cdns3_ep_inc_enq(priv_ep);
1260 trb = priv_ep->trb_pool + priv_ep->enqueue;
1261 } while (sg_iter < num_trb);
1263 trb = priv_req->trb;
1265 priv_req->flags |= REQUEST_PENDING;
1266 priv_req->num_of_trb = num_trb;
1269 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1271 if (priv_dev->dev_ver < DEV_VER_V2 &&
1272 (priv_ep->flags & EP_TDLCHK_EN)) {
1273 u16 tdl = total_tdl;
1274 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1276 if (tdl > EP_CMD_TDL_MAX) {
1277 tdl = EP_CMD_TDL_MAX;
1278 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1281 if (old_tdl < tdl) {
1283 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1284 &priv_dev->regs->ep_cmd);
1289 * Memory barrier - cycle bit must be set before other filds in trb.
1293 /* give the TD to the consumer*/
1295 trb->control = trb->control ^ cpu_to_le32(1);
1297 if (priv_dev->dev_ver <= DEV_VER_V2)
1298 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1303 while (i < num_trb) {
1304 trace_cdns3_prepare_trb(priv_ep, trb + i);
1305 if (trb + i == link_trb) {
1306 trb = priv_ep->trb_pool;
1307 num_trb = num_trb - i;
1314 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1318 * Memory barrier - Cycle Bit must be set before trb->length and
1319 * trb->buffer fields.
1324 * For DMULT mode we can set address to transfer ring only once after
1325 * enabling endpoint.
1327 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1329 * Until SW is not ready to handle the OUT transfer the ISO OUT
1330 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1331 * EP_CFG_ENABLE must be set before updating ep_traddr.
1333 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
1334 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1335 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1336 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1340 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1341 priv_req->start_trb * TRB_SIZE),
1342 &priv_dev->regs->ep_traddr);
1344 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1347 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1348 trace_cdns3_ring(priv_ep);
1349 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1350 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1351 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1352 trace_cdns3_doorbell_epx(priv_ep->name,
1353 readl(&priv_dev->regs->ep_traddr));
1356 /* WORKAROUND for transition to L0 */
1357 __cdns3_gadget_wakeup(priv_dev);
1362 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1364 struct cdns3_endpoint *priv_ep;
1367 if (priv_dev->hw_configured_flag)
1370 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1372 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1373 USB_CONF_U1EN | USB_CONF_U2EN);
1375 priv_dev->hw_configured_flag = 1;
1377 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1379 priv_ep = ep_to_cdns3_ep(ep);
1380 cdns3_start_all_request(priv_dev, priv_ep);
1384 cdns3_allow_enable_l1(priv_dev, 1);
1388 * cdns3_trb_handled - check whether trb has been handled by DMA
1390 * @priv_ep: extended endpoint object.
1391 * @priv_req: request object for checking
1393 * Endpoint must be selected before invoking this function.
1395 * Returns false if request has not been handled by DMA, else returns true.
1399 * DQ = priv_ep->dequeue - dequeue position
1400 * EQ = priv_ep->enqueue - enqueue position
1401 * ST = priv_req->start_trb - index of first TRB in transfer ring
1402 * ET = priv_req->end_trb - index of last TRB in transfer ring
1403 * CI = current_index - index of processed TRB by DMA.
1405 * As first step, we check if the TRB between the ST and ET.
1406 * Then, we check if cycle bit for index priv_ep->dequeue
1410 * 1. priv_ep->dequeue never equals to current_index.
1411 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1412 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1413 * and priv_ep->free_trbs is zero.
1414 * This case indicate that TR is full.
1416 * At below two cases, the request have been handled.
1417 * Case 1 - priv_ep->dequeue < current_index
1418 * SR ... EQ ... DQ ... CI ... ER
1419 * SR ... DQ ... CI ... EQ ... ER
1421 * Case 2 - priv_ep->dequeue > current_index
1422 * This situation takes place when CI go through the LINK TRB at the end of
1424 * SR ... CI ... EQ ... DQ ... ER
1426 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1427 struct cdns3_request *priv_req)
1429 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1430 struct cdns3_trb *trb;
1431 int current_index = 0;
1435 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1436 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1438 /* current trb doesn't belong to this request */
1439 if (priv_req->start_trb < priv_req->end_trb) {
1440 if (priv_ep->dequeue > priv_req->end_trb)
1443 if (priv_ep->dequeue < priv_req->start_trb)
1447 if ((priv_req->start_trb > priv_req->end_trb) &&
1448 (priv_ep->dequeue > priv_req->end_trb) &&
1449 (priv_ep->dequeue < priv_req->start_trb))
1452 if ((priv_req->start_trb == priv_req->end_trb) &&
1453 (priv_ep->dequeue != priv_req->end_trb))
1456 trb = &priv_ep->trb_pool[priv_ep->dequeue];
1458 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1461 if (doorbell == 1 && current_index == priv_ep->dequeue)
1464 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1465 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1470 if (priv_ep->enqueue == priv_ep->dequeue &&
1471 priv_ep->free_trbs == 0) {
1473 } else if (priv_ep->dequeue < current_index) {
1474 if ((current_index == (priv_ep->num_trbs - 1)) &&
1479 } else if (priv_ep->dequeue > current_index) {
1484 trace_cdns3_request_handled(priv_req, current_index, handled);
1489 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1490 struct cdns3_endpoint *priv_ep)
1492 struct cdns3_request *priv_req;
1493 struct usb_request *request;
1494 struct cdns3_trb *trb;
1495 bool request_handled = false;
1496 bool transfer_end = false;
1498 while (!list_empty(&priv_ep->pending_req_list)) {
1499 request = cdns3_next_request(&priv_ep->pending_req_list);
1500 priv_req = to_cdns3_request(request);
1502 trb = priv_ep->trb_pool + priv_ep->dequeue;
1504 /* Request was dequeued and TRB was changed to TRB_LINK. */
1505 if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1506 trace_cdns3_complete_trb(priv_ep, trb);
1507 cdns3_move_deq_to_next_trb(priv_req);
1510 if (!request->stream_id) {
1511 /* Re-select endpoint. It could be changed by other CPU
1512 * during handling usb_gadget_giveback_request.
1514 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1516 while (cdns3_trb_handled(priv_ep, priv_req)) {
1517 priv_req->finished_trb++;
1518 if (priv_req->finished_trb >= priv_req->num_of_trb)
1519 request_handled = true;
1521 trb = priv_ep->trb_pool + priv_ep->dequeue;
1522 trace_cdns3_complete_trb(priv_ep, trb);
1526 TRB_LEN(le32_to_cpu(trb->length));
1528 if (priv_req->num_of_trb > 1 &&
1529 le32_to_cpu(trb->control) & TRB_SMM)
1530 transfer_end = true;
1532 cdns3_ep_inc_deq(priv_ep);
1535 if (request_handled) {
1536 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1537 request_handled = false;
1538 transfer_end = false;
1540 goto prepare_next_td;
1543 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1544 TRBS_PER_SEGMENT == 2)
1547 /* Re-select endpoint. It could be changed by other CPU
1548 * during handling usb_gadget_giveback_request.
1550 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1552 trb = priv_ep->trb_pool;
1553 trace_cdns3_complete_trb(priv_ep, trb);
1555 if (trb != priv_req->trb)
1556 dev_warn(priv_dev->dev,
1557 "request_trb=0x%p, queue_trb=0x%p\n",
1558 priv_req->trb, trb);
1560 request->actual += TRB_LEN(le32_to_cpu(trb->length));
1562 if (!request->num_sgs ||
1563 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1564 priv_ep->stream_sg_idx = 0;
1565 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1567 priv_ep->stream_sg_idx++;
1568 cdns3_ep_run_stream_transfer(priv_ep, request);
1573 priv_ep->flags &= ~EP_PENDING_REQUEST;
1576 if (!(priv_ep->flags & EP_STALLED) &&
1577 !(priv_ep->flags & EP_STALL_PENDING))
1578 cdns3_start_all_request(priv_dev, priv_ep);
1581 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1583 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1585 cdns3_wa1_restore_cycle_bit(priv_ep);
1588 trace_cdns3_ring(priv_ep);
1590 /* Cycle Bit must be updated before arming DMA. */
1592 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1594 __cdns3_gadget_wakeup(priv_dev);
1596 trace_cdns3_doorbell_epx(priv_ep->name,
1597 readl(&priv_dev->regs->ep_traddr));
1601 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1603 u16 tdl = priv_ep->pending_tdl;
1604 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1606 if (tdl > EP_CMD_TDL_MAX) {
1607 tdl = EP_CMD_TDL_MAX;
1608 priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1610 priv_ep->pending_tdl = 0;
1613 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1617 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1618 * @priv_ep: endpoint object
1622 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1624 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1626 struct usb_request *deferred_request;
1627 struct usb_request *pending_request;
1630 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1632 trace_cdns3_epx_irq(priv_dev, priv_ep);
1634 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1635 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1637 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1638 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1640 tdl = cdns3_get_tdl(priv_dev);
1643 * Continue the previous transfer:
1644 * There is some racing between ERDY and PRIME. The device send
1645 * ERDY and almost in the same time Host send PRIME. It cause
1646 * that host ignore the ERDY packet and driver has to send it
1649 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1650 EP_STS_HOSTPP(ep_sts_reg))) {
1651 writel(EP_CMD_ERDY |
1652 EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1653 &priv_dev->regs->ep_cmd);
1654 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1656 priv_ep->prime_flag = true;
1658 pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1659 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1661 if (deferred_request && !pending_request) {
1662 cdns3_start_all_request(priv_dev, priv_ep);
1667 if (ep_sts_reg & EP_STS_TRBERR) {
1668 if (priv_ep->flags & EP_STALL_PENDING &&
1669 !(ep_sts_reg & EP_STS_DESCMIS &&
1670 priv_dev->dev_ver < DEV_VER_V2)) {
1671 cdns3_ep_stall_flush(priv_ep);
1675 * For isochronous transfer driver completes request on
1676 * IOC or on TRBERR. IOC appears only when device receive
1677 * OUT data packet. If host disable stream or lost some packet
1678 * then the only way to finish all queued transfer is to do it
1681 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1682 !priv_ep->wa1_set) {
1683 if (!priv_ep->dir) {
1684 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1686 ep_cfg &= ~EP_CFG_ENABLE;
1687 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1688 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1690 cdns3_transfer_completed(priv_dev, priv_ep);
1691 } else if (!(priv_ep->flags & EP_STALLED) &&
1692 !(priv_ep->flags & EP_STALL_PENDING)) {
1693 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1694 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1695 cdns3_start_all_request(priv_dev, priv_ep);
1697 cdns3_rearm_transfer(priv_ep,
1703 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1704 (ep_sts_reg & EP_STS_IOT)) {
1705 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1706 if (ep_sts_reg & EP_STS_ISP)
1707 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1709 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1712 if (!priv_ep->use_streams) {
1713 if ((ep_sts_reg & EP_STS_IOC) ||
1714 (ep_sts_reg & EP_STS_ISP)) {
1715 cdns3_transfer_completed(priv_dev, priv_ep);
1716 } else if ((priv_ep->flags & EP_TDLCHK_EN) &
1717 priv_ep->pending_tdl) {
1718 /* handle IOT with pending tdl */
1719 cdns3_reprogram_tdl(priv_ep);
1721 } else if (priv_ep->dir == USB_DIR_OUT) {
1722 priv_ep->ep_sts_pending |= ep_sts_reg;
1723 } else if (ep_sts_reg & EP_STS_IOT) {
1724 cdns3_transfer_completed(priv_dev, priv_ep);
1729 * MD_EXIT interrupt sets when stream capable endpoint exits
1730 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1732 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1733 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1734 priv_ep->ep_sts_pending = 0;
1735 cdns3_transfer_completed(priv_dev, priv_ep);
1739 * WA2: this condition should only be meet when
1740 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1741 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1742 * In other cases this interrupt will be disabled.
1744 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1745 !(priv_ep->flags & EP_STALLED))
1746 cdns3_wa2_descmissing_packet(priv_ep);
1751 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1753 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1754 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1758 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1759 * @priv_dev: extended gadget object
1760 * @usb_ists: bitmap representation of device's reported interrupts
1761 * (usb_ists register value)
1763 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1765 __must_hold(&priv_dev->lock)
1769 trace_cdns3_usb_irq(priv_dev, usb_ists);
1770 if (usb_ists & USB_ISTS_L1ENTI) {
1772 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1773 * from L1. To fix it, if any DMA transfer is pending driver
1774 * must starts driving resume signal immediately.
1776 if (readl(&priv_dev->regs->drbl))
1777 __cdns3_gadget_wakeup(priv_dev);
1780 /* Connection detected */
1781 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1782 speed = cdns3_get_speed(priv_dev);
1783 priv_dev->gadget.speed = speed;
1784 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1785 cdns3_ep0_config(priv_dev);
1788 /* Disconnection detected */
1789 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1790 spin_unlock(&priv_dev->lock);
1791 cdns3_disconnect_gadget(priv_dev);
1792 spin_lock(&priv_dev->lock);
1793 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1794 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1795 cdns3_hw_reset_eps_config(priv_dev);
1798 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1799 if (priv_dev->gadget_driver &&
1800 priv_dev->gadget_driver->suspend) {
1801 spin_unlock(&priv_dev->lock);
1802 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1803 spin_lock(&priv_dev->lock);
1807 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1808 if (priv_dev->gadget_driver &&
1809 priv_dev->gadget_driver->resume) {
1810 spin_unlock(&priv_dev->lock);
1811 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1812 spin_lock(&priv_dev->lock);
1817 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1818 if (priv_dev->gadget_driver) {
1819 spin_unlock(&priv_dev->lock);
1820 usb_gadget_udc_reset(&priv_dev->gadget,
1821 priv_dev->gadget_driver);
1822 spin_lock(&priv_dev->lock);
1824 /*read again to check the actual speed*/
1825 speed = cdns3_get_speed(priv_dev);
1826 priv_dev->gadget.speed = speed;
1827 cdns3_hw_reset_eps_config(priv_dev);
1828 cdns3_ep0_config(priv_dev);
1834 * cdns3_device_irq_handler- interrupt handler for device part of controller
1836 * @irq: irq number for cdns3 core device
1837 * @data: structure of cdns3
1839 * Returns IRQ_HANDLED or IRQ_NONE
1841 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1843 struct cdns3_device *priv_dev = data;
1844 struct cdns3 *cdns = dev_get_drvdata(priv_dev->dev);
1845 irqreturn_t ret = IRQ_NONE;
1851 /* check USB device interrupt */
1852 reg = readl(&priv_dev->regs->usb_ists);
1854 /* After masking interrupts the new interrupts won't be
1855 * reported in usb_ists/ep_ists. In order to not lose some
1856 * of them driver disables only detected interrupts.
1857 * They will be enabled ASAP after clearing source of
1858 * interrupt. This an unusual behavior only applies to
1859 * usb_ists register.
1861 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1862 /* mask deferred interrupt. */
1863 writel(reg, &priv_dev->regs->usb_ien);
1864 ret = IRQ_WAKE_THREAD;
1867 /* check endpoint interrupt */
1868 reg = readl(&priv_dev->regs->ep_ists);
1870 writel(0, &priv_dev->regs->ep_ien);
1871 ret = IRQ_WAKE_THREAD;
1878 * cdns3_device_thread_irq_handler- interrupt handler for device part
1881 * @irq: irq number for cdns3 core device
1882 * @data: structure of cdns3
1884 * Returns IRQ_HANDLED or IRQ_NONE
1886 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1888 struct cdns3_device *priv_dev = data;
1889 irqreturn_t ret = IRQ_NONE;
1890 unsigned long flags;
1894 spin_lock_irqsave(&priv_dev->lock, flags);
1896 reg = readl(&priv_dev->regs->usb_ists);
1898 writel(reg, &priv_dev->regs->usb_ists);
1899 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1900 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1904 reg = readl(&priv_dev->regs->ep_ists);
1906 /* handle default endpoint OUT */
1907 if (reg & EP_ISTS_EP_OUT0) {
1908 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1912 /* handle default endpoint IN */
1913 if (reg & EP_ISTS_EP_IN0) {
1914 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1918 /* check if interrupt from non default endpoint, if no exit */
1919 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1923 for_each_set_bit(bit, ®,
1924 sizeof(u32) * BITS_PER_BYTE) {
1925 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1929 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1930 cdns3_wa2_check_outq_status(priv_dev);
1933 writel(~0, &priv_dev->regs->ep_ien);
1934 spin_unlock_irqrestore(&priv_dev->lock, flags);
1940 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1942 * The real reservation will occur during write to EP_CFG register,
1943 * this function is used to check if the 'size' reservation is allowed.
1945 * @priv_dev: extended gadget object
1946 * @size: the size (KB) for EP would like to allocate
1947 * @is_in: endpoint direction
1949 * Return 0 if the required size can met or negative value on failure
1951 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1952 int size, int is_in)
1956 /* 2KB are reserved for EP0*/
1957 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1960 if (remained < size)
1963 priv_dev->onchip_used_size += size;
1968 * ALL OUT EPs are shared the same chunk onchip memory, so
1969 * driver checks if it already has assigned enough buffers
1971 if (priv_dev->out_mem_is_allocated >= size)
1974 required = size - priv_dev->out_mem_is_allocated;
1976 if (required > remained)
1979 priv_dev->out_mem_is_allocated += required;
1980 priv_dev->onchip_used_size += required;
1986 static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1987 struct cdns3_endpoint *priv_ep)
1989 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1991 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1992 if (priv_dev->dev_ver <= DEV_VER_V2)
1993 writel(USB_CONF_DMULT, ®s->usb_conf);
1995 if (priv_dev->dev_ver == DEV_VER_V2)
1996 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
1998 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2002 mask = BIT(priv_ep->num + 16);
2004 mask = BIT(priv_ep->num);
2006 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
2007 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2008 cdns3_set_register_bit(®s->tdl_beh, mask);
2009 cdns3_set_register_bit(®s->tdl_beh2, mask);
2010 cdns3_set_register_bit(®s->dma_adv_td, mask);
2013 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2014 cdns3_set_register_bit(®s->tdl_from_trb, mask);
2016 cdns3_set_register_bit(®s->dtrans, mask);
2021 * cdns3_ep_config Configure hardware endpoint
2022 * @priv_ep: extended endpoint object
2023 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2025 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2027 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2028 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2029 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2030 u32 max_packet_size = 0;
2037 buffering = CDNS3_EP_BUF_SIZE - 1;
2039 cdns3_configure_dmult(priv_dev, priv_ep);
2041 switch (priv_ep->type) {
2042 case USB_ENDPOINT_XFER_INT:
2043 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2045 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
2046 priv_dev->dev_ver > DEV_VER_V2)
2047 ep_cfg |= EP_CFG_TDL_CHK;
2049 case USB_ENDPOINT_XFER_BULK:
2050 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2052 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
2053 priv_dev->dev_ver > DEV_VER_V2)
2054 ep_cfg |= EP_CFG_TDL_CHK;
2057 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2058 mult = CDNS3_EP_ISO_HS_MULT - 1;
2059 buffering = mult + 1;
2062 switch (priv_dev->gadget.speed) {
2063 case USB_SPEED_FULL:
2064 max_packet_size = is_iso_ep ? 1023 : 64;
2066 case USB_SPEED_HIGH:
2067 max_packet_size = is_iso_ep ? 1024 : 512;
2069 case USB_SPEED_SUPER:
2070 /* It's limitation that driver assumes in driver. */
2072 max_packet_size = 1024;
2073 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2074 maxburst = CDNS3_EP_ISO_SS_BURST - 1;
2075 buffering = (mult + 1) *
2078 if (priv_ep->interval > 1)
2081 maxburst = CDNS3_EP_BUF_SIZE - 1;
2085 /* all other speed are not supported */
2089 if (max_packet_size == 1024)
2090 priv_ep->trb_burst_size = 128;
2091 else if (max_packet_size >= 512)
2092 priv_ep->trb_burst_size = 64;
2094 priv_ep->trb_burst_size = 16;
2096 /* onchip buffer is only allocated before configuration */
2097 if (!priv_dev->hw_configured_flag) {
2098 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2101 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2107 ep_cfg |= EP_CFG_ENABLE;
2109 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2110 if (priv_dev->dev_ver >= DEV_VER_V3) {
2111 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2114 * Stream capable endpoints are handled by using ep_tdl
2115 * register. Other endpoints use TDL from TRB feature.
2117 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2121 /* Enable Stream Bit TDL chk and SID chk */
2122 ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2125 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2127 EP_CFG_BUFFERING(buffering) |
2128 EP_CFG_MAXBURST(maxburst);
2130 cdns3_select_ep(priv_dev, bEndpointAddress);
2131 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2132 priv_ep->flags |= EP_CONFIGURED;
2134 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2135 priv_ep->name, ep_cfg);
2140 /* Find correct direction for HW endpoint according to description */
2141 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2142 struct cdns3_endpoint *priv_ep)
2144 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2145 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2149 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2150 struct usb_endpoint_descriptor *desc)
2153 struct cdns3_endpoint *priv_ep;
2155 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2158 /* ep name pattern likes epXin or epXout */
2159 char c[2] = {ep->name[2], '\0'};
2161 ret = kstrtoul(c, 10, &num);
2163 return ERR_PTR(ret);
2165 priv_ep = ep_to_cdns3_ep(ep);
2166 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2167 if (!(priv_ep->flags & EP_CLAIMED)) {
2174 return ERR_PTR(-ENOENT);
2178 * Cadence IP has one limitation that all endpoints must be configured
2179 * (Type & MaxPacketSize) before setting configuration through hardware
2180 * register, it means we can't change endpoints configuration after
2181 * set_configuration.
2183 * This function set EP_CLAIMED flag which is added when the gadget driver
2184 * uses usb_ep_autoconfig to configure specific endpoint;
2185 * When the udc driver receives set_configurion request,
2186 * it goes through all claimed endpoints, and configure all endpoints
2189 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2190 * ep_cfg register which can be changed after set_configuration, and do
2191 * some software operation accordingly.
2194 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2195 struct usb_endpoint_descriptor *desc,
2196 struct usb_ss_ep_comp_descriptor *comp_desc)
2198 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2199 struct cdns3_endpoint *priv_ep;
2200 unsigned long flags;
2202 priv_ep = cdns3_find_available_ep(priv_dev, desc);
2203 if (IS_ERR(priv_ep)) {
2204 dev_err(priv_dev->dev, "no available ep\n");
2208 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2210 spin_lock_irqsave(&priv_dev->lock, flags);
2211 priv_ep->endpoint.desc = desc;
2212 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2213 priv_ep->type = usb_endpoint_type(desc);
2214 priv_ep->flags |= EP_CLAIMED;
2215 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2217 spin_unlock_irqrestore(&priv_dev->lock, flags);
2218 return &priv_ep->endpoint;
2222 * cdns3_gadget_ep_alloc_request Allocates request
2223 * @ep: endpoint object associated with request
2224 * @gfp_flags: gfp flags
2226 * Returns allocated request address, NULL on allocation error
2228 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2231 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2232 struct cdns3_request *priv_req;
2234 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2238 priv_req->priv_ep = priv_ep;
2240 trace_cdns3_alloc_request(priv_req);
2241 return &priv_req->request;
2245 * cdns3_gadget_ep_free_request Free memory occupied by request
2246 * @ep: endpoint object associated with request
2247 * @request: request to free memory
2249 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2250 struct usb_request *request)
2252 struct cdns3_request *priv_req = to_cdns3_request(request);
2254 if (priv_req->aligned_buf)
2255 priv_req->aligned_buf->in_use = 0;
2257 trace_cdns3_free_request(priv_req);
2262 * cdns3_gadget_ep_enable Enable endpoint
2263 * @ep: endpoint object
2264 * @desc: endpoint descriptor
2266 * Returns 0 on success, error code elsewhere
2268 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2269 const struct usb_endpoint_descriptor *desc)
2271 struct cdns3_endpoint *priv_ep;
2272 struct cdns3_device *priv_dev;
2273 const struct usb_ss_ep_comp_descriptor *comp_desc;
2274 u32 reg = EP_STS_EN_TRBERREN;
2275 u32 bEndpointAddress;
2276 unsigned long flags;
2281 priv_ep = ep_to_cdns3_ep(ep);
2282 priv_dev = priv_ep->cdns3_dev;
2283 comp_desc = priv_ep->endpoint.comp_desc;
2285 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2286 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2290 if (!desc->wMaxPacketSize) {
2291 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2295 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2296 "%s is already enabled\n", priv_ep->name))
2299 spin_lock_irqsave(&priv_dev->lock, flags);
2301 priv_ep->endpoint.desc = desc;
2302 priv_ep->type = usb_endpoint_type(desc);
2303 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2305 if (priv_ep->interval > ISO_MAX_INTERVAL &&
2306 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2307 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2314 bEndpointAddress = priv_ep->num | priv_ep->dir;
2315 cdns3_select_ep(priv_dev, bEndpointAddress);
2318 * For some versions of controller at some point during ISO OUT traffic
2319 * DMA reads Transfer Ring for the EP which has never got doorbell.
2320 * This issue was detected only on simulation, but to avoid this issue
2321 * driver add protection against it. To fix it driver enable ISO OUT
2322 * endpoint before setting DRBL. This special treatment of ISO OUT
2323 * endpoints are recommended by controller specification.
2325 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2328 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2330 * Enable stream support (SS mode) related interrupts
2331 * in EP_STS_EN Register
2333 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2334 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2335 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2336 EP_STS_EN_STREAMREN;
2337 priv_ep->use_streams = true;
2338 ret = cdns3_ep_config(priv_ep, enable);
2339 priv_dev->using_streams |= true;
2342 ret = cdns3_ep_config(priv_ep, enable);
2348 ret = cdns3_allocate_trb_pool(priv_ep);
2352 bEndpointAddress = priv_ep->num | priv_ep->dir;
2353 cdns3_select_ep(priv_dev, bEndpointAddress);
2355 trace_cdns3_gadget_ep_enable(priv_ep);
2357 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2359 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2360 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2363 if (unlikely(ret)) {
2364 cdns3_free_trb_pool(priv_ep);
2369 /* enable interrupt for selected endpoint */
2370 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2371 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2373 if (priv_dev->dev_ver < DEV_VER_V2)
2374 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2376 writel(reg, &priv_dev->regs->ep_sts_en);
2379 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2380 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2381 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2382 priv_ep->wa1_set = 0;
2383 priv_ep->enqueue = 0;
2384 priv_ep->dequeue = 0;
2385 reg = readl(&priv_dev->regs->ep_sts);
2386 priv_ep->pcs = !!EP_STS_CCS(reg);
2387 priv_ep->ccs = !!EP_STS_CCS(reg);
2388 /* one TRB is reserved for link TRB used in DMULT mode*/
2389 priv_ep->free_trbs = priv_ep->num_trbs - 1;
2391 spin_unlock_irqrestore(&priv_dev->lock, flags);
2397 * cdns3_gadget_ep_disable Disable endpoint
2398 * @ep: endpoint object
2400 * Returns 0 on success, error code elsewhere
2402 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2404 struct cdns3_endpoint *priv_ep;
2405 struct cdns3_request *priv_req;
2406 struct cdns3_device *priv_dev;
2407 struct usb_request *request;
2408 unsigned long flags;
2414 pr_err("usbss: invalid parameters\n");
2418 priv_ep = ep_to_cdns3_ep(ep);
2419 priv_dev = priv_ep->cdns3_dev;
2421 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2422 "%s is already disabled\n", priv_ep->name))
2425 spin_lock_irqsave(&priv_dev->lock, flags);
2427 trace_cdns3_gadget_ep_disable(priv_ep);
2429 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2431 ep_cfg = readl(&priv_dev->regs->ep_cfg);
2432 ep_cfg &= ~EP_CFG_ENABLE;
2433 writel(ep_cfg, &priv_dev->regs->ep_cfg);
2436 * Driver needs some time before resetting endpoint.
2437 * It need waits for clearing DBUSY bit or for timeout expired.
2438 * 10us is enough time for controller to stop transfer.
2440 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2441 !(val & EP_STS_DBUSY), 1, 10);
2442 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2444 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2445 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2448 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2451 while (!list_empty(&priv_ep->pending_req_list)) {
2452 request = cdns3_next_request(&priv_ep->pending_req_list);
2454 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2458 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2459 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2461 kfree(priv_req->request.buf);
2462 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2463 &priv_req->request);
2464 list_del_init(&priv_req->list);
2465 --priv_ep->wa2_counter;
2468 while (!list_empty(&priv_ep->deferred_req_list)) {
2469 request = cdns3_next_request(&priv_ep->deferred_req_list);
2471 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2475 priv_ep->descmis_req = NULL;
2478 priv_ep->flags &= ~EP_ENABLED;
2479 priv_ep->use_streams = false;
2481 spin_unlock_irqrestore(&priv_dev->lock, flags);
2487 * cdns3_gadget_ep_queue Transfer data on endpoint
2488 * @ep: endpoint object
2489 * @request: request object
2490 * @gfp_flags: gfp flags
2492 * Returns 0 on success, error code elsewhere
2494 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2495 struct usb_request *request,
2498 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2499 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2500 struct cdns3_request *priv_req;
2503 request->actual = 0;
2504 request->status = -EINPROGRESS;
2505 priv_req = to_cdns3_request(request);
2506 trace_cdns3_ep_queue(priv_req);
2508 if (priv_dev->dev_ver < DEV_VER_V2) {
2509 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2512 if (ret == EINPROGRESS)
2516 ret = cdns3_prepare_aligned_request_buf(priv_req);
2520 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2521 usb_endpoint_dir_in(ep->desc));
2525 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2528 * For stream capable endpoint if prime irq flag is set then only start
2530 * If hardware endpoint configuration has not been set yet then
2531 * just queue request in deferred list. Transfer will be started in
2532 * cdns3_set_hw_configuration.
2534 if (!request->stream_id) {
2535 if (priv_dev->hw_configured_flag &&
2536 !(priv_ep->flags & EP_STALLED) &&
2537 !(priv_ep->flags & EP_STALL_PENDING))
2538 cdns3_start_all_request(priv_dev, priv_ep);
2540 if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2541 cdns3_start_all_request(priv_dev, priv_ep);
2547 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2550 struct usb_request *zlp_request;
2551 struct cdns3_endpoint *priv_ep;
2552 struct cdns3_device *priv_dev;
2553 unsigned long flags;
2556 if (!request || !ep)
2559 priv_ep = ep_to_cdns3_ep(ep);
2560 priv_dev = priv_ep->cdns3_dev;
2562 spin_lock_irqsave(&priv_dev->lock, flags);
2564 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2566 if (ret == 0 && request->zero && request->length &&
2567 (request->length % ep->maxpacket == 0)) {
2568 struct cdns3_request *priv_req;
2570 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2571 zlp_request->buf = priv_dev->zlp_buf;
2572 zlp_request->length = 0;
2574 priv_req = to_cdns3_request(zlp_request);
2575 priv_req->flags |= REQUEST_ZLP;
2577 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2579 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2582 spin_unlock_irqrestore(&priv_dev->lock, flags);
2587 * cdns3_gadget_ep_dequeue Remove request from transfer queue
2588 * @ep: endpoint object associated with request
2589 * @request: request object
2591 * Returns 0 on success, error code elsewhere
2593 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2594 struct usb_request *request)
2596 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2597 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2598 struct usb_request *req, *req_temp;
2599 struct cdns3_request *priv_req;
2600 struct cdns3_trb *link_trb;
2601 u8 req_on_hw_ring = 0;
2602 unsigned long flags;
2605 if (!ep || !request || !ep->desc)
2608 spin_lock_irqsave(&priv_dev->lock, flags);
2610 priv_req = to_cdns3_request(request);
2612 trace_cdns3_ep_dequeue(priv_req);
2614 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2616 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2618 if (request == req) {
2624 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2633 link_trb = priv_req->trb;
2635 /* Update ring only if removed request is on pending_req_list list */
2636 if (req_on_hw_ring && link_trb) {
2637 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2638 ((priv_req->end_trb + 1) * TRB_SIZE)));
2639 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2640 TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2642 if (priv_ep->wa1_trb == priv_req->trb)
2643 cdns3_wa1_restore_cycle_bit(priv_ep);
2646 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2649 spin_unlock_irqrestore(&priv_dev->lock, flags);
2654 * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2655 * Should be called after acquiring spin_lock and selecting ep
2656 * @priv_ep: endpoint object to set stall on.
2658 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2660 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2662 trace_cdns3_halt(priv_ep, 1, 0);
2664 if (!(priv_ep->flags & EP_STALLED)) {
2665 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2667 if (!(ep_sts_reg & EP_STS_DBUSY))
2668 cdns3_ep_stall_flush(priv_ep);
2670 priv_ep->flags |= EP_STALL_PENDING;
2675 * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2676 * Should be called after acquiring spin_lock and selecting ep
2677 * @priv_ep: endpoint object to clear stall on
2679 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2681 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2682 struct usb_request *request;
2683 struct cdns3_request *priv_req;
2684 struct cdns3_trb *trb = NULL;
2688 trace_cdns3_halt(priv_ep, 0, 0);
2690 request = cdns3_next_request(&priv_ep->pending_req_list);
2692 priv_req = to_cdns3_request(request);
2693 trb = priv_req->trb;
2695 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2698 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2700 /* wait for EPRST cleared */
2701 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2702 !(val & EP_CMD_EPRST), 1, 100);
2706 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2710 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2712 cdns3_rearm_transfer(priv_ep, 1);
2715 cdns3_start_all_request(priv_dev, priv_ep);
2720 * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2721 * @ep: endpoint object to set/clear stall on
2722 * @value: 1 for set stall, 0 for clear stall
2724 * Returns 0 on success, error code elsewhere
2726 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2728 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2729 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2730 unsigned long flags;
2733 if (!(priv_ep->flags & EP_ENABLED))
2736 spin_lock_irqsave(&priv_dev->lock, flags);
2738 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2741 priv_ep->flags &= ~EP_WEDGE;
2742 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2744 __cdns3_gadget_ep_set_halt(priv_ep);
2747 spin_unlock_irqrestore(&priv_dev->lock, flags);
2752 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2754 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2755 .enable = cdns3_gadget_ep_enable,
2756 .disable = cdns3_gadget_ep_disable,
2757 .alloc_request = cdns3_gadget_ep_alloc_request,
2758 .free_request = cdns3_gadget_ep_free_request,
2759 .queue = cdns3_gadget_ep_queue,
2760 .dequeue = cdns3_gadget_ep_dequeue,
2761 .set_halt = cdns3_gadget_ep_set_halt,
2762 .set_wedge = cdns3_gadget_ep_set_wedge,
2766 * cdns3_gadget_get_frame Returns number of actual ITP frame
2767 * @gadget: gadget object
2769 * Returns number of actual ITP frame
2771 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2773 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2775 return readl(&priv_dev->regs->usb_itpn);
2778 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2780 enum usb_device_speed speed;
2782 speed = cdns3_get_speed(priv_dev);
2784 if (speed >= USB_SPEED_SUPER)
2787 /* Start driving resume signaling to indicate remote wakeup. */
2788 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2793 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2795 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2796 unsigned long flags;
2799 spin_lock_irqsave(&priv_dev->lock, flags);
2800 ret = __cdns3_gadget_wakeup(priv_dev);
2801 spin_unlock_irqrestore(&priv_dev->lock, flags);
2805 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2808 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2809 unsigned long flags;
2811 spin_lock_irqsave(&priv_dev->lock, flags);
2812 priv_dev->is_selfpowered = !!is_selfpowered;
2813 spin_unlock_irqrestore(&priv_dev->lock, flags);
2817 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2819 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2822 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2824 writel(~0, &priv_dev->regs->ep_ists);
2825 writel(~0, &priv_dev->regs->usb_ists);
2826 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2832 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2834 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2837 cdns3_ep0_config(priv_dev);
2839 /* enable interrupts for endpoint 0 (in and out) */
2840 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2843 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2844 * revision of controller.
2846 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2847 reg = readl(®s->dbg_link1);
2849 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2850 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2851 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2852 writel(reg, ®s->dbg_link1);
2856 * By default some platforms has set protected access to memory.
2857 * This cause problem with cache, so driver restore non-secure
2860 reg = readl(®s->dma_axi_ctrl);
2861 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2862 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2863 writel(reg, ®s->dma_axi_ctrl);
2865 /* enable generic interrupt*/
2866 writel(USB_IEN_INIT, ®s->usb_ien);
2867 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2868 /* keep Fast Access bit */
2869 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2871 cdns3_configure_dmult(priv_dev, NULL);
2875 * cdns3_gadget_udc_start Gadget start
2876 * @gadget: gadget object
2877 * @driver: driver which operates on this gadget
2879 * Returns 0 on success, error code elsewhere
2881 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2882 struct usb_gadget_driver *driver)
2884 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2885 unsigned long flags;
2886 enum usb_device_speed max_speed = driver->max_speed;
2888 spin_lock_irqsave(&priv_dev->lock, flags);
2889 priv_dev->gadget_driver = driver;
2891 /* limit speed if necessary */
2892 max_speed = min(driver->max_speed, gadget->max_speed);
2894 switch (max_speed) {
2895 case USB_SPEED_FULL:
2896 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2897 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2899 case USB_SPEED_HIGH:
2900 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2902 case USB_SPEED_SUPER:
2905 dev_err(priv_dev->dev,
2906 "invalid maximum_speed parameter %d\n",
2909 case USB_SPEED_UNKNOWN:
2910 /* default to superspeed */
2911 max_speed = USB_SPEED_SUPER;
2915 cdns3_gadget_config(priv_dev);
2916 spin_unlock_irqrestore(&priv_dev->lock, flags);
2921 * cdns3_gadget_udc_stop Stops gadget
2922 * @gadget: gadget object
2926 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2928 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2929 struct cdns3_endpoint *priv_ep;
2930 u32 bEndpointAddress;
2934 priv_dev->gadget_driver = NULL;
2936 priv_dev->onchip_used_size = 0;
2937 priv_dev->out_mem_is_allocated = 0;
2938 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2940 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2941 priv_ep = ep_to_cdns3_ep(ep);
2942 bEndpointAddress = priv_ep->num | priv_ep->dir;
2943 cdns3_select_ep(priv_dev, bEndpointAddress);
2944 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2945 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2946 !(val & EP_CMD_EPRST), 1, 100);
2948 priv_ep->flags &= ~EP_CLAIMED;
2951 /* disable interrupt for device */
2952 writel(0, &priv_dev->regs->usb_ien);
2953 writel(0, &priv_dev->regs->usb_pwr);
2954 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2959 static const struct usb_gadget_ops cdns3_gadget_ops = {
2960 .get_frame = cdns3_gadget_get_frame,
2961 .wakeup = cdns3_gadget_wakeup,
2962 .set_selfpowered = cdns3_gadget_set_selfpowered,
2963 .pullup = cdns3_gadget_pullup,
2964 .udc_start = cdns3_gadget_udc_start,
2965 .udc_stop = cdns3_gadget_udc_stop,
2966 .match_ep = cdns3_gadget_match_ep,
2969 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2973 /* ep0 OUT point to ep0 IN. */
2974 priv_dev->eps[16] = NULL;
2976 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2977 if (priv_dev->eps[i]) {
2978 cdns3_free_trb_pool(priv_dev->eps[i]);
2979 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2984 * cdns3_init_eps Initializes software endpoints of gadget
2985 * @priv_dev: extended gadget object
2987 * Returns 0 on success, error code elsewhere
2989 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2991 u32 ep_enabled_reg, iso_ep_reg;
2992 struct cdns3_endpoint *priv_ep;
2993 int ep_dir, ep_number;
2998 /* Read it from USB_CAP3 to USB_CAP5 */
2999 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3000 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3002 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3004 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3005 ep_dir = i >> 4; /* i div 16 */
3006 ep_number = i & 0xF; /* i % 16 */
3009 if (!(ep_enabled_reg & ep_mask))
3012 if (ep_dir && !ep_number) {
3013 priv_dev->eps[i] = priv_dev->eps[0];
3017 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3022 /* set parent of endpoint object */
3023 priv_ep->cdns3_dev = priv_dev;
3024 priv_dev->eps[i] = priv_ep;
3025 priv_ep->num = ep_number;
3026 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3029 ret = cdns3_init_ep0(priv_dev, priv_ep);
3031 dev_err(priv_dev->dev, "Failed to init ep0\n");
3035 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3036 ep_number, !!ep_dir ? "in" : "out");
3037 priv_ep->endpoint.name = priv_ep->name;
3039 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3040 CDNS3_EP_MAX_PACKET_LIMIT);
3041 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3042 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3044 priv_ep->endpoint.caps.dir_in = 1;
3046 priv_ep->endpoint.caps.dir_out = 1;
3048 if (iso_ep_reg & ep_mask)
3049 priv_ep->endpoint.caps.type_iso = 1;
3051 priv_ep->endpoint.caps.type_bulk = 1;
3052 priv_ep->endpoint.caps.type_int = 1;
3054 list_add_tail(&priv_ep->endpoint.ep_list,
3055 &priv_dev->gadget.ep_list);
3060 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n",
3062 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3063 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3065 INIT_LIST_HEAD(&priv_ep->pending_req_list);
3066 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3067 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3072 cdns3_free_all_eps(priv_dev);
3076 static void cdns3_gadget_release(struct device *dev)
3078 struct cdns3_device *priv_dev = container_of(dev,
3079 struct cdns3_device, gadget.dev);
3084 void cdns3_gadget_exit(struct cdns3 *cdns)
3086 struct cdns3_device *priv_dev;
3088 priv_dev = cdns->gadget_dev;
3091 pm_runtime_mark_last_busy(cdns->dev);
3092 pm_runtime_put_autosuspend(cdns->dev);
3094 usb_del_gadget(&priv_dev->gadget);
3095 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3097 cdns3_free_all_eps(priv_dev);
3099 while (!list_empty(&priv_dev->aligned_buf_list)) {
3100 struct cdns3_aligned_buf *buf;
3102 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3103 dma_free_coherent(priv_dev->sysdev, buf->size,
3107 list_del(&buf->list);
3111 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3112 priv_dev->setup_dma);
3114 kfree(priv_dev->zlp_buf);
3115 usb_put_gadget(&priv_dev->gadget);
3116 cdns->gadget_dev = NULL;
3117 cdns3_drd_gadget_off(cdns);
3120 static int cdns3_gadget_start(struct cdns3 *cdns)
3122 struct cdns3_device *priv_dev;
3126 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3130 usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3131 cdns3_gadget_release);
3132 cdns->gadget_dev = priv_dev;
3133 priv_dev->sysdev = cdns->dev;
3134 priv_dev->dev = cdns->dev;
3135 priv_dev->regs = cdns->dev_regs;
3137 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3138 &priv_dev->onchip_buffers);
3140 if (priv_dev->onchip_buffers <= 0) {
3141 u32 reg = readl(&priv_dev->regs->usb_cap2);
3143 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3146 if (!priv_dev->onchip_buffers)
3147 priv_dev->onchip_buffers = 256;
3149 max_speed = usb_get_maximum_speed(cdns->dev);
3151 /* Check the maximum_speed parameter */
3152 switch (max_speed) {
3153 case USB_SPEED_FULL:
3154 case USB_SPEED_HIGH:
3155 case USB_SPEED_SUPER:
3158 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3161 case USB_SPEED_UNKNOWN:
3162 /* default to superspeed */
3163 max_speed = USB_SPEED_SUPER;
3167 /* fill gadget fields */
3168 priv_dev->gadget.max_speed = max_speed;
3169 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3170 priv_dev->gadget.ops = &cdns3_gadget_ops;
3171 priv_dev->gadget.name = "usb-ss-gadget";
3172 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3173 priv_dev->gadget.irq = cdns->dev_irq;
3175 spin_lock_init(&priv_dev->lock);
3176 INIT_WORK(&priv_dev->pending_status_wq,
3177 cdns3_pending_setup_status_handler);
3179 INIT_WORK(&priv_dev->aligned_buf_wq,
3180 cdns3_free_aligned_request_buf);
3182 /* initialize endpoint container */
3183 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3184 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3186 ret = cdns3_init_eps(priv_dev);
3188 dev_err(priv_dev->dev, "Failed to create endpoints\n");
3192 /* allocate memory for setup packet buffer */
3193 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3194 &priv_dev->setup_dma, GFP_DMA);
3195 if (!priv_dev->setup_buf) {
3200 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3202 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3203 readl(&priv_dev->regs->usb_cap6));
3204 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3205 readl(&priv_dev->regs->usb_cap1));
3206 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3207 readl(&priv_dev->regs->usb_cap2));
3209 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3210 if (priv_dev->dev_ver >= DEV_VER_V2)
3211 priv_dev->gadget.sg_supported = 1;
3213 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3214 if (!priv_dev->zlp_buf) {
3219 /* add USB gadget device */
3220 ret = usb_add_gadget(&priv_dev->gadget);
3222 dev_err(priv_dev->dev, "Failed to add gadget\n");
3228 kfree(priv_dev->zlp_buf);
3230 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3231 priv_dev->setup_dma);
3233 cdns3_free_all_eps(priv_dev);
3235 usb_put_gadget(&priv_dev->gadget);
3236 cdns->gadget_dev = NULL;
3240 static int __cdns3_gadget_init(struct cdns3 *cdns)
3244 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3245 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3247 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3251 cdns3_drd_gadget_on(cdns);
3252 pm_runtime_get_sync(cdns->dev);
3254 ret = cdns3_gadget_start(cdns);
3259 * Because interrupt line can be shared with other components in
3260 * driver it can't use IRQF_ONESHOT flag here.
3262 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3263 cdns3_device_irq_handler,
3264 cdns3_device_thread_irq_handler,
3265 IRQF_SHARED, dev_name(cdns->dev),
3273 cdns3_gadget_exit(cdns);
3277 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
3278 __must_hold(&cdns->lock)
3280 struct cdns3_device *priv_dev = cdns->gadget_dev;
3282 spin_unlock(&cdns->lock);
3283 cdns3_disconnect_gadget(priv_dev);
3284 spin_lock(&cdns->lock);
3286 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3287 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3288 cdns3_hw_reset_eps_config(priv_dev);
3290 /* disable interrupt for device */
3291 writel(0, &priv_dev->regs->usb_ien);
3296 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
3298 struct cdns3_device *priv_dev = cdns->gadget_dev;
3300 if (!priv_dev->gadget_driver)
3303 cdns3_gadget_config(priv_dev);
3309 * cdns3_gadget_init - initialize device structure
3311 * @cdns: cdns3 instance
3313 * This function initializes the gadget.
3315 int cdns3_gadget_init(struct cdns3 *cdns)
3317 struct cdns3_role_driver *rdrv;
3319 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3323 rdrv->start = __cdns3_gadget_init;
3324 rdrv->stop = cdns3_gadget_exit;
3325 rdrv->suspend = cdns3_gadget_suspend;
3326 rdrv->resume = cdns3_gadget_resume;
3327 rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
3328 rdrv->name = "gadget";
3329 cdns->roles[USB_ROLE_DEVICE] = rdrv;