usb:cdns3 Add Cadence USB3 DRD Driver

This patch introduce new Cadence USBSS DRD driver to Linux kernel.

The Cadence USBSS DRD Controller is a highly configurable IP Core which
can be instantiated as Dual-Role Device (DRD), Peripheral Only and
Host Only (XHCI)configurations.

The current driver has been validated with FPGA platform. We have
support for PCIe bus, which is used on FPGA prototyping.

The host side of USBSS-DRD controller is compliant with XHCI
specification, so it works with standard XHCI Linux driver.

Signed-off-by: Pawel Laszczak <pawell@cadence.com>
Signed-off-by: Felipe Balbi <felipe.balbi@linux.intel.com>

1 files changed, 2319 insertions, 0 deletions

diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
new file mode 100644
index 000000000000..4e9e8e43f634
--- /dev/null
+++ b/drivers/usb/cdns3/gadget.c
@@ -0,0 +1,2319 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver - gadget side.
+ *
+ * Copyright (C) 2018-2019 Cadence Design Systems.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Pawel Jez <pjez@cadence.com>,
+ *          Pawel Laszczak <pawell@cadence.com>
+ *          Peter Chen <peter.chen@nxp.com>
+ */
+
+/*
+ * Work around 1:
+ * At some situations, the controller may get stale data address in TRB
+ * at below sequences:
+ * 1. Controller read TRB includes data address
+ * 2. Software updates TRBs includes data address and Cycle bit
+ * 3. Controller read TRB which includes Cycle bit
+ * 4. DMA run with stale data address
+ *
+ * To fix this problem, driver needs to make the first TRB in TD as invalid.
+ * After preparing all TRBs driver needs to check the position of DMA and
+ * if the DMA point to the first just added TRB and doorbell is 1,
+ * then driver must defer making this TRB as valid. This TRB will be make
+ * as valid during adding next TRB only if DMA is stopped or at TRBERR
+ * interrupt.
+ *
+ * Issue has been fixed in DEV_VER_V3 version of controller.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/usb/gadget.h>
+#include <linux/module.h>
+
+#include "core.h"
+#include "gadget-export.h"
+#include "gadget.h"
+#include "trace.h"
+#include "drd.h"
+
+static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
+				   struct usb_request *request,
+				   gfp_t gfp_flags);
+
+/**
+ * cdns3_set_register_bit - set bit in given register.
+ * @ptr: address of device controller register to be read and changed
+ * @mask: bits requested to set
+ */
+void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
+{
+	mask = readl(ptr) | mask;
+	writel(mask, ptr);
+}
+
+/**
+ * cdns3_ep_addr_to_index - Macro converts endpoint address to
+ * index of endpoint object in cdns3_device.eps[] container
+ * @ep_addr: endpoint address for which endpoint object is required
+ *
+ */
+u8 cdns3_ep_addr_to_index(u8 ep_addr)
+{
+	return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
+}
+
+static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
+			     struct cdns3_endpoint *priv_ep)
+{
+	int dma_index;
+
+	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
+
+	return dma_index / TRB_SIZE;
+}
+
+/**
+ * cdns3_next_request - returns next request from list
+ * @list: list containing requests
+ *
+ * Returns request or NULL if no requests in list
+ */
+struct usb_request *cdns3_next_request(struct list_head *list)
+{
+	return list_first_entry_or_null(list, struct usb_request, list);
+}
+
+/**
+ * cdns3_next_align_buf - returns next buffer from list
+ * @list: list containing buffers
+ *
+ * Returns buffer or NULL if no buffers in list
+ */
+struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
+{
+	return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
+}
+
+/**
+ * select_ep - selects endpoint
+ * @priv_dev:  extended gadget object
+ * @ep: endpoint address
+ */
+void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
+{
+	if (priv_dev->selected_ep == ep)
+		return;
+
+	priv_dev->selected_ep = ep;
+	writel(ep, &priv_dev->regs->ep_sel);
+}
+
+dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
+				 struct cdns3_trb *trb)
+{
+	u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
+
+	return priv_ep->trb_pool_dma + offset;
+}
+
+int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
+{
+	switch (priv_ep->type) {
+	case USB_ENDPOINT_XFER_ISOC:
+		return TRB_ISO_RING_SIZE;
+	case USB_ENDPOINT_XFER_CONTROL:
+		return TRB_CTRL_RING_SIZE;
+	default:
+		return TRB_RING_SIZE;
+	}
+}
+
+/**
+ * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
+ * @priv_ep:  endpoint object
+ *
+ * Function will return 0 on success or -ENOMEM on allocation error
+ */
+int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	int ring_size = cdns3_ring_size(priv_ep);
+	struct cdns3_trb *link_trb;
+
+	if (!priv_ep->trb_pool) {
+		priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
+						       ring_size,
+						       &priv_ep->trb_pool_dma,
+						       GFP_DMA32 | GFP_ATOMIC);
+		if (!priv_ep->trb_pool)
+			return -ENOMEM;
+	} else {
+		memset(priv_ep->trb_pool, 0, ring_size);
+	}
+
+	if (!priv_ep->num)
+		return 0;
+
+	priv_ep->num_trbs = ring_size / TRB_SIZE;
+	/* Initialize the last TRB as Link TRB */
+	link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
+	link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
+	link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
+
+	return 0;
+}
+
+static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	if (priv_ep->trb_pool) {
+		dma_free_coherent(priv_dev->sysdev,
+				  cdns3_ring_size(priv_ep),
+				  priv_ep->trb_pool, priv_ep->trb_pool_dma);
+		priv_ep->trb_pool = NULL;
+	}
+}
+
+/**
+ * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
+ * @priv_ep: endpoint object
+ *
+ * Endpoint must be selected before call to this function
+ */
+static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	cdns3_dbg(priv_ep->cdns3_dev, "Stall & flush endpoint %s\n",
+		  priv_ep->name);
+
+	writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
+	       &priv_dev->regs->ep_cmd);
+
+	/* wait for DFLUSH cleared */
+	cdns3_handshake(&priv_dev->regs->ep_cmd, EP_CMD_DFLUSH, 0, 1000);
+	priv_ep->flags |= EP_STALL;
+}
+
+/**
+ * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
+ * @priv_dev: extended gadget object
+ */
+void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
+{
+	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
+
+	cdns3_allow_enable_l1(priv_dev, 0);
+	priv_dev->hw_configured_flag = 0;
+	priv_dev->onchip_used_size = 0;
+	priv_dev->out_mem_is_allocated = 0;
+	priv_dev->wait_for_setup = 0;
+}
+
+/**
+ * cdns3_ep_inc_trb - increment a trb index.
+ * @index: Pointer to the TRB index to increment.
+ * @cs: Cycle state
+ * @trb_in_seg: number of TRBs in segment
+ *
+ * The index should never point to the link TRB. After incrementing,
+ * if it is point to the link TRB, wrap around to the beginning and revert
+ * cycle state bit The
+ * link TRB is always at the last TRB entry.
+ */
+static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
+{
+	(*index)++;
+	if (*index == (trb_in_seg - 1)) {
+		*index = 0;
+		*cs ^=  1;
+	}
+}
+
+/**
+ * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
+ * @priv_ep: The endpoint whose enqueue pointer we're incrementing
+ */
+static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
+{
+	priv_ep->free_trbs--;
+	cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
+}
+
+/**
+ * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
+ * @priv_ep: The endpoint whose dequeue pointer we're incrementing
+ */
+static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
+{
+	priv_ep->free_trbs++;
+	cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
+}
+
+void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
+{
+	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+	int current_trb = priv_req->start_trb;
+
+	while (current_trb != priv_req->end_trb) {
+		cdns3_ep_inc_deq(priv_ep);
+		current_trb = priv_ep->dequeue;
+	}
+
+	cdns3_ep_inc_deq(priv_ep);
+}
+
+/**
+ * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
+ * @priv_dev: Extended gadget object
+ * @enable: Enable/disable permit to transition to L1.
+ *
+ * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
+ * then controller answer with ACK handshake.
+ * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
+ * then controller answer with NYET handshake.
+ */
+void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
+{
+	if (enable)
+		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
+	else
+		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
+}
+
+enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
+{
+	u32 reg;
+
+	reg = readl(&priv_dev->regs->usb_sts);
+
+	if (DEV_SUPERSPEED(reg))
+		return USB_SPEED_SUPER;
+	else if (DEV_HIGHSPEED(reg))
+		return USB_SPEED_HIGH;
+	else if (DEV_FULLSPEED(reg))
+		return USB_SPEED_FULL;
+	else if (DEV_LOWSPEED(reg))
+		return USB_SPEED_LOW;
+	return USB_SPEED_UNKNOWN;
+}
+
+/**
+ * cdns3_start_all_request - add to ring all request not started
+ * @priv_dev: Extended gadget object
+ * @priv_ep: The endpoint for whom request will be started.
+ *
+ * Returns return ENOMEM if transfer ring i not enough TRBs to start
+ *         all requests.
+ */
+static int cdns3_start_all_request(struct cdns3_device *priv_dev,
+				   struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_request *priv_req;
+	struct usb_request *request;
+	int ret = 0;
+
+	while (!list_empty(&priv_ep->deferred_req_list)) {
+		request = cdns3_next_request(&priv_ep->deferred_req_list);
+		priv_req = to_cdns3_request(request);
+
+		ret = cdns3_ep_run_transfer(priv_ep, request);
+		if (ret)
+			return ret;
+
+		list_del(&request->list);
+		list_add_tail(&request->list,
+			      &priv_ep->pending_req_list);
+	}
+
+	priv_ep->flags &= ~EP_RING_FULL;
+	return ret;
+}
+
+/**
+ * cdns3_gadget_giveback - call struct usb_request's ->complete callback
+ * @priv_ep: The endpoint to whom the request belongs to
+ * @priv_req: The request we're giving back
+ * @status: completion code for the request
+ *
+ * Must be called with controller's lock held and interrupts disabled. This
+ * function will unmap @req and call its ->complete() callback to notify upper
+ * layers that it has completed.
+ */
+void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
+			   struct cdns3_request *priv_req,
+			   int status)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct usb_request *request = &priv_req->request;
+
+	list_del_init(&request->list);
+
+	if (request->status == -EINPROGRESS)
+		request->status = status;
+
+	usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
+					priv_ep->dir);
+
+	if ((priv_req->flags & REQUEST_UNALIGNED) &&
+	    priv_ep->dir == USB_DIR_OUT && !request->status)
+		memcpy(request->buf, priv_req->aligned_buf->buf,
+		       request->length);
+
+	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
+	trace_cdns3_gadget_giveback(priv_req);
+
+	if (request->complete) {
+		spin_unlock(&priv_dev->lock);
+		usb_gadget_giveback_request(&priv_ep->endpoint,
+					    request);
+		spin_lock(&priv_dev->lock);
+	}
+
+	if (request->buf == priv_dev->zlp_buf)
+		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
+}
+
+void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	/* Work around for stale data address in TRB*/
+	if (priv_ep->wa1_set) {
+		cdns3_dbg(priv_dev, "WA1: update cycle bit\n");
+		priv_ep->wa1_set = 0;
+		priv_ep->wa1_trb_index = 0xFFFF;
+		if (priv_ep->wa1_cycle_bit) {
+			priv_ep->wa1_trb->control =
+				priv_ep->wa1_trb->control | 0x1;
+		} else {
+			priv_ep->wa1_trb->control =
+				priv_ep->wa1_trb->control & ~0x1;
+		}
+	}
+}
+
+static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
+{
+	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_aligned_buf *buf;
+
+	/* check if buffer is aligned to 8. */
+	if (!((uintptr_t)priv_req->request.buf & 0x7))
+		return 0;
+
+	buf = priv_req->aligned_buf;
+
+	if (!buf || priv_req->request.length > buf->size) {
+		buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
+		if (!buf)
+			return -ENOMEM;
+
+		buf->size = priv_req->request.length;
+
+		buf->buf = dma_alloc_coherent(priv_dev->sysdev,
+					      buf->size,
+					      &buf->dma,
+					      GFP_ATOMIC);
+		if (!buf->buf) {
+			kfree(buf);
+			return -ENOMEM;
+		}
+
+		if (priv_req->aligned_buf) {
+			trace_cdns3_free_aligned_request(priv_req);
+			priv_req->aligned_buf->in_use = 0;
+			priv_dev->run_garbage_colector = 1;
+		}
+
+		buf->in_use = 1;
+		priv_req->aligned_buf = buf;
+
+		list_add_tail(&buf->list,
+			      &priv_dev->aligned_buf_list);
+	}
+
+	if (priv_ep->dir == USB_DIR_IN) {
+		memcpy(buf->buf, priv_req->request.buf,
+		       priv_req->request.length);
+	}
+
+	priv_req->flags |= REQUEST_UNALIGNED;
+	trace_cdns3_prepare_aligned_request(priv_req);
+
+	return 0;
+}
+
+static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
+				  struct cdns3_trb *trb)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	if (!priv_ep->wa1_set) {
+		u32 doorbell;
+
+		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+		if (doorbell) {
+			priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
+			priv_ep->wa1_set = 1;
+			priv_ep->wa1_trb = trb;
+			priv_ep->wa1_trb_index = priv_ep->enqueue;
+			cdns3_dbg(priv_dev, "WA1 set guard\n");
+			return 0;
+		}
+	}
+	return 1;
+}
+
+static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
+					     struct cdns3_endpoint *priv_ep)
+{
+	int dma_index;
+	u32 doorbell;
+
+	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+	if (!doorbell || dma_index != priv_ep->wa1_trb_index)
+		cdns3_wa1_restore_cycle_bit(priv_ep);
+}
+
+/**
+ * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
+ * @priv_ep: endpoint object
+ *
+ * Returns zero on success or negative value on failure
+ */
+int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
+			  struct usb_request *request)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_request *priv_req;
+	struct cdns3_trb *trb;
+	dma_addr_t trb_dma;
+	u32 togle_pcs = 1;
+	int sg_iter = 0;
+	int num_trb;
+	int address;
+	u32 control;
+	int pcs;
+
+	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
+		num_trb = priv_ep->interval;
+	else
+		num_trb = request->num_sgs ? request->num_sgs : 1;
+
+	if (num_trb > priv_ep->free_trbs) {
+		priv_ep->flags |= EP_RING_FULL;
+		return -ENOBUFS;
+	}
+
+	priv_req = to_cdns3_request(request);
+	address = priv_ep->endpoint.desc->bEndpointAddress;
+
+	priv_ep->flags |= EP_PENDING_REQUEST;
+
+	/* must allocate buffer aligned to 8 */
+	if (priv_req->flags & REQUEST_UNALIGNED)
+		trb_dma = priv_req->aligned_buf->dma;
+	else
+		trb_dma = request->dma;
+
+	trb = priv_ep->trb_pool + priv_ep->enqueue;
+	priv_req->start_trb = priv_ep->enqueue;
+	priv_req->trb = trb;
+
+	cdns3_select_ep(priv_ep->cdns3_dev, address);
+
+	/* prepare ring */
+	if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
+		struct cdns3_trb *link_trb;
+		int doorbell, dma_index;
+		u32 ch_bit = 0;
+
+		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+		/* Driver can't update LINK TRB if it is current processed. */
+		if (doorbell && dma_index == priv_ep->num_trbs - 1) {
+			priv_ep->flags |= EP_DEFERRED_DRDY;
+			return -ENOBUFS;
+		}
+
+		/*updating C bt in  Link TRB before starting DMA*/
+		link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
+		/*
+		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
+		 * that DMA stuck at the LINK TRB.
+		 * On the other hand, removing TRB_CHAIN for longer TRs for
+		 * epXout cause that DMA stuck after handling LINK TRB.
+		 * To eliminate this strange behavioral driver set TRB_CHAIN
+		 * bit only for TR size > 2.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
+		    TRBS_PER_SEGMENT > 2)
+			ch_bit = TRB_CHAIN;
+
+		link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
+				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
+	}
+
+	if (priv_dev->dev_ver <= DEV_VER_V2)
+		togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
+
+	/* set incorrect Cycle Bit for first trb*/
+	control = priv_ep->pcs ? 0 : TRB_CYCLE;
+
+	do {
+		u32 length;
+		u16 td_size = 0;
+
+		/* fill TRB */
+		control |= TRB_TYPE(TRB_NORMAL);
+		trb->buffer = TRB_BUFFER(request->num_sgs == 0
+				? trb_dma : request->sg[sg_iter].dma_address);
+
+		if (likely(!request->num_sgs))
+			length = request->length;
+		else
+			length = request->sg[sg_iter].length;
+
+		if (likely(priv_dev->dev_ver >= DEV_VER_V2))
+			td_size = DIV_ROUND_UP(length,
+					       priv_ep->endpoint.maxpacket);
+
+		trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
+					TRB_LEN(length);
+		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
+			trb->length |= TRB_TDL_SS_SIZE(td_size);
+		else
+			control |= TRB_TDL_HS_SIZE(td_size);
+
+		pcs = priv_ep->pcs ? TRB_CYCLE : 0;
+
+		/*
+		 * first trb should be prepared as last to avoid processing
+		 *  transfer to early
+		 */
+		if (sg_iter != 0)
+			control |= pcs;
+
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
+			control |= TRB_IOC | TRB_ISP;
+		} else {
+			/* for last element in TD or in SG list */
+			if (sg_iter == (num_trb - 1) && sg_iter != 0)
+				control |= pcs | TRB_IOC | TRB_ISP;
+		}
+
+		if (sg_iter)
+			trb->control = control;
+		else
+			priv_req->trb->control = control;
+
+		control = 0;
+		++sg_iter;
+		priv_req->end_trb = priv_ep->enqueue;
+		cdns3_ep_inc_enq(priv_ep);
+		trb = priv_ep->trb_pool + priv_ep->enqueue;
+	} while (sg_iter < num_trb);
+
+	trb = priv_req->trb;
+
+	priv_req->flags |= REQUEST_PENDING;
+
+	if (sg_iter == 1)
+		trb->control |= TRB_IOC | TRB_ISP;
+	/*
+	 * Memory barrier - cycle bit must be set before other filds in trb.
+	 */
+	wmb();
+
+	/* give the TD to the consumer*/
+	if (togle_pcs)
+		trb->control =  trb->control ^ 1;
+
+	if (priv_dev->dev_ver <= DEV_VER_V2)
+		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
+
+	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
+
+	/*
+	 * Memory barrier - Cycle Bit must be set before trb->length  and
+	 * trb->buffer fields.
+	 */
+	wmb();
+
+	/*
+	 * For DMULT mode we can set address to transfer ring only once after
+	 * enabling endpoint.
+	 */
+	if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
+		/*
+		 * Until SW is not ready to handle the OUT transfer the ISO OUT
+		 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
+		 * EP_CFG_ENABLE must be set before updating ep_traddr.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
+		    !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
+			priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
+			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
+					       EP_CFG_ENABLE);
+		}
+
+		writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
+					priv_req->start_trb * TRB_SIZE),
+		       &priv_dev->regs->ep_traddr);
+
+		cdns3_dbg(priv_ep->cdns3_dev, "Update ep_trbaddr for %s to %08x\n",
+			  priv_ep->name, readl(&priv_dev->regs->ep_traddr));
+
+		priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
+	}
+
+	if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALL)) {
+		trace_cdns3_ring(priv_ep);
+		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
+		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
+		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+		trace_cdns3_doorbell_epx(priv_ep->name,
+					 readl(&priv_dev->regs->ep_traddr));
+	}
+
+	/* WORKAROUND for transition to L0 */
+	__cdns3_gadget_wakeup(priv_dev);
+
+	return 0;
+}
+
+void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
+{
+	struct cdns3_endpoint *priv_ep;
+	struct usb_ep *ep;
+	int result = 0;
+
+	if (priv_dev->hw_configured_flag)
+		return;
+
+	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
+	writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
+
+	cdns3_set_register_bit(&priv_dev->regs->usb_conf,
+			       USB_CONF_U1EN | USB_CONF_U2EN);
+
+	/* wait until configuration set */
+	result = cdns3_handshake(&priv_dev->regs->usb_sts,
+				 USB_STS_CFGSTS_MASK, 1, 100);
+
+	priv_dev->hw_configured_flag = 1;
+
+	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+		if (ep->enabled) {
+			priv_ep = ep_to_cdns3_ep(ep);
+			cdns3_start_all_request(priv_dev, priv_ep);
+		}
+	}
+}
+
+/**
+ * cdns3_request_handled - check whether request has been handled by DMA
+ *
+ * @priv_ep: extended endpoint object.
+ * @priv_req: request object for checking
+ *
+ * Endpoint must be selected before invoking this function.
+ *
+ * Returns false if request has not been handled by DMA, else returns true.
+ *
+ * SR - start ring
+ * ER -  end ring
+ * DQ = priv_ep->dequeue - dequeue position
+ * EQ = priv_ep->enqueue -  enqueue position
+ * ST = priv_req->start_trb - index of first TRB in transfer ring
+ * ET = priv_req->end_trb - index of last TRB in transfer ring
+ * CI = current_index - index of processed TRB by DMA.
+ *
+ * As first step, function checks if cycle bit for priv_req->start_trb is
+ * correct.
+ *
+ * some rules:
+ * 1. priv_ep->dequeue never exceed current_index.
+ * 2  priv_ep->enqueue never exceed priv_ep->dequeue
+ * 3. exception: priv_ep->enqueue == priv_ep->dequeue
+ *    and priv_ep->free_trbs is zero.
+ *    This case indicate that TR is full.
+ *
+ * Then We can split recognition into two parts:
+ * Case 1 - priv_ep->dequeue < current_index
+ *      SR ... EQ ... DQ ... CI ... ER
+ *      SR ... DQ ... CI ... EQ ... ER
+ *
+ *      Request has been handled by DMA if ST and ET is between DQ and CI.
+ *
+ * Case 2 - priv_ep->dequeue > current_index
+ * This situation take place when CI go through the LINK TRB at the end of
+ * transfer ring.
+ *      SR ... CI ... EQ ... DQ ... ER
+ *
+ *      Request has been handled by DMA if ET is less then CI or
+ *      ET is greater or equal DQ.
+ */
+static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
+				  struct cdns3_request *priv_req)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	struct cdns3_trb *trb = priv_req->trb;
+	int current_index = 0;
+	int handled = 0;
+	int doorbell;
+
+	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+	trb = &priv_ep->trb_pool[priv_req->start_trb];
+
+	if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
+		goto finish;
+
+	if (doorbell == 1 && current_index == priv_ep->dequeue)
+		goto finish;
+
+	/* The corner case for TRBS_PER_SEGMENT equal 2). */
+	if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
+		handled = 1;
+		goto finish;
+	}
+
+	if (priv_ep->enqueue == priv_ep->dequeue &&
+	    priv_ep->free_trbs == 0) {
+		handled = 1;
+	} else if (priv_ep->dequeue < current_index) {
+		if ((current_index == (priv_ep->num_trbs - 1)) &&
+		    !priv_ep->dequeue)
+			goto finish;
+
+		if (priv_req->end_trb >= priv_ep->dequeue &&
+		    priv_req->end_trb < current_index)
+			handled = 1;
+	} else if (priv_ep->dequeue  > current_index) {
+		if (priv_req->end_trb  < current_index ||
+		    priv_req->end_trb >= priv_ep->dequeue)
+			handled = 1;
+	}
+
+finish:
+	trace_cdns3_request_handled(priv_req, current_index, handled);
+
+	return handled;
+}
+
+static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
+				     struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_request *priv_req;
+	struct usb_request *request;
+	struct cdns3_trb *trb;
+
+	while (!list_empty(&priv_ep->pending_req_list)) {
+		request = cdns3_next_request(&priv_ep->pending_req_list);
+		priv_req = to_cdns3_request(request);
+
+		/* Re-select endpoint. It could be changed by other CPU during
+		 * handling usb_gadget_giveback_request.
+		 */
+		cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+
+		if (!cdns3_request_handled(priv_ep, priv_req))
+			goto prepare_next_td;
+
+		trb = priv_ep->trb_pool + priv_ep->dequeue;
+		trace_cdns3_complete_trb(priv_ep, trb);
+
+		if (trb != priv_req->trb)
+			dev_warn(priv_dev->dev,
+				 "request_trb=0x%p, queue_trb=0x%p\n",
+				 priv_req->trb, trb);
+
+		request->actual = TRB_LEN(le32_to_cpu(trb->length));
+		cdns3_move_deq_to_next_trb(priv_req);
+		cdns3_gadget_giveback(priv_ep, priv_req, 0);
+
+		if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
+		    TRBS_PER_SEGMENT == 2)
+			break;
+	}
+	priv_ep->flags &= ~EP_PENDING_REQUEST;
+
+prepare_next_td:
+	cdns3_start_all_request(priv_dev, priv_ep);
+}
+
+void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+	cdns3_wa1_restore_cycle_bit(priv_ep);
+
+	if (rearm) {
+		trace_cdns3_ring(priv_ep);
+
+		/* Cycle Bit must be updated before arming DMA. */
+		wmb();
+		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+
+		__cdns3_gadget_wakeup(priv_dev);
+
+		trace_cdns3_doorbell_epx(priv_ep->name,
+					 readl(&priv_dev->regs->ep_traddr));
+	}
+}
+
+/**
+ * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
+ * @priv_ep: endpoint object
+ *
+ * Returns 0
+ */
+static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
+{
+	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+	u32 ep_sts_reg;
+
+	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+
+	trace_cdns3_epx_irq(priv_dev, priv_ep);
+
+	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+	writel(ep_sts_reg, &priv_dev->regs->ep_sts);
+
+	if (ep_sts_reg & EP_STS_TRBERR) {
+		/*
+		 * For isochronous transfer driver completes request on
+		 * IOC or on TRBERR. IOC appears only when device receive
+		 * OUT data packet. If host disable stream or lost some packet
+		 * then the only way to finish all queued transfer is to do it
+		 * on TRBERR event.
+		 */
+		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
+		    !priv_ep->wa1_set) {
+			if (!priv_ep->dir) {
+				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
+
+				ep_cfg &= ~EP_CFG_ENABLE;
+				writel(ep_cfg, &priv_dev->regs->ep_cfg);
+				priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
+			}
+			cdns3_transfer_completed(priv_dev, priv_ep);
+		} else {
+			if (priv_ep->flags & EP_DEFERRED_DRDY) {
+				priv_ep->flags &= ~EP_DEFERRED_DRDY;
+				cdns3_start_all_request(priv_dev, priv_ep);
+			} else {
+				cdns3_rearm_transfer(priv_ep, priv_ep->wa1_set);
+			}
+		}
+	}
+
+	if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP))
+		cdns3_transfer_completed(priv_dev, priv_ep);
+
+	return 0;
+}
+
+/**
+ * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
+ * @priv_dev: extended gadget object
+ * @usb_ists: bitmap representation of device's reported interrupts
+ * (usb_ists register value)
+ */
+static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
+					      u32 usb_ists)
+{
+	int speed = 0;
+
+	trace_cdns3_usb_irq(priv_dev, usb_ists);
+	if (usb_ists & USB_ISTS_L1ENTI) {
+		/*
+		 * WORKAROUND: CDNS3 controller has issue with hardware resuming
+		 * from L1. To fix it, if any DMA transfer is pending driver
+		 * must starts driving resume signal immediately.
+		 */
+		if (readl(&priv_dev->regs->drbl))
+			__cdns3_gadget_wakeup(priv_dev);
+	}
+
+	/* Connection detected */
+	if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
+		speed = cdns3_get_speed(priv_dev);
+		priv_dev->gadget.speed = speed;
+		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
+		cdns3_ep0_config(priv_dev);
+	}
+
+	/* Disconnection detected */
+	if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
+		if (priv_dev->gadget_driver &&
+		    priv_dev->gadget_driver->disconnect) {
+			spin_unlock(&priv_dev->lock);
+			priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
+			spin_lock(&priv_dev->lock);
+		}
+
+		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
+		cdns3_hw_reset_eps_config(priv_dev);
+	}
+
+	if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
+		if (priv_dev->gadget_driver &&
+		    priv_dev->gadget_driver->suspend) {
+			spin_unlock(&priv_dev->lock);
+			priv_dev->gadget_driver->suspend(&priv_dev->gadget);
+			spin_lock(&priv_dev->lock);
+		}
+	}
+
+	if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
+		if (priv_dev->gadget_driver &&
+		    priv_dev->gadget_driver->resume) {
+			spin_unlock(&priv_dev->lock);
+			priv_dev->gadget_driver->resume(&priv_dev->gadget);
+			spin_lock(&priv_dev->lock);
+		}
+	}
+
+	/* reset*/
+	if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
+		if (priv_dev->gadget_driver) {
+			spin_unlock(&priv_dev->lock);
+			usb_gadget_udc_reset(&priv_dev->gadget,
+					     priv_dev->gadget_driver);
+			spin_lock(&priv_dev->lock);
+
+			/*read again to check the actual speed*/
+			speed = cdns3_get_speed(priv_dev);
+			priv_dev->gadget.speed = speed;
+			cdns3_hw_reset_eps_config(priv_dev);
+			cdns3_ep0_config(priv_dev);
+		}
+	}
+}
+
+/**
+ * cdns3_device_irq_handler- interrupt handler for device part of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
+{
+	struct cdns3_device *priv_dev;
+	struct cdns3 *cdns = data;
+	irqreturn_t ret = IRQ_NONE;
+	unsigned long flags;
+	u32 reg;
+
+	priv_dev = cdns->gadget_dev;
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	/* check USB device interrupt */
+	reg = readl(&priv_dev->regs->usb_ists);
+
+	if (reg) {
+		writel(reg, &priv_dev->regs->usb_ists);
+		cdns3_check_usb_interrupt_proceed(priv_dev, reg);
+		ret = IRQ_HANDLED;
+	}
+
+	/* check endpoint interrupt */
+	reg = readl(&priv_dev->regs->ep_ists);
+
+	if (reg) {
+		priv_dev->shadow_ep_en |= reg;
+		reg = ~reg & readl(&priv_dev->regs->ep_ien);
+		/* mask deferred interrupt. */
+		writel(reg, &priv_dev->regs->ep_ien);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	spin_unlock_irqrestore(&priv_dev->lock, flags);
+	return ret;
+}
+
+/**
+ * cdns3_device_thread_irq_handler- interrupt handler for device part
+ * of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
+{
+	struct cdns3_device *priv_dev;
+	struct cdns3 *cdns = data;
+	irqreturn_t ret = IRQ_NONE;
+	unsigned long flags;
+	u32 ep_ien;
+	int bit;
+	u32 reg;
+
+	priv_dev = cdns->gadget_dev;
+	spin_lock_irqsave(&priv_dev->lock, flags);
+
+	reg = readl(&priv_dev->regs->ep_ists);
+
+	/* handle default endpoint OUT */
+	if (reg & EP_ISTS_EP_OUT0) {
+		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
+		ret = IRQ_HANDLED;
+	}
+
+	/* handle default endpoint IN */
+	if (reg & EP_ISTS_EP_IN0) {
+		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
+		ret = IRQ_HANDLED;
+	}
+