path: root/drivers/dma/tegra186-gpc-dma.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * DMA driver for NVIDIA Tegra GPC DMA controller.
 *
 * Copyright (c) 2014-2022, NVIDIA CORPORATION.  All rights reserved.
 */

#include <linux/bitfield.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/minmax.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
#include <linux/reset.h>
#include <linux/slab.h>
#include <dt-bindings/memory/tegra186-mc.h>
#include "virt-dma.h"

/* CSR register */
#define TEGRA_GPCDMA_CHAN_CSR			0x00
#define TEGRA_GPCDMA_CSR_ENB			BIT(31)
#define TEGRA_GPCDMA_CSR_IE_EOC			BIT(30)
#define TEGRA_GPCDMA_CSR_ONCE			BIT(27)

#define TEGRA_GPCDMA_CSR_FC_MODE		GENMASK(25, 24)
#define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)
#define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)
#define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)
#define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)

#define TEGRA_GPCDMA_CSR_DMA			GENMASK(23, 21)
#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)
#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC		\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)
#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC	\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)
#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC		\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)
#define TEGRA_GPCDMA_CSR_DMA_MEM2MEM		\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)
#define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT		\
		FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)

#define TEGRA_GPCDMA_CSR_REQ_SEL_MASK		GENMASK(20, 16)
#define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED		\
					FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)
#define TEGRA_GPCDMA_CSR_IRQ_MASK		BIT(15)
#define TEGRA_GPCDMA_CSR_WEIGHT			GENMASK(13, 10)

/* STATUS register */
#define TEGRA_GPCDMA_CHAN_STATUS		0x004
#define TEGRA_GPCDMA_STATUS_BUSY		BIT(31)
#define TEGRA_GPCDMA_STATUS_ISE_EOC		BIT(30)
#define TEGRA_GPCDMA_STATUS_PING_PONG		BIT(28)
#define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY	BIT(27)
#define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE	BIT(26)
#define TEGRA_GPCDMA_STATUS_CHANNEL_RX		BIT(25)
#define TEGRA_GPCDMA_STATUS_CHANNEL_TX		BIT(24)
#define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA	BIT(23)
#define TEGRA_GPCDMA_STATUS_IRQ_STA		BIT(21)
#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA	BIT(20)

#define TEGRA_GPCDMA_CHAN_CSRE			0x008
#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE		BIT(31)

/* Source address */
#define TEGRA_GPCDMA_CHAN_SRC_PTR		0x00C

/* Destination address */
#define TEGRA_GPCDMA_CHAN_DST_PTR		0x010

/* High address pointer */
#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR		0x014
#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR		GENMASK(7, 0)
#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR		GENMASK(23, 16)

/* MC sequence register */
#define TEGRA_GPCDMA_CHAN_MCSEQ			0x18
#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP		BIT(31)
#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT		GENMASK(30, 25)
#define TEGRA_GPCDMA_MCSEQ_BURST		GENMASK(24, 23)
#define TEGRA_GPCDMA_MCSEQ_BURST_2		\
		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)
#define TEGRA_GPCDMA_MCSEQ_BURST_16		\
		FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)
#define TEGRA_GPCDMA_MCSEQ_WRAP1		GENMASK(22, 20)
#define TEGRA_GPCDMA_MCSEQ_WRAP0		GENMASK(19, 17)
#define TEGRA_GPCDMA_MCSEQ_WRAP_NONE		0

#define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK	GENMASK(13, 7)
#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK	GENMASK(6, 0)

/* MMIO sequence register */
#define TEGRA_GPCDMA_CHAN_MMIOSEQ			0x01c
#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF		BIT(31)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH		GENMASK(30, 28)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8	\
		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16	\
		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)
#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32	\
		FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)
#define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP		BIT(27)
#define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT	23
#define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN		2U
#define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX		32U
#define TEGRA_GPCDMA_MMIOSEQ_BURST(bs)	\
		(GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)
#define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID		GENMASK(22, 19)
#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD		GENMASK(18, 16)
#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT		GENMASK(8, 7)

/* Channel WCOUNT */
#define TEGRA_GPCDMA_CHAN_WCOUNT		0x20

/* Transfer count */
#define TEGRA_GPCDMA_CHAN_XFER_COUNT		0x24

/* DMA byte count status */
#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS	0x28

/* Error Status Register */
#define TEGRA_GPCDMA_CHAN_ERR_STATUS		0x30
#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT	8
#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK	0xF
#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err)	(			\
		((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) &	\
		TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)
#define TEGRA_DMA_BM_FIFO_FULL_ERR		0xF
#define TEGRA_DMA_PERIPH_FIFO_FULL_ERR		0xE
#define TEGRA_DMA_PERIPH_ID_ERR			0xD
#define TEGRA_DMA_STREAM_ID_ERR			0xC
#define TEGRA_DMA_MC_SLAVE_ERR			0xB
#define TEGRA_DMA_MMIO_SLAVE_ERR		0xA

/* Fixed Pattern */
#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN		0x34

#define TEGRA_GPCDMA_CHAN_TZ			0x38
#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1	BIT(0)
#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1		BIT(1)

#define TEGRA_GPCDMA_CHAN_SPARE			0x3c
#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC	BIT(16)

/*
 * If any burst is in flight and DMA paused then this is the time to complete
 * on-flight burst and update DMA status register.
 */
#define TEGRA_GPCDMA_BURST_COMPLETE_TIME	10
#define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT	5000 /* 5 msec */

/* Channel base address offset from GPCDMA base address */
#define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET	0x10000

/* Default channel mask reserving channel0 */
#define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK	0xfffffffe

struct tegra_dma;
struct tegra_dma_channel;

/*
 * tegra_dma_chip_data Tegra chip specific DMA data
 * @nr_channels: Number of channels available in the controller.
 * @channel_reg_size: Channel register size.
 * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
 * @hw_support_pause: DMA HW engine support pause of the channel.
 */
struct tegra_dma_chip_data {
	bool hw_support_pause;
	unsigned int nr_channels;
	unsigned int channel_reg_size;
	unsigned int max_dma_count;
	int (*terminate)(struct tegra_dma_channel *tdc);
};

/* DMA channel registers */
struct tegra_dma_channel_regs {
	u32 csr;
	u32 src_ptr;
	u32 dst_ptr;
	u32 high_addr_ptr;
	u32 mc_seq;
	u32 mmio_seq;
	u32 wcount;
	u32 fixed_pattern;
};

/*
 * tegra_dma_sg_req: DMA request details to configure hardware. This
 * contains the details for one transfer to configure DMA hw.
 * The client's request for data transfer can be broken into multiple
 * sub-transfer as per requester details and hw support. This sub transfer
 * get added as an array in Tegra DMA desc which manages the transfer details.
 */
struct tegra_dma_sg_req {
	unsigned int len;
	struct tegra_dma_channel_regs ch_regs;
};

/*
 * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to
 * manage client request and keep track of transfer status, callbacks
 * and request counts etc.
 */
struct tegra_dma_desc {
	bool