scsi: mpi3mr: Base driver code

Implement basic pci device driver requirements: Device probing, memory
allocation, mapping system registers, allocate irq lines, etc.

Source is managed in mainly three different files:

 - mpi3mr_fw.c:  Common code which interacts with underlying fw/hw.

 - mpi3mr_os.c:  Common code which interacts with SCSI midlayer.

 - mpi3mr_app.c: Common code which interacts with application/ioctl.
		 This is currently work in progress.

Link: https://lore.kernel.org/r/20210520152545.2710479-3-kashyap.desai@broadcom.com
Cc: sathya.prakash@broadcom.com
Cc: bvanassche@acm.org
Reviewed-by: Hannes Reinecke <hare@suse.de>
Reviewed-by: Tomas Henzl <thenzl@redhat.com>
Reviewed-by: Himanshu Madhani <himanshu.madhani@oracle.com>
Signed-off-by: Kashyap Desai <kashyap.desai@broadcom.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

5 files changed, 2761 insertions, 0 deletions

diff --git a/drivers/scsi/mpi3mr/Makefile b/drivers/scsi/mpi3mr/Makefile
new file mode 100644
index 000000000000..7c2063e04c81
--- /dev/null
+++ b/drivers/scsi/mpi3mr/Makefile
@@ -0,0 +1,4 @@
+# mpi3mr makefile
+obj-m += mpi3mr.o
+mpi3mr-y +=  mpi3mr_os.o     \
+		mpi3mr_fw.o \
diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
new file mode 100644
index 000000000000..babb5c52470c
--- /dev/null
+++ b/drivers/scsi/mpi3mr/mpi3mr.h
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Broadcom MPI3 Storage Controllers
+ *
+ * Copyright (C) 2017-2021 Broadcom Inc.
+ *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
+ *
+ */
+
+#ifndef MPI3MR_H_INCLUDED
+#define MPI3MR_H_INCLUDED
+
+#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
+#include <linux/blk-mq-pci.h>
+#include <linux/delay.h>
+#include <linux/dmapool.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/poll.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
+#include <linux/workqueue.h>
+#include <asm/unaligned.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_dbg.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_tcq.h>
+
+#include "mpi/mpi30_transport.h"
+#include "mpi/mpi30_image.h"
+#include "mpi/mpi30_init.h"
+#include "mpi/mpi30_ioc.h"
+#include "mpi3mr_debug.h"
+
+/* Global list and lock for storing multiple adapters managed by the driver */
+extern spinlock_t mrioc_list_lock;
+extern struct list_head mrioc_list;
+
+#define MPI3MR_DRIVER_VERSION	"00.255.45.01"
+#define MPI3MR_DRIVER_RELDATE	"12-December-2020"
+
+#define MPI3MR_DRIVER_NAME	"mpi3mr"
+#define MPI3MR_DRIVER_LICENSE	"GPL"
+#define MPI3MR_DRIVER_AUTHOR	"Broadcom Inc. <mpi3mr-linuxdrv.pdl@broadcom.com>"
+#define MPI3MR_DRIVER_DESC	"MPI3 Storage Controller Device Driver"
+
+#define MPI3MR_NAME_LENGTH	32
+#define IOCNAME			"%s: "
+
+/* Definitions for internal SGL and Chain SGL buffers */
+#define MPI3MR_PAGE_SIZE_4K		4096
+#define MPI3MR_SG_DEPTH		(MPI3MR_PAGE_SIZE_4K / sizeof(struct mpi3_sge_common))
+
+/* Definitions for MAX values for shost */
+#define MPI3MR_MAX_CMDS_LUN	7
+#define MPI3MR_MAX_CDB_LENGTH	32
+
+/* Admin queue management definitions */
+#define MPI3MR_ADMIN_REQ_Q_SIZE		(2 * MPI3MR_PAGE_SIZE_4K)
+#define MPI3MR_ADMIN_REPLY_Q_SIZE	(4 * MPI3MR_PAGE_SIZE_4K)
+#define MPI3MR_ADMIN_REQ_FRAME_SZ	128
+#define MPI3MR_ADMIN_REPLY_FRAME_SZ	16
+
+
+/* Reserved Host Tag definitions */
+#define MPI3MR_HOSTTAG_INVALID		0xFFFF
+#define MPI3MR_HOSTTAG_INITCMDS		1
+#define MPI3MR_HOSTTAG_IOCTLCMDS	2
+#define MPI3MR_HOSTTAG_BLK_TMS		5
+
+#define MPI3MR_NUM_DEVRMCMD		1
+#define MPI3MR_HOSTTAG_DEVRMCMD_MIN	(MPI3MR_HOSTTAG_BLK_TMS + 1)
+#define MPI3MR_HOSTTAG_DEVRMCMD_MAX	(MPI3MR_HOSTTAG_DEVRMCMD_MIN + \
+						MPI3MR_NUM_DEVRMCMD - 1)
+
+#define MPI3MR_INTERNAL_CMDS_RESVD     MPI3MR_HOSTTAG_DEVRMCMD_MAX
+
+/* Reduced resource count definition for crash kernel */
+#define MPI3MR_HOST_IOS_KDUMP		128
+
+/* command/controller interaction timeout definitions in seconds */
+#define MPI3MR_INTADMCMD_TIMEOUT		10
+#define MPI3MR_RESETTM_TIMEOUT			30
+#define MPI3MR_DEFAULT_SHUTDOWN_TIME		120
+
+#define MPI3MR_WATCHDOG_INTERVAL		1000 /* in milli seconds */
+
+/* Internal admin command state definitions*/
+#define MPI3MR_CMD_NOTUSED	0x8000
+#define MPI3MR_CMD_COMPLETE	0x0001
+#define MPI3MR_CMD_PENDING	0x0002
+#define MPI3MR_CMD_REPLY_VALID	0x0004
+#define MPI3MR_CMD_RESET	0x0008
+
+/* Definitions for Event replies and sense buffer allocated per controller */
+#define MPI3MR_NUM_EVT_REPLIES	64
+#define MPI3MR_SENSEBUF_SZ	256
+#define MPI3MR_SENSEBUF_FACTOR	3
+#define MPI3MR_CHAINBUF_FACTOR	3
+
+/* Invalid target device handle */
+#define MPI3MR_INVALID_DEV_HANDLE	0xFFFF
+
+/* Controller Reset related definitions */
+#define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT	5
+#define MPI3MR_MAX_RESET_RETRY_COUNT		3
+
+/* ResponseCode definitions */
+#define MPI3MR_RI_MASK_RESPCODE		(0x000000FF)
+#define MPI3MR_RSP_TM_COMPLETE		0x00
+#define MPI3MR_RSP_INVALID_FRAME	0x02
+#define MPI3MR_RSP_TM_NOT_SUPPORTED	0x04
+#define MPI3MR_RSP_TM_FAILED		0x05
+#define MPI3MR_RSP_TM_SUCCEEDED		0x08
+#define MPI3MR_RSP_TM_INVALID_LUN	0x09
+#define MPI3MR_RSP_TM_OVERLAPPED_TAG	0x0A
+#define MPI3MR_RSP_IO_QUEUED_ON_IOC \
+			MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC
+
+/* SGE Flag definition */
+#define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \
+	(MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \
+	MPI3_SGE_FLAGS_END_OF_LIST)
+
+/* IOC State definitions */
+enum mpi3mr_iocstate {
+	MRIOC_STATE_READY = 1,
+	MRIOC_STATE_RESET,
+	MRIOC_STATE_FAULT,
+	MRIOC_STATE_BECOMING_READY,
+	MRIOC_STATE_RESET_REQUESTED,
+	MRIOC_STATE_UNRECOVERABLE,
+};
+
+/* Reset reason code definitions*/
+enum mpi3mr_reset_reason {
+	MPI3MR_RESET_FROM_BRINGUP = 1,
+	MPI3MR_RESET_FROM_FAULT_WATCH = 2,
+	MPI3MR_RESET_FROM_IOCTL = 3,
+	MPI3MR_RESET_FROM_EH_HOS = 4,
+	MPI3MR_RESET_FROM_TM_TIMEOUT = 5,
+	MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6,
+	MPI3MR_RESET_FROM_MUR_FAILURE = 7,
+	MPI3MR_RESET_FROM_CTLR_CLEANUP = 8,
+	MPI3MR_RESET_FROM_CIACTIV_FAULT = 9,
+	MPI3MR_RESET_FROM_PE_TIMEOUT = 10,
+	MPI3MR_RESET_FROM_TSU_TIMEOUT = 11,
+	MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12,
+	MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13,
+	MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14,
+	MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15,
+	MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16,
+	MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17,
+	MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18,
+	MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19,
+	MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20,
+	MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21,
+};
+
+/**
+ * struct mpi3mr_compimg_ver - replica of component image
+ * version defined in mpi30_image.h in host endianness
+ *
+ */
+struct mpi3mr_compimg_ver {
+	u16 build_num;
+	u16 cust_id;
+	u8 ph_minor;
+	u8 ph_major;
+	u8 gen_minor;
+	u8 gen_major;
+};
+
+/**
+ * struct mpi3mr_ioc_facs - replica of component image version
+ * defined in mpi30_ioc.h in host endianness
+ *
+ */
+struct mpi3mr_ioc_facts {
+	u32 ioc_capabilities;
+	struct mpi3mr_compimg_ver fw_ver;
+	u32 mpi_version;
+	u16 max_reqs;
+	u16 product_id;
+	u16 op_req_sz;
+	u16 reply_sz;
+	u16 exceptions;
+	u16 max_perids;
+	u16 max_pds;
+	u16 max_sasexpanders;
+	u16 max_sasinitiators;
+	u16 max_enclosures;
+	u16 max_pcie_switches;
+	u16 max_nvme;
+	u16 max_vds;
+	u16 max_hpds;
+	u16 max_advhpds;
+	u16 max_raidpds;
+	u16 min_devhandle;
+	u16 max_devhandle;
+	u16 max_op_req_q;
+	u16 max_op_reply_q;
+	u16 shutdown_timeout;
+	u8 ioc_num;
+	u8 who_init;
+	u16 max_msix_vectors;
+	u8 personality;
+	u8 dma_mask;
+	u8 protocol_flags;
+	u8 sge_mod_mask;
+	u8 sge_mod_value;
+	u8 sge_mod_shift;
+};
+
+/**
+ * struct op_req_qinfo -  Operational Request Queue Information
+ *
+ * @ci: consumer index
+ * @pi: producer index
+ */
+struct op_req_qinfo {
+	u16 ci;
+	u16 pi;
+};
+
+/**
+ * struct op_reply_qinfo -  Operational Reply Queue Information
+ *
+ * @ci: consumer index
+ * @qid: Queue Id starting from 1
+ */
+struct op_reply_qinfo {
+	u16 ci;
+	u16 qid;
+};
+
+/**
+ * struct mpi3mr_intr_info -  Interrupt cookie information
+ *
+ * @mrioc: Adapter instance reference
+ * @msix_index: MSIx index
+ * @op_reply_q: Associated operational reply queue
+ * @name: Dev name for the irq claiming device
+ */
+struct mpi3mr_intr_info {
+	struct mpi3mr_ioc *mrioc;
+	u16 msix_index;
+	struct op_reply_qinfo *op_reply_q;
+	char name[MPI3MR_NAME_LENGTH];
+};
+
+
+
+/**
+ * struct mpi3mr_drv_cmd - Internal command tracker
+ *
+ * @mutex: Command mutex
+ * @done: Completeor for wakeup
+ * @reply: Firmware reply for internal commands
+ * @sensebuf: Sensebuf for SCSI IO commands
+ * @state: Command State
+ * @dev_handle: Firmware handle for device specific commands
+ * @ioc_status: IOC status from the firmware
+ * @ioc_loginfo:IOC log info from the firmware
+ * @is_waiting: Is the command issued in block mode
+ * @retry_count: Retry count for retriable commands
+ * @host_tag: Host tag used by the command
+ * @callback: Callback for non blocking commands
+ */
+struct mpi3mr_drv_cmd {
+	struct mutex mutex;
+	struct completion done;
+	void *reply;
+	u8 *sensebuf;
+	u16 state;
+	u16 dev_handle;
+	u16 ioc_status;
+	u32 ioc_loginfo;
+	u8 is_waiting;
+	u8 retry_count;
+	u16 host_tag;
+
+	void (*callback)(struct mpi3mr_ioc *mrioc,
+	    struct mpi3mr_drv_cmd *drv_cmd);
+};
+
+
+/**
+ * struct chain_element - memory descriptor structure to store
+ * virtual and dma addresses for chain elements.
+ *
+ * @addr: virtual address
+ * @dma_addr: dma address
+ */
+struct chain_element {
+	void *addr;
+	dma_addr_t dma_addr;
+};
+
+/**
+ * struct scmd_priv - SCSI command private data
+ *
+ * @host_tag: Host tag specific to operational queue
+ * @in_lld_scope: Command in LLD scope or not
+ * @scmd: SCSI Command pointer
+ * @req_q_idx: Operational request queue index
+ * @chain_idx: Chain frame index
+ * @mpi3mr_scsiio_req: MPI SCSI IO request
+ */
+struct scmd_priv {
+	u16 host_tag;
+	u8 in_lld_scope;
+	struct scsi_cmnd *scmd;
+	u16 req_q_idx;
+	int chain_idx;
+	u8 mpi3mr_scsiio_req[MPI3MR_ADMIN_REQ_FRAME_SZ];
+};
+
+/**
+ * struct mpi3mr_ioc - Adapter anchor structure stored in shost
+ * private data
+ *
+ * @list: List pointer
+ * @pdev: PCI device pointer
+ * @shost: Scsi_Host pointer
+ * @id: Controller ID
+ * @cpu_count: Number of online CPUs
+ * @name: Controller ASCII name
+ * @driver_name: Driver ASCII name
+ * @sysif_regs: System interface registers virtual address
+ * @sysif_regs_phys: System interface registers physical address
+ * @bars: PCI BARS
+ * @dma_mask: DMA mask
+ * @msix_count: Number of MSIX vectors used
+ * @intr_enabled: Is interrupts enabled
+ * @num_admin_req: Number of admin requests
+ * @admin_req_q_sz: Admin request queue size
+ * @admin_req_pi: Admin request queue producer index
+ * @admin_req_ci: Admin request queue consumer index
+ * @admin_req_base: Admin request queue base virtual address
+ * @admin_req_dma: Admin request queue base dma address
+ * @admin_req_lock: Admin queue access lock
+ * @num_admin_replies: Number of admin replies
+ * @admin_reply_q_sz: Admin reply queue size
+ * @admin_reply_ci: Admin reply queue consumer index
+ * @admin_reply_ephase:Admin reply queue expected phase
+ * @admin_reply_base: Admin reply queue base virtual address
+ * @admin_reply_dma: Admin reply queue base dma address
+ * @ready_timeout: Controller ready timeout
+ * @intr_info: Interrupt cookie pointer
+ * @intr_info_count: Number of interrupt cookies
+ * @num_queues: Number of operational queues
+ * @num_op_req_q: Number of operational request queues
+ * @req_qinfo: Operational request queue info pointer
+ * @num_op_reply_q: Number of operational reply queues
+ * @op_reply_qinfo: Operational reply queue info pointer
+ * @init_cmds: Command tracker for initialization commands
+ * @facts: Cached IOC facts data
+ * @op_reply_desc_sz: Operational reply descriptor size
+ * @num_reply_bufs: Number of reply buffers allocated
+ * @reply_buf_pool: Reply buffer pool
+ * @reply_buf: Reply buffer base virtual address
+ * @reply_buf_dma: Reply buffer DMA address
+ * @reply_buf_dma_max_address: Reply DMA address max limit
+ * @reply_free_qsz: Reply free queue size
+ * @reply_free_q_pool: Reply free queue pool
+ * @reply_free_q: Reply free queue base virtual address
+ * @reply_free_q_dma: Reply free queue base DMA address
+ * @reply_free_queue_lock: Reply free queue lock
+ * @reply_free_queue_host_index: Reply free queue host index
+ * @num_sense_bufs: Number of sense buffers
+ * @sense_buf_pool: Sense buffer pool
+ * @sense_buf: Sense buffer base virtual address
+ * @sense_buf_dma: Sense buffer base DMA address
+ * @sense_buf_q_sz: Sense buffer queue size
+ * @sense_buf_q_pool: Sense buffer queue pool
+ * @sense_buf_q: Sense buffer queue virtual address
+ * @sense_buf_q_dma: Sense buffer queue DMA address
+ * @sbq_lock: Sense buffer queue lock
+ * @sbq_host_index: Sense buffer queuehost index
+ * @is_driver_loading: Is driver still loading
+ * @max_host_ios: Maximum host I/O count
+ * @chain_buf_count: Chain buffer count
+ * @chain_buf_pool: Chain buffer pool
+ * @chain_sgl_list: Chain SGL list
+ * @chain_bitmap_sz: Chain buffer allocator bitmap size
+ * @chain_bitmap: Chain buffer allocator bitmap
+ * @reset_in_progress: Reset in progress flag
+ * @unrecoverable: Controller unrecoverable flag
+ * @logging_level: Controller debug logging level
+ * @current_event: Firmware event currently in process
+ * @driver_info: Driver, Kernel, OS information to firmware
+ * @change_count: Topology change count
+ */
+struct mpi3mr_ioc {
+	struct list_head list;
+	struct pci_dev *pdev;
+	struct Scsi_Host *shost;
+	u8 id;
+	int cpu_count;
+
+	char name[MPI3MR_NAME_LENGTH];
+	char driver_name[MPI3MR_NAME_LENGTH];
+
+	volatile struct mpi3_sysif_registers __iomem *sysif_regs;
+	resource_size_t sysif_regs_phys;
+	int bars;
+	u64 dma_mask;
+
+	u16 msix_count;
+	u8 intr_enabled;
+
+	u16 num_admin_req;
+	u32 admin_req_q_sz;
+	u16 admin_req_pi;
+	u16 admin_req_ci;
+	void *admin_req_base;
+	dma_addr_t admin_req_dma;
+	spinlock_t admin_req_lock;
+
+	u16 num_admin_replies;
+	u32 admin_reply_q_sz;
+	u16 admin_reply_ci;
+	u8 admin_reply_ephase;
+	void *admin_reply_base;
+	dma_addr_t admin_reply_dma;
+
+	u32 ready_timeout;
+
+	struct mpi3mr_intr_info *intr_info;
+	u16 intr_info_count;
+
+	u16 num_queues;
+	u16 num_op_req_q;
+	struct op_req_qinfo *req_qinfo;
+
+	u16 num_op_reply_q;
+	struct op_reply_qinfo *op_reply_qinfo;
+
+	struct mpi3mr_drv_cmd init_cmds;
+	struct mpi3mr_ioc_facts facts;
+	u16 op_reply_desc_sz;
+
+	u32 num_reply_bufs;
+	struct dma_pool *reply_buf_pool;
+	u8 *reply_buf;
+	dma_addr_t reply_buf_dma;
+	dma_addr_t reply_buf_dma_max_address;
+
+	u16 reply_free_qsz;
+	struct dma_pool *reply_free_q_pool;
+	__le64 *reply_free_q;
+	dma_addr_t reply_free_q_dma;
+	spinlock_t reply_free_queue_lock;
+	u32 reply_free_queue_host_index;
+
+	u32 num_sense_bufs;
+	struct dma_pool *sense_buf_pool;
+	u8 *sense_buf;
+	dma_addr_t sense_buf_dma;
+
+	u16 sense_buf_q_sz;
+	struct dma_pool *sense_buf_q_pool;
+	__le64 *sense_buf_q;
+	dma_addr_t sense_buf_q_dma;
+	spinlock_t sbq_lock;
+	u32 sbq_host_index;
+
+	u8 is_driver_loading;
+
+	u16 max_host_ios;
+
+	u32 chain_buf_count;
+	struct dma_pool *chain_buf_pool;
+	struct chain_element *chain_sgl_list;
+	u16  chain_bitmap_sz;
+	void *chain_bitmap;
+
+	u8 reset_in_progress;
+	u8 unrecoverable;
+
+	int logging_level;
+
+	struct mpi3mr_fwevt *current_event;
+	struct mpi3_driver_info_layout driver_info;
+	u16 change_count;
+};
+
+int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc);
+void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc);
+int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc);
+void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc);
+int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
+u16 admin_req_sz, u8 ignore_reset);
+void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
+			  dma_addr_t dma_addr);
+void mpi3mr_build_zero_len_sge(void *paddr);
+void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
+				     dma_addr_t phys_addr);
+void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
+				     dma_addr_t phys_addr);
+void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
+				     u64 sense_buf_dma);
+
+void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc);
+void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc);
+
+int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
+			      u32 reset_reason, u8 snapdump);
+void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc);
+void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc);
+
+enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc);
+
+#endif /*MPI3MR_H_INCLUDED*/
diff --git a/drivers/scsi/mpi3mr/mpi3mr_debug.h b/drivers/scsi/mpi3mr/mpi3mr_debug.h
new file mode 100644
index 000000000000..c085bb048d41
--- /dev/null
+++ b/drivers/scsi/mpi3mr/mpi3mr_debug.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Broadcom MPI3 Storage Controllers
+ *
+ * Copyright (C) 2017-2021 Broadcom Inc.
+ *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
+ *
+ */
+
+#ifndef MPI3SAS_DEBUG_H_INCLUDED
+
+#define MPI3SAS_DEBUG_H_INCLUDED
+
+/*
+ * debug levels
+ */
+#define MPI3_DEBUG			0x00000001
+#define MPI3_DEBUG_MSG_FRAME		0x00000002
+#define MPI3_DEBUG_SG			0x00000004
+#define MPI3_DEBUG_EVENTS		0x00000008
+#define MPI3_DEBUG_EVENT_WORK_TASK	0x00000010
+#define MPI3_DEBUG_INIT			0x00000020
+#define MPI3_DEBUG_EXIT			0x00000040
+#define MPI3_DEBUG_FAIL			0x00000080
+#define MPI3_DEBUG_TM			0x00000100
+#define MPI3_DEBUG_REPLY		0x00000200
+#define MPI3_DEBUG_HANDSHAKE		0x00000400
+#define MPI3_DEBUG_CONFIG		0x00000800
+#define MPI3_DEBUG_DL			0x00001000
+#define MPI3_DEBUG_RESET		0x00002000
+#define MPI3_DEBUG_SCSI			0x00004000
+#define MPI3_DEBUG_IOCTL		0x00008000
+#define MPI3_DEBUG_CSMISAS		0x00010000
+#define MPI3_DEBUG_SAS			0x00020000
+#define MPI3_DEBUG_TRANSPORT		0x00040000
+#define MPI3_DEBUG_TASK_SET_FULL	0x00080000
+#define MPI3_DEBUG_TRIGGER_DIAG		0x00200000
+
+
+/*
+ * debug macros
+ */
+
+#define ioc_err(ioc, fmt, ...) \
+	pr_err("%s: " fmt, (ioc)->name, ##__VA_ARGS__)
+#define ioc_notice(ioc, fmt, ...) \
+	pr_notice("%s: " fmt, (ioc)->name, ##__VA_ARGS__)
+#define ioc_warn(ioc, fmt, ...) \
+	pr_warn("%s: " fmt, (ioc)->name, ##__VA_ARGS__)
+#define ioc_info(ioc, fmt, ...) \
+	pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__)
+
+
+#define dbgprint(IOC, FMT, ...) \
+	do { \
+		if (IOC->logging_level & MPI3_DEBUG) \
+			pr_info("%s: " FMT, (IOC)->name, ##__VA_ARGS__); \
+	} while (0)
+
+#endif /* MPT3SAS_DEBUG_H_INCLUDED */
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
new file mode 100644
index 000000000000..40d67b8201a1
--- /dev/null
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -0,0 +1,1805 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Broadcom MPI3 Storage Controllers
+ *
+ * Copyright (C) 2017-2021 Broadcom Inc.
+ *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
+ *
+ */
+
+#include "mpi3mr.h"
+#include <linux/io-64-nonatomic-lo-hi.h>
+
+#if defined(writeq) && defined(CONFIG_64BIT)
+static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
+{
+	writeq(b, addr);
+}
+#else
+static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
+{
+	__u64 data_out = b;
+
+	writel((u32)(data_out), addr);
+	writel((u32)(data_out >> 32), (addr + 4));
+}
+#endif
+
+static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
+{
+	u16 i, max_vectors;
+
+	max_vectors = mrioc->intr_info_count;
+
+	for (i = 0; i < max_vectors; i++)
+		synchronize_irq(pci_irq_vector(mrioc->pdev, i));
+}
+
+void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
+{
+	mrioc->intr_enabled = 0;
+	mpi3mr_sync_irqs(mrioc);
+}
+
+void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
+{
+	mrioc->intr_enabled = 1;
+}
+
+static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
+{
+	u16 i;
+
+	mpi3mr_ioc_disable_intr(mrioc);
+
+	if (!mrioc->intr_info)
+		return;
+
+	for (i = 0; i < mrioc->intr_info_count; i++)
+		free_irq(pci_irq_vector(mrioc->pdev, i),
+		    (mrioc->intr_info + i));
+
+	kfree(mrioc->intr_info);
+	mrioc->intr_info = NULL;
+	mrioc->intr_info_count = 0;
+	pci_free_irq_vectors(mrioc->pdev);
+}
+
+void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
+	dma_addr_t dma_addr)
+{
+	struct mpi3_sge_common *sgel = paddr;
+
+	sgel->flags = flags;
+	sgel->length = cpu_to_le32(length);
+	sgel->address = cpu_to_le64(dma_addr);
+}
+
+void mpi3mr_build_zero_len_sge(void *paddr)
+{
+	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
+
+	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
+}
+
+void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
+	dma_addr_t phys_addr)
+{
+	if (!phys_addr)
+		return NULL;
+
+	if ((phys_addr < mrioc->reply_buf_dma) ||
+	    (phys_addr > mrioc->reply_buf_dma_max_address))
+		return NULL;
+
+	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
+}
+
+void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
+	dma_addr_t phys_addr)
+{
+	if (!phys_addr)
+		return NULL;
+
+	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
+}
+
+static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
+	u64 reply_dma)
+{
+	u32 old_idx = 0;
+
+	spin_lock(&mrioc->reply_free_queue_lock);
+	old_idx  =  mrioc->reply_free_queue_host_index;
+	mrioc->reply_free_queue_host_index = (
+	    (mrioc->reply_free_queue_host_index ==
+	    (mrioc->reply_free_qsz - 1)) ? 0 :
+	    (mrioc->reply_free_queue_host_index + 1));
+	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
+	writel(mrioc->reply_free_queue_host_index,
+	    &mrioc->sysif_regs->reply_free_host_index);
+	spin_unlock(&mrioc->reply_free_queue_lock);
+}
+
+void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
+	u64 sense_buf_dma)
+{
+	u32 old_idx = 0;
+
+	spin_lock(&mrioc->sbq_lock);
+	old_idx  =  mrioc->sbq_host_index;
+	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
+	    (mrioc->sense_buf_q_sz - 1)) ? 0 :
+	    (mrioc->sbq_host_index + 1));
+	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
+	writel(mrioc->sbq_host_index,
+	    &mrioc->sysif_regs->sense_buffer_free_host_index);
+	spin_unlock(&mrioc->sbq_lock);
+}
+
+static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
+	struct mpi3_default_reply *def_reply)
+{
+	struct mpi3_event_notification_reply *event_reply =
+	    (struct mpi3_event_notification_reply *)def_reply;
+
+	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
+}
+
+static struct mpi3mr_drv_cmd *
+mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
+	struct mpi3_default_reply *def_reply)
+{
+	switch (host_tag) {
+	case MPI3MR_HOSTTAG_INITCMDS:
+		return &mrioc->init_cmds;
+	case MPI3MR_HOSTTAG_INVALID:
+		if (def_reply && def_reply->function ==
+		    MPI3_FUNCTION_EVENT_NOTIFICATION)
+			mpi3mr_handle_events(mrioc, def_reply);
+		return NULL;
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
+static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
+	struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
+{
+	u16 reply_desc_type, host_tag = 0;
+	u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
+	u32 ioc_loginfo = 0;
+	struct mpi3_status_reply_descriptor *status_desc;
+	struct mpi3_address_reply_descriptor *addr_desc;
+	struct mpi3_success_reply_descriptor *success_desc;
+	struct mpi3_default_reply *def_reply = NULL;
+	struct mpi3mr_drv_cmd *cmdptr = NULL;
+	struct mpi3_scsi_io_reply *scsi_reply;
+	u8 *sense_buf = NULL;
+
+	*reply_dma = 0;
+	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
+	    MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
+	switch (reply_desc_type) {
+	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
+		status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
+		host_tag = le16_to_cpu(status_desc->host_tag);
+		ioc_status = le16_to_cpu(status_desc->ioc_status);
+		if (ioc_status &
+		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
+			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
+		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
+		break;
+	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
+		addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
+		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
+		def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
+		if (!def_reply)
+			goto out;
+		host_tag = le16_to_cpu(def_reply->host_tag);
+		ioc_status = le16_to_cpu(def_reply->ioc_status);
+		if (ioc_status &
+		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
+			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
+		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
+		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
+			scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
+			sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
+			    le64_to_cpu(scsi_reply->sense_data_buffer_address));
+		}
+		break;
+	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
+		success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
+		host_tag = le16_to_cpu(success_desc->host_tag);
+		break;
+	default:
+		break;
+	}
+
+	cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
+	if (cmdptr) {
+		if (cmdptr->state & MPI3MR_CMD_PENDING) {
+			cmdptr->state |= MPI3MR_CMD_COMPLETE;
+			cmdptr->ioc_loginfo = ioc_loginfo;
+			cmdptr->ioc_status = ioc_status;
+			cmdptr->state &= ~MPI3MR_CMD_PENDING;
+			if (def_reply) {
+				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
+				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
+				    mrioc->facts.reply_sz);
+			}
+			if (cmdptr->is_waiting) {
+				complete(&cmdptr->done);
+				cmdptr->is_waiting = 0;
+			} else if (cmdptr->callback)
+				cmdptr->callback(mrioc, cmdptr);
+		}
+	}
+out:
+	if (sense_buf)
+		mpi3mr_repost_sense_buf(mrioc,
+		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
+}
+
+static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
+{
+	u32 exp_phase = mrioc->admin_reply_ephase;
+	u32 admin_reply_ci = mrioc->admin_reply_ci;
+	u32 num_admin_replies = 0;
+	u64 reply_dma = 0;
+	struct mpi3_default_reply_descriptor *reply_desc;
+
+	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
+	    admin_reply_ci;
+
+	if ((le16_to_cpu(reply_desc->reply_flags) &
+	    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
+		return 0;
+
+	do {
+		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
+		mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
+		if (reply_dma)
+			mpi3mr_repost_reply_buf(mrioc, reply_dma);
+		num_admin_replies++;
+		if (++admin_reply_ci == mrioc->num_admin_replies) {
+			admin_reply_ci = 0;
+			exp_phase ^= 1;
+		}
+		reply_desc =
+		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
+		    admin_reply_ci;
+		if ((le16_to_cpu(reply_desc->reply_flags) &
+		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
+			break;
+	} while (1);
+
+	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
+	mrioc->admin_reply_ci = admin_reply_ci;
+	mrioc->admin_reply_ephase = exp_phase;
+
+	return num_admin_replies;
+}
+
+static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
+{
+	struct mpi3mr_intr_info *intr_info = privdata;
+	struct mpi3mr_ioc *mrioc;
+	u16 midx;
+	u32 num_admin_replies = 0;
+
+	if (!intr_info)
+		return IRQ_NONE;
+
+	mrioc = intr_info->mrioc;
+
+	if (!mrioc->intr_enabled)
+		return IRQ_NONE;
+
+	midx = intr_info->msix_index;
+
+	if (!midx)
+		num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
+
+	if (num_admin_replies)
+		return IRQ_HANDLED;
+	else
+		return IRQ_NONE;
+}
+
+static irqreturn_t mpi3mr_isr(int irq, void *privdata)
+{
+	struct mpi3mr_intr_info *intr_info = privdata;
+	int ret;
+
+	if (!intr_info)
+		return IRQ_NONE;
+
+	/* Call primary ISR routine */
+	ret = mpi3mr_isr_primary(irq, privdata);
+
+	return ret;
+}
+
+/**
+ * mpi3mr_isr_poll - Reply queue polling routine
+ * @irq: IRQ
+ * @privdata: Interrupt info
+ *
+ * poll for pending I/O completions in a loop until pending I/Os
+ * present or controller queue depth I/Os are processed.
+ *
+ * Return: IRQ_NONE or IRQ_HANDLED
+ */
+static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
+{
+	return IRQ_HANDLED;
+}
+
+/**
+ * mpi3mr_request_irq - Request IRQ and register ISR
+ * @mrioc: Adapter instance reference
+ * @index: IRQ vector index
+ *
+ * Request threaded ISR with primary ISR and secondary
+ *
+ * Return: 0 on success and non zero on failures.
+ */
+static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
+{
+	struct pci_dev *pdev = mrioc->pdev;
+	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
+	int retval = 0;
+
+	intr_info->mrioc = mrioc;
+	intr_info->msix_index = index;
+	intr_info->op_reply_q = NULL;
+
+	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
+	    mrioc->driver_name, mrioc->id, index);
+
+	retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
+	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
+	if (retval) {
+		ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
+		    intr_info->name, pci_irq_vector(pdev, index));
+		return retval;
+	}
+
+	return retval;
+}
+
+/**
+ * mpi3mr_setup_isr - Setup ISR for the controller
+ * @mrioc: Adapter instance reference
+ * @setup_one: Request one IRQ or more
+ *
+ * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
+ *
+ * Return: 0 on success and non zero on failures.
+ */
+static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
+{
+	unsigned int irq_flags = PCI_IRQ_MSIX;
+	u16 max_vectors = 0, i;
+	int retval = 0;
+	struct irq_affinity desc = { .pre_vectors =  1};
+
+	mpi3mr_cleanup_isr(mrioc);
+
+	if (setup_one || reset_devices)
+		max_vectors = 1;
+	else {
+		max_vectors =
+		    min_t(int, mrioc->cpu_count + 1, mrioc->msix_count);
+
+		ioc_info(mrioc,
+		    "MSI-X vectors supported: %d, no of cores: %d,",
+		    mrioc->msix_count, mrioc->cpu_count);
+		ioc_info(mrioc,
+		    "MSI-x vectors requested: %d\n", max_vectors);
+	}
+
+	irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
+
+	i = pci_alloc_irq_vectors_affinity(mrioc->pdev,
+	    1, max_vectors, irq_flags, &desc);
+	if (i <= 0) {
+		ioc_err(mrioc, "Cannot alloc irq vectors\n");
+		goto out_failed;
+	}
+	if (i != max_vectors) {
+		ioc_info(mrioc,
+		    "allocated vectors (%d) are less than configured (%d)\n",
+		    i, max_vectors);
+
+		max_vectors = i;
+	}
+	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
+	    GFP_KERNEL);
+	if (!mrioc->intr_info) {
+		retval = -1;
+		pci_free_irq_vectors(mrioc->pdev);
+		goto out_failed;
+	}
+	for (i = 0; i < max_vectors; i++) {
+		retval = mpi3mr_request_irq(mrioc, i);
+		if (retval) {
+			mrioc->intr_info_count = i;
+			goto out_failed;
+		}
+	}
+	mrioc->intr_info_count = max_vectors;
+	mpi3mr_ioc_enable_intr(mrioc);
+	return retval;
+out_failed:
+	mpi3mr_cleanup_isr(mrioc);
+
+	return retval;
+}
+
+static const struct {
+	enum mpi3mr_iocstate value;
+	char *name;
+} mrioc_states[] = {
+	{ MRIOC_STATE_READY, "ready" },
+	{ MRIOC_STATE_FAULT, "fault" },
+	{ MRIOC_STATE_RESET, "reset" },
+	{ MRIOC_STATE_BECOMING_READY, "becoming ready" },
+	{ MRIOC_STATE_RESET_REQUESTED, "reset requested" },
+	{ MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
+};
+
+static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
+{
+	int i;
+	char *name = NULL;
+
+	for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
+		if (mrioc_states[i].value == mrioc_state) {
+			name = mrioc_states[i].name;
+			break;
+		}
+	}
+	return name;
+}
+
+/**
+ * mpi3mr_print_fault_info - Display fault information
+ * @mrioc: Adapter instance reference
+ *
+ * Display the controller fault information if there is a
+ * controller fault.
+ *
+ * Return: Nothing.