path: root/drivers/cxl/pci.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
#include <linux/unaligned.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/moduleparam.h>
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/sizes.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/pci.h>
#include <linux/aer.h>
#include <linux/io.h>
#include <cxl/mailbox.h>
#include "cxlmem.h"
#include "cxlpci.h"
#include "cxl.h"
#include "pmu.h"

/**
 * DOC: cxl pci
 *
 * This implements the PCI exclusive functionality for a CXL device as it is
 * defined by the Compute Express Link specification. CXL devices may surface
 * certain functionality even if it isn't CXL enabled. While this driver is
 * focused around the PCI specific aspects of a CXL device, it binds to the
 * specific CXL memory device class code, and therefore the implementation of
 * cxl_pci is focused around CXL memory devices.
 *
 * The driver has several responsibilities, mainly:
 *  - Create the memX device and register on the CXL bus.
 *  - Enumerate device's register interface and map them.
 *  - Registers nvdimm bridge device with cxl_core.
 *  - Registers a CXL mailbox with cxl_core.
 */

#define cxl_doorbell_busy(cxlds)                                                \
	(readl((cxlds)->regs.mbox + CXLDEV_MBOX_CTRL_OFFSET) &                  \
	 CXLDEV_MBOX_CTRL_DOORBELL)

/* CXL 2.0 - 8.2.8.4 */
#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ)

/*
 * CXL 2.0 ECN "Add Mailbox Ready Time" defines a capability field to
 * dictate how long to wait for the mailbox to become ready. The new
 * field allows the device to tell software the amount of time to wait
 * before mailbox ready. This field per the spec theoretically allows
 * for up to 255 seconds. 255 seconds is unreasonably long, its longer
 * than the maximum SATA port link recovery wait. Default to 60 seconds
 * until someone builds a CXL device that needs more time in practice.
 */
static unsigned short mbox_ready_timeout = 60;
module_param(mbox_ready_timeout, ushort, 0644);
MODULE_PARM_DESC(mbox_ready_timeout, "seconds to wait for mailbox ready");

static int cxl_pci_mbox_wait_for_doorbell(struct cxl_dev_state *cxlds)
{
	const unsigned long start = jiffies;
	unsigned long end = start;

	while (cxl_doorbell_busy(cxlds)) {
		end = jiffies;

		if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) {
			/* Check again in case preempted before timeout test */
			if (!cxl_doorbell_busy(cxlds))
				break;
			return -ETIMEDOUT;
		}
		cpu_relax();
	}

	dev_dbg(cxlds->dev, "Doorbell wait took %dms",
		jiffies_to_msecs(end) - jiffies_to_msecs(start));
	return 0;
}

#define cxl_err(dev, status, msg)                                        \
	dev_err_ratelimited(dev, msg ", device state %s%s\n",                  \
			    status & CXLMDEV_DEV_FATAL ? " fatal" : "",        \
			    status & CXLMDEV_FW_HALT ? " firmware-halt" : "")

#define cxl_cmd_err(dev, cmd, status, msg)                               \
	dev_err_ratelimited(dev, msg " (opcode: %#x), device state %s%s\n",    \
			    (cmd)->opcode,                                     \
			    status & CXLMDEV_DEV_FATAL ? " fatal" : "",        \
			    status & CXLMDEV_FW_HALT ? " firmware-halt" : "")

/*
 * Threaded irq dev_id's must be globally unique.  cxl_dev_id provides a unique
 * wrapper object for each irq within the same cxlds.
 */
struct cxl_dev_id {
	struct cxl_dev_state *cxlds;
};

static int cxl_request_irq(struct cxl_dev_state *cxlds, int irq,
			   irq_handler_t thread_fn)
{
	struct device *dev = cxlds->dev;
	struct cxl_dev_id *dev_id;

	dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL);
	if (!dev_id)
		return -ENOMEM;
	dev_id->cxlds = cxlds;

	return devm_request_threaded_irq(dev, irq, NULL, thread_fn,
					 IRQF_SHARED | IRQF_ONESHOT, NULL,
					 dev_id);
}

static bool cxl_mbox_background_complete(struct cxl_dev_state *cxlds)
{
	u64 reg;

	reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
	return FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK, reg) == 100;
}

static irqreturn_t cxl_pci_mbox_irq(int irq, void *id)
{
	u64 reg;
	u16 opcode;
	struct cxl_dev_id *dev_id = id;
	struct cxl_dev_state *cxlds = dev_id->cxlds;
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);

	if (!cxl_mbox_background_complete(cxlds))
		return IRQ_NONE;

	reg = readq(cxlds->regs.mbox + CXLDEV_MBOX_BG_CMD_STATUS_OFFSET);
	opcode = FIELD_GET(CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK, reg);
	if (opcode == CXL_MBOX_OP_SANITIZE) {
		mutex_lock(&cxl_mbox->mbox_mutex);
		if (mds->security.sanitize_node)
			mod_delayed_work(system_wq, &mds->security.poll_dwork, 0);
		mutex_unlock(&cxl_mbox->mbox_mutex);
	} else {
		/* short-circuit the wait in __cxl_pci_mbox_send_cmd() */
		rcuwait_wake_up(&cxl_mbox->mbox_wait);
	}

	return IRQ_HANDLED;
}

/*
 * Sanitization operation polling mode.
 */
static void cxl_mbox_sanitize_work(struct work_struct *work)
{
	struct cxl_memdev_state *mds =
		container_of(work, typeof(*mds), security.poll_dwork.work);
	struct cxl_dev_state *cxlds = &mds->cxlds;
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;

	mutex_lock(&cxl_mbox->mbox_mutex);
	if (cxl_mbox_background_complete(cxlds)) {
		mds->security.poll_tmo_secs = 0;
		if (mds->security.sanitize_node)
			sysfs_notify_dirent(mds->security.sanitize_node);
		mds->security.sanitize_active = false;

		dev_dbg(cxlds->dev, "Sanitization operation ended\n");
	} else {
		int timeout = mds->security.poll_tmo_secs + 10;

		mds->security.poll_tmo_secs =