path: root/drivers/misc/mic/scif/scif_nodeqp.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Intel MIC Platform Software Stack (MPSS)
 *
 * Copyright(c) 2014 Intel Corporation.
 *
 * Intel SCIF driver.
 */
#include "../bus/scif_bus.h"
#include "scif_peer_bus.h"
#include "scif_main.h"
#include "scif_nodeqp.h"
#include "scif_map.h"

/*
 ************************************************************************
 * SCIF node Queue Pair (QP) setup flow:
 *
 * 1) SCIF driver gets probed with a scif_hw_dev via the scif_hw_bus
 * 2) scif_setup_qp(..) allocates the local qp and calls
 *	scif_setup_qp_connect(..) which allocates and maps the local
 *	buffer for the inbound QP
 * 3) The local node updates the device page with the DMA address of the QP
 * 4) A delayed work is scheduled (qp_dwork) which periodically reads if
 *	the peer node has updated its QP DMA address
 * 5) Once a valid non zero address is found in the QP DMA address field
 *	in the device page, the local node maps the remote node's QP,
 *	updates its outbound QP and sends a SCIF_INIT message to the peer
 * 6) The SCIF_INIT message is received by the peer node QP interrupt bottom
 *	half handler by calling scif_init(..)
 * 7) scif_init(..) registers a new SCIF peer node by calling
 *	scif_peer_register_device(..) which signifies the addition of a new
 *	SCIF node
 * 8) On the mgmt node, P2P network setup/teardown is initiated if all the
 *	remote nodes are online via scif_p2p_setup(..)
 * 9) For P2P setup, the host maps the remote nodes' aperture and memory
 *	bars and sends a SCIF_NODE_ADD message to both nodes
 * 10) As part of scif_nodeadd, both nodes set up their local inbound
 *	QPs and send a SCIF_NODE_ADD_ACK to the mgmt node
 * 11) As part of scif_node_add_ack(..) the mgmt node forwards the
 *	SCIF_NODE_ADD_ACK to the remote nodes
 * 12) As part of scif_node_add_ack(..) the remote nodes update their
 *	outbound QPs, make sure they can access memory on the remote node
 *	and then add a new SCIF peer node by calling
 *	scif_peer_register_device(..) which signifies the addition of a new
 *	SCIF node.
 * 13) The SCIF network is now established across all nodes.
 *
 ************************************************************************
 * SCIF node QP teardown flow (initiated by non mgmt node):
 *
 * 1) SCIF driver gets a remove callback with a scif_hw_dev via the scif_hw_bus
 * 2) The device page QP DMA address field is updated with 0x0
 * 3) A non mgmt node now cleans up all local data structures and sends a
 *	SCIF_EXIT message to the peer and waits for a SCIF_EXIT_ACK
 * 4) As part of scif_exit(..) handling scif_disconnect_node(..) is called
 * 5) scif_disconnect_node(..) sends a SCIF_NODE_REMOVE message to all the
 *	peers and waits for a SCIF_NODE_REMOVE_ACK
 * 6) As part of scif_node_remove(..) a remote node unregisters the peer
 *	node from the SCIF network and sends a SCIF_NODE_REMOVE_ACK
 * 7) When the mgmt node has received all the SCIF_NODE_REMOVE_ACKs
 *	it sends itself a node remove message whose handling cleans up local
 *	data structures and unregisters the peer node from the SCIF network
 * 8) The mgmt node sends a SCIF_EXIT_ACK
 * 9) Upon receipt of the SCIF_EXIT_ACK the node initiating the teardown
 *	completes the SCIF remove routine
 * 10) The SCIF network is now torn down for the node initiating the
 *	teardown sequence
 *
 ************************************************************************
 * SCIF node QP teardown flow (initiated by mgmt node):
 *
 * 1) SCIF driver gets a remove callback with a scif_hw_dev via the scif_hw_bus
 * 2) The device page QP DMA address field is updated with 0x0
 * 3) The mgmt node calls scif_disconnect_node(..)
 * 4) scif_disconnect_node(..) sends a SCIF_NODE_REMOVE message to all the peers
 *	and waits for a SCIF_NODE_REMOVE_ACK
 * 5) As part of scif_node_remove(..) a remote node unregisters the peer
 *	node from the SCIF network and sends a SCIF_NODE_REMOVE_ACK
 * 6) When the mgmt node has received all the SCIF_NODE_REMOVE_ACKs
 *	it unregisters the peer node from the SCIF network
 * 7) The mgmt node sends a SCIF_EXIT message and waits for a SCIF_EXIT_ACK.
 * 8) A non mgmt node upon receipt of a SCIF_EXIT message calls scif_stop(..)
 *	which would clean up local data structures for all SCIF nodes and
 *	then send a SCIF_EXIT_ACK back to the mgmt node
 * 9) Upon receipt of the SCIF_EXIT_ACK the the mgmt node sends itself a node
 *	remove message whose handling cleans up local data structures and
 *	destroys any P2P mappings.
 * 10) The SCIF hardware device for which a remove callback was received is now
 *	disconnected from the SCIF network.
 */
/*
 * Initializes "local" data structures for the QP. Allocates the QP
 * ring buffer (rb) and initializes the "in bound" queue.
 */
int scif_setup_qp_connect(struct scif_qp *qp, dma_addr_t *qp_offset,
			  int local_size, struct scif_dev *scifdev)
{
	void *local_q = qp->inbound_q.rb_base;
	int err = 0;
	u32 tmp_rd = 0;

	spin_lock_init(&qp->send_lock);
	spin_lock_init(&qp->recv_lock);

	/* Allocate rb only if not already allocated */
	if (!local_q) {
		local_q = kzalloc(local_size, GFP_KERNEL);
		if (!local_q) {
			err = -ENOMEM;
			return err;
		}
	}