path: root/drivers/infiniband/sw/rxe/rxe_req.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
 */

#include <linux/skbuff.h>
#include <crypto/hash.h>

#include "rxe.h"
#include "rxe_loc.h"
#include "rxe_queue.h"

static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
		       u32 opcode);

static inline void retry_first_write_send(struct rxe_qp *qp,
					  struct rxe_send_wqe *wqe, int npsn)
{
	int i;

	for (i = 0; i < npsn; i++) {
		int to_send = (wqe->dma.resid > qp->mtu) ?
				qp->mtu : wqe->dma.resid;

		qp->req.opcode = next_opcode(qp, wqe,
					     wqe->wr.opcode);

		if (wqe->wr.send_flags & IB_SEND_INLINE) {
			wqe->dma.resid -= to_send;
			wqe->dma.sge_offset += to_send;
		} else {
			advance_dma_data(&wqe->dma, to_send);
		}
	}
}

static void req_retry(struct rxe_qp *qp)
{
	struct rxe_send_wqe *wqe;
	unsigned int wqe_index;
	unsigned int mask;
	int npsn;
	int first = 1;
	struct rxe_queue *q = qp->sq.queue;
	unsigned int cons;
	unsigned int prod;

	cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
	prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);

	qp->req.wqe_index	= cons;
	qp->req.psn		= qp->comp.psn;
	qp->req.opcode		= -1;

	for (wqe_index = cons; wqe_index != prod;
			wqe_index = queue_next_index(q, wqe_index)) {
		wqe = queue_addr_from_index(qp->sq.queue, wqe_index);
		mask = wr_opcode_mask(wqe->wr.opcode, qp);

		if (wqe->state == wqe_state_posted)
			break;

		if (wqe->state == wqe_state_done)
			continue;

		wqe->iova = (mask & WR_ATOMIC_MASK) ?
			     wqe->wr.wr.atomic.remote_addr :
			     (mask & WR_READ_OR_WRITE_MASK) ?
			     wqe->wr.wr.rdma.remote_addr :
			     0;

		if (!first || (mask & WR_READ_MASK) == 0) {
			wqe->dma.resid = wqe->dma.length;
			wqe->dma.cur_sge = 0;
			wqe->dma.sge_offset = 0;
		}

		if (first) {
			first = 0;

			if (mask & WR_WRITE_OR_SEND_MASK) {
				npsn = (qp->comp.psn - wqe->first_psn) &
					BTH_PSN_MASK;
				retry_first_write_send(qp, wqe, npsn);
			}

			if (mask & WR_READ_MASK) {
				npsn = (wqe->dma.length - wqe->dma.resid) /
					qp->mtu;
				wqe->iova += npsn * qp->mtu;
			}
		}

		wqe->state = wqe_state_posted;
	}
}

void rnr_nak_timer(struct timer_list *t)
{
	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
	unsigned long flags;

	rxe_dbg_qp(qp, "nak timer fired\n");

	spin_lock_irqsave(&qp->state_lock, flags);
	if (qp->valid) {
		/* request a send queue retry */
		qp->req.need_retry = 1;
		qp->req.wait_for_rnr_timer = 0;
		rxe_sched_task(&qp->req.task);
	}
	spin_unlock_irqrestore(&qp->state_lock, flags);
}

static void req_check_sq_drain_done(struct rxe_qp *qp)
{
	struct rxe_queue *q;
	unsigned int index;
	unsigned int cons;
	struct rxe_send_wqe *wqe;
	unsigned long flags;

	spin_lock_irqsave(&qp->state_lock, flags);
	if (qp_state(qp) == IB_QPS_SQD) {
		q = qp->sq.queue;
		index = qp->req.wqe_index;
		cons = queue_get_consumer(q, QUEUE_TYPE_FROM_CLIENT);
		wqe = queue_addr_from_index(q, cons);

		/* check to see if we are drained;
		 * state_lock used by requester and completer
		 */
		do {
			if (!qp->attr.sq_draining)
				/* comp just finished */
				break;

			if (wqe && ((index != cons) ||
				(wqe->state != wqe_state_posted)))
				/* comp not done yet */
				break;

			qp->attr.sq_draining = 0;
			spin_unlock_irqrestore(&qp->state_lock, flags);

			if (qp->ibqp.event_handler) {
				struct ib_event ev;

				ev.device = qp->ibqp.device;
				ev.element.qp = &qp->ibqp;
				ev.event = IB_EVENT_SQ_DRAINED;
				qp->ibqp.event_handler(&ev,
					qp->ibqp.qp_context);
			}
			return;
		} while (0);
	}
	spin_unlock_irqrestore(&qp->state_lock, flags);
}

static struct rxe_send_wqe *__req_next_wqe(struct rxe_qp *qp)
{
	struct rxe_queue *q = qp->sq.queue;
	unsigned int index = qp->req.wqe_index;
	unsigned int prod;

	prod = queue_get_producer(q, QUEUE_TYPE_FROM_CLIENT);
	if (index == prod)
		return NULL;
	else
		return queue_addr_from_index(q, index);
}

static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
{
	struct rxe_send_wqe *wqe;
	unsigned long flags;

	req_check_sq_drain_done(qp);

	wqe = __req_next_wqe(qp);
	if (wqe == NULL)
		return NULL;

	spin_lock_irqsave(&qp->state_lock, flags);
	if (unlikely((qp_state(qp) == IB_QPS_SQD) &&
		     (wqe->state != wqe_state_processing))) {
		spin_unlock_irqrestore(&qp->state_lock, flags);
		return NULL;
	}
	spin_unlock_irqrestore(&qp->state_lock, flags);

	wqe->mask = wr_opcode_mask(