rdma: Remove nes

This driver was first merged over 10 years ago and has not seen major
activity by the authors in the last 7 years. However, in that time it has
been patched 150 times to adapt it to changing kernel APIs.

Further, the hardware has several issues, like not supporting 64 bit DMA,
that make it rather uninteresting for use with modern systems and RDMA.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
Reviewed-by: Shiraz Saleem <shiraz.saleem@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

1 files changed, 0 insertions, 3721 deletions

diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
deleted file mode 100644
index 0420203820f6..000000000000
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ /dev/null
@@ -1,3721 +0,0 @@
-/*
- * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/random.h>
-#include <linux/highmem.h>
-#include <linux/slab.h>
-#include <asm/byteorder.h>
-
-#include <rdma/ib_verbs.h>
-#include <rdma/iw_cm.h>
-#include <rdma/ib_user_verbs.h>
-#include <rdma/uverbs_ioctl.h>
-
-#include "nes.h"
-
-#include <rdma/ib_umem.h>
-
-atomic_t mod_qp_timouts;
-atomic_t qps_created;
-atomic_t sw_qps_destroyed;
-
-static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
-static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
-
-/**
- * nes_alloc_mw
- */
-static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type,
-				  struct ib_udata *udata)
-{
-	struct nes_pd *nespd = to_nespd(ibpd);
-	struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_cqp_request *cqp_request;
-	struct nes_mr *nesmr;
-	struct ib_mw *ibmw;
-	struct nes_hw_cqp_wqe *cqp_wqe;
-	int ret;
-	u32 stag;
-	u32 stag_index = 0;
-	u32 next_stag_index = 0;
-	u32 driver_key = 0;
-	u8 stag_key = 0;
-
-	if (type != IB_MW_TYPE_1)
-		return ERR_PTR(-EINVAL);
-
-	get_random_bytes(&next_stag_index, sizeof(next_stag_index));
-	stag_key = (u8)next_stag_index;
-
-	driver_key = 0;
-
-	next_stag_index >>= 8;
-	next_stag_index %= nesadapter->max_mr;
-
-	ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
-			nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW);
-	if (ret) {
-		return ERR_PTR(ret);
-	}
-
-	nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
-	if (!nesmr) {
-		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	stag = stag_index << 8;
-	stag |= driver_key;
-	stag += (u32)stag_key;
-
-	nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n",
-			stag, stag_index);
-
-	/* Register the region with the adapter */
-	cqp_request = nes_get_cqp_request(nesdev);
-	if (cqp_request == NULL) {
-		kfree(nesmr);
-		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	cqp_request->waiting = 1;
-	cqp_wqe = &cqp_request->cqp_wqe;
-
-	cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
-			cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ |
-			NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO |
-			NES_CQP_STAG_REM_ACC_EN);
-
-	nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff));
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
-
-	atomic_set(&cqp_request->refcount, 2);
-	nes_post_cqp_request(nesdev, cqp_request);
-
-	/* Wait for CQP */
-	ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
-			NES_EVENT_TIMEOUT);
-	nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
-			" CQP Major:Minor codes = 0x%04X:0x%04X.\n",
-			stag, ret, cqp_request->major_code, cqp_request->minor_code);
-	if ((!ret) || (cqp_request->major_code)) {
-		nes_put_cqp_request(nesdev, cqp_request);
-		kfree(nesmr);
-		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-		if (!ret) {
-			return ERR_PTR(-ETIME);
-		} else {
-			return ERR_PTR(-ENOMEM);
-		}
-	}
-	nes_put_cqp_request(nesdev, cqp_request);
-
-	nesmr->ibmw.rkey = stag;
-	nesmr->mode = IWNES_MEMREG_TYPE_MW;
-	ibmw = &nesmr->ibmw;
-	nesmr->pbl_4k = 0;
-	nesmr->pbls_used = 0;
-
-	return ibmw;
-}
-
-
-/**
- * nes_dealloc_mw
- */
-static int nes_dealloc_mw(struct ib_mw *ibmw)
-{
-	struct nes_mr *nesmr = to_nesmw(ibmw);
-	struct nes_vnic *nesvnic = to_nesvnic(ibmw->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_hw_cqp_wqe *cqp_wqe;
-	struct nes_cqp_request *cqp_request;
-	int err = 0;
-	int ret;
-
-	/* Deallocate the window with the adapter */
-	cqp_request = nes_get_cqp_request(nesdev);
-	if (cqp_request == NULL) {
-		nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
-		return -ENOMEM;
-	}
-	cqp_request->waiting = 1;
-	cqp_wqe = &cqp_request->cqp_wqe;
-	nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey);
-
-	atomic_set(&cqp_request->refcount, 2);
-	nes_post_cqp_request(nesdev, cqp_request);
-
-	/* Wait for CQP */
-	nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n",
-			ibmw->rkey);
-	ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done),
-			NES_EVENT_TIMEOUT);
-	nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u,"
-			" CQP Major:Minor codes = 0x%04X:0x%04X.\n",
-			ret, cqp_request->major_code, cqp_request->minor_code);
-	if (!ret)
-		err = -ETIME;
-	else if (cqp_request->major_code)
-		err = -EIO;
-
-	nes_put_cqp_request(nesdev, cqp_request);
-
-	nes_free_resource(nesadapter, nesadapter->allocated_mrs,
-			(ibmw->rkey & 0x0fffff00) >> 8);
-	kfree(nesmr);
-
-	return err;
-}
-
-
-/*
- * nes_alloc_fast_mr
- */
-static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
-			     u32 stag, u32 page_count)
-{
-	struct nes_hw_cqp_wqe *cqp_wqe;
-	struct nes_cqp_request *cqp_request;
-	unsigned long flags;
-	int ret;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	u32 opcode = 0;
-	u16 major_code;
-	u64 region_length = page_count * PAGE_SIZE;
-
-
-	cqp_request = nes_get_cqp_request(nesdev);
-	if (cqp_request == NULL) {
-		nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n");
-		return -ENOMEM;
-	}
-	nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, "
-			      "region_length = %llu\n",
-			      page_count, region_length);
-	cqp_request->waiting = 1;
-	cqp_wqe = &cqp_request->cqp_wqe;
-
-	spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-	if (nesadapter->free_4kpbl > 0) {
-		nesadapter->free_4kpbl--;
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-	} else {
-		/* No 4kpbl's available: */
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-		nes_debug(NES_DBG_MR, "Out of Pbls\n");
-		nes_free_cqp_request(nesdev, cqp_request);
-		return -ENOMEM;
-	}
-
-	opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR |
-		 NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO |
-		 NES_CQP_STAG_REM_ACC_EN;
-	/*
-	 * The current OFED API does not support the zero based TO option.
-	 * If added then need to changed the NES_CQP_STAG_VA* option.  Also,
-	 * the API does not support that ability to have the MR set for local
-	 * access only when created and not allow the SQ op to override. Given
-	 * this the remote enable must be set here.
-	 */
-
-	nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1);
-
-	cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] =
-			cpu_to_le32((u32)(region_length >> 8) & 0xff000000);
-	cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |=
-			cpu_to_le32(nespd->pd_id & 0x00007fff);
-
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag);
-	set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0);
-	set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0);
-	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8));
-	cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE);
-	barrier();
-
-	atomic_set(&cqp_request->refcount, 2);
-	nes_post_cqp_request(nesdev, cqp_request);
-
-	/* Wait for CQP */
-	ret = wait_event_timeout(cqp_request->waitq,
-				 (0 != cqp_request->request_done),
-				 NES_EVENT_TIMEOUT);
-
-	nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, "
-		  "wait_event_timeout ret = %u, CQP Major:Minor codes = "
-		  "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code,
-		  cqp_request->minor_code);
-	major_code = cqp_request->major_code;
-	nes_put_cqp_request(nesdev, cqp_request);
-
-	if (!ret || major_code) {
-		spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-		nesadapter->free_4kpbl++;
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-	}
-
-	if (!ret)
-		return -ETIME;
-	else if (major_code)
-		return -EIO;
-	return 0;
-}
-
-/*
- * nes_alloc_mr
- */
-static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
-				  u32 max_num_sg, struct ib_udata *udata)
-{
-	struct nes_pd *nespd = to_nespd(ibpd);
-	struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-
-	u32 next_stag_index;
-	u8 stag_key = 0;
-	u32 driver_key = 0;
-	int err = 0;
-	u32 stag_index = 0;
-	struct nes_mr *nesmr;
-	u32 stag;
-	int ret;
-	struct ib_mr *ibmr;
-
-	if (mr_type != IB_MR_TYPE_MEM_REG)
-		return ERR_PTR(-EINVAL);
-
-	if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
-		return ERR_PTR(-E2BIG);
-
-/*
- * Note:  Set to always use a fixed length single page entry PBL.  This is to allow
- *	 for the fast_reg_mr operation to always know the size of the PBL.
- */
-	if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64)))
-		return ERR_PTR(-E2BIG);
-
-	get_random_bytes(&next_stag_index, sizeof(next_stag_index));
-	stag_key = (u8)next_stag_index;
-	next_stag_index >>= 8;
-	next_stag_index %= nesadapter->max_mr;
-
-	err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs,
-				 nesadapter->max_mr, &stag_index,
-				 &next_stag_index, NES_RESOURCE_FAST_MR);
-	if (err)
-		return ERR_PTR(err);
-
-	nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL);
-	if (!nesmr) {
-		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	stag = stag_index << 8;
-	stag |= driver_key;
-	stag += (u32)stag_key;
-
-	nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n",
-		  stag, stag_index);
-
-	ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg);
-
-	if (ret == 0) {
-		nesmr->ibmr.rkey = stag;
-		nesmr->ibmr.lkey = stag;
-		nesmr->mode = IWNES_MEMREG_TYPE_FMEM;
-		ibmr = &nesmr->ibmr;
-	} else {
-		kfree(nesmr);
-		nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	nesmr->pages = pci_alloc_consistent(nesdev->pcidev,
-					    max_num_sg * sizeof(u64),
-					    &nesmr->paddr);
-	if (!nesmr->paddr)
-		goto err;
-
-	nesmr->max_pages = max_num_sg;
-
-	return ibmr;
-
-err:
-	nes_dereg_mr(ibmr, udata);
-
-	return ERR_PTR(-ENOMEM);
-}
-
-static int nes_set_page(struct ib_mr *ibmr, u64 addr)
-{
-	struct nes_mr *nesmr = to_nesmr(ibmr);
-
-	if (unlikely(nesmr->npages == nesmr->max_pages))
-		return -ENOMEM;
-
-	nesmr->pages[nesmr->npages++] = cpu_to_le64(addr);
-
-	return 0;
-}
-
-static int nes_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-			 int sg_nents, unsigned int *sg_offset)
-{
-	struct nes_mr *nesmr = to_nesmr(ibmr);
-
-	nesmr->npages = 0;
-
-	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, nes_set_page);
-}
-
-/**
- * nes_query_device
- */
-static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
-			    struct ib_udata *uhw)
-{
-	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_ib_device *nesibdev = nesvnic->nesibdev;
-
-	if (uhw->inlen || uhw->outlen)
-		return -EINVAL;
-
-	memset(props, 0, sizeof(*props));
-	memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6);
-
-	props->fw_ver = nesdev->nesadapter->firmware_version;
-	props->device_cap_flags = nesdev->nesadapter->device_cap_flags;
-	props->vendor_id = nesdev->nesadapter->vendor_id;
-	props->vendor_part_id = nesdev->nesadapter->vendor_part_id;
-	props->hw_ver = nesdev->nesadapter->hw_rev;
-	props->max_mr_size = 0x80000000;
-	props->max_qp = nesibdev->max_qp;
-	props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2;
-	props->max_send_sge = nesdev->nesadapter->max_sge;
-	props->max_recv_sge = nesdev->nesadapter->max_sge;
-	props->max_cq = nesibdev->max_cq;
-	props->max_cqe = nesdev->nesadapter->max_cqe;
-	props->max_mr = nesibdev->max_mr;
-	props->max_mw = nesibdev->max_mr;
-	props->max_pd = nesibdev->max_pd;
-	props->max_sge_rd = 1;
-	switch (nesdev->nesadapter->max_irrq_wr) {
-		case 0:
-			props->max_qp_rd_atom = 2;
-			break;
-		case 1:
-			props->max_qp_rd_atom = 8;
-			break;
-		case 2:
-			props->max_qp_rd_atom = 32;
-			break;
-		case 3:
-			props->max_qp_rd_atom = 64;
-			break;
-		default:
-			props->max_qp_rd_atom = 0;
-	}
-	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
-	props->atomic_cap = IB_ATOMIC_NONE;
-	props->max_map_per_fmr = 1;
-
-	return 0;
-}
-
-
-/**
- * nes_query_port
- */
-static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props)
-{
-	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-	struct net_device *netdev = nesvnic->netdev;
-
-	/* props being zeroed by the caller, avoid zeroing it here */
-
-	props->max_mtu = IB_MTU_4096;
-	props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
-
-	props->lid = 1;
-	if (netif_queue_stopped(netdev))
-		props->state = IB_PORT_DOWN;
-	else if (nesvnic->linkup)
-		props->state = IB_PORT_ACTIVE;
-	else
-		props->state = IB_PORT_DOWN;
-	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
-			IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
-	props->gid_tbl_len = 1;
-	props->pkey_tbl_len = 1;
-	props->active_width = IB_WIDTH_4X;
-	props->active_speed = IB_SPEED_SDR;
-	props->max_msg_sz = 0x80000000;
-
-	return 0;
-}
-
-/**
- * nes_query_pkey
- */
-static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
-{
-	*pkey = 0;
-	return 0;
-}
-
-
-/**
- * nes_query_gid
- */
-static int nes_query_gid(struct ib_device *ibdev, u8 port,
-		int index, union ib_gid *gid)
-{
-	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-
-	memset(&(gid->raw[0]), 0, sizeof(gid->raw));
-	memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6);
-
-	return 0;
-}
-
-
-/**
- * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
- * of all objects associated with a particular user-mode client.
- */
-static int nes_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
-{
-	struct ib_device *ibdev = uctx->device;
-	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_alloc_ucontext_req req;
-	struct nes_alloc_ucontext_resp uresp = {};
-	struct nes_ucontext *nes_ucontext = to_nesucontext(uctx);
-	struct nes_ib_device *nesibdev = nesvnic->nesibdev;
-
-
-	if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) {
-		printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n");
-		return -EINVAL;
-	}
-
-	if (req.userspace_ver != NES_ABI_USERSPACE_VER) {
-		printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n",
-			req.userspace_ver, NES_ABI_USERSPACE_VER);
-		return -EINVAL;
-	}
-
-
-	uresp.max_qps = nesibdev->max_qp;
-	uresp.max_pds = nesibdev->max_pd;
-	uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2;
-	uresp.virtwq = nesadapter->virtwq;
-	uresp.kernel_ver = NES_ABI_KERNEL_VER;
-
-	nes_ucontext->nesdev = nesdev;
-	nes_ucontext->mmap_wq_offset = uresp.max_pds;
-	nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset +
-			((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) /
-			PAGE_SIZE;
-
-
-	if (ib_copy_to_udata(udata, &uresp, sizeof(uresp)))
-		return -EFAULT;
-
-	INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list);
-	INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list);
-	return 0;
-}
-
-/**
- * nes_dealloc_ucontext
- */
-static void nes_dealloc_ucontext(struct ib_ucontext *context)
-{
-	return;
-}
-
-/**
- * nes_mmap
- */
-static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
-{
-	unsigned long index;
-	struct nes_vnic *nesvnic = to_nesvnic(context->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	/* struct nes_adapter *nesadapter = nesdev->nesadapter; */
-	struct nes_ucontext *nes_ucontext;
-	struct nes_qp *nesqp;
-
-	nes_ucontext = to_nesucontext(context);
-
-
-	if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) {
-		index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE;
-		index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) +
-				PAGE_SIZE-1) & (~(PAGE_SIZE-1));
-		if (!test_bit(index, nes_ucontext->allocated_wqs)) {
-			nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index);
-			return -EFAULT;
-		}
-		nesqp = nes_ucontext->mmap_nesqp[index];
-		if (nesqp == NULL) {
-			nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index);
-			return -EFAULT;
-		}
-		if (remap_pfn_range(vma, vma->vm_start,
-				virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT,
-				vma->vm_end - vma->vm_start,
-				vma->vm_page_prot)) {
-			nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n");
-			return -EAGAIN;
-		}
-		vma->vm_private_data = nesqp;
-		return 0;
-	} else {
-		index = vma->vm_pgoff;
-		if (!test_bit(index, nes_ucontext->allocated_doorbells))
-			return -EFAULT;
-
-		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-		if (io_remap_pfn_range(vma, vma->vm_start,
-				(nesdev->doorbell_start +
-				((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096))
-				>> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot))
-			return -EAGAIN;
-		vma->vm_private_data = nes_ucontext;
-		return 0;
-	}
-
-	return -ENOSYS;
-}
-
-
-/**
- * nes_alloc_pd
- */
-static int nes_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
-{
-	struct ib_device *ibdev = pd->device;
-	struct nes_pd *nespd = to_nespd(pd);
-	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_alloc_pd_resp uresp;
-	u32 pd_num = 0;
-	int err;
-	struct nes_ucontext *nesucontext = rdma_udata_to_drv_context(
-		udata, struct nes_ucontext, ibucontext);
-
-	nes_debug(
-		NES_DBG_PD,
-		"nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
-		nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev,
-		&nesucontext->ibucontext, netdev_refcnt_read(nesvnic->netdev));
-
-	err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds,
-			nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD);
-	if (err)
-		return err;
-
-	nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n",
-			nespd, dev_name(&nesvnic->nesibdev->ibdev.dev));
-
-	nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd;
-
-	if (udata) {
-		nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells,
-				NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db);
-		nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
-				nespd->mmap_db_index, nespd->pd_id);
-		if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) {
-			nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n");
-			nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
-			return -ENOMEM;
-		}
-
-		uresp.pd_id = nespd->pd_id;
-		uresp.mmap_db_index = nespd->mmap_db_index;
-		if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) {
-			nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num);
-			return -EFAULT;
-		}
-
-		set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
-		nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id;
-		nesucontext->first_free_db = nespd->mmap_db_index + 1;
-	}
-
-	nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd);
-	return 0;
-}
-
-
-/**
- * nes_dealloc_pd
- */
-static void nes_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
-{
-	struct nes_ucontext *nesucontext;
-	struct nes_pd *nespd = to_nespd(ibpd);
-	struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-
-	if (udata) {
-		nesucontext =
-			rdma_udata_to_drv_context(
-				udata,
-				struct nes_ucontext,
-				ibucontext);
-		nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n",
-				nespd->mmap_db_index);
-		clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
-		nesucontext->mmap_db_index[nespd->mmap_db_index] = 0;
-		if (nesucontext->first_free_db > nespd->mmap_db_index) {
-			nesucontext->first_free_db = nespd->mmap_db_index;
-		}
-	}
-
-	nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n",
-			nespd->pd_id, nespd);
-	nes_free_resource(nesadapter, nesadapter->allocated_pds,
-			(nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12));
-}
-
-
-/**
- * nes_get_encoded_size
- */
-static inline u8 nes_get_encoded_size(int *size)
-{
-	u8 encoded_size = 0;
-	if (*size <= 32) {
-		*size = 32;
-		encoded_size = 1;
-	} else if (*size <= 128) {
-		*size = 128;
-		encoded_size = 2;
-	} else if (*size <= 512) {
-		*size = 512;
-		encoded_size = 3;
-	}
-	return (encoded_size);
-}
-
-
-
-/**
- * nes_setup_virt_qp
- */
-static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,
-		struct nes_vnic *nesvnic, int sq_size, int rq_size)
-{
-	unsigned long flags;
-	void *mem;
-	__le64 *pbl = NULL;
-	__le64 *tpbl;
-	__le64 *pblbuffer;
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	u32 pbl_entries;
-	u8 rq_pbl_entries;
-	u8 sq_pbl_entries;
-
-	pbl_entries = nespbl->pbl_size >> 3;
-	nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
-			nespbl->pbl_size, pbl_entries,
-			(void *)nespbl->pbl_vbase,
-			(unsigned long) nespbl->pbl_pbase);
-	pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */
-	/* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
-	/* the first pbl to be fro the rq_vbase... */
-	rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
-	sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12;
-	nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
-	if (!nespbl->page) {
-		nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n");
-		kfree(nespbl);
-		return -ENOMEM;
-	}
-
-	nesqp->hwqp.sq_vbase = kmap(nespbl->page);
-	nesqp->page = nespbl->page;
-	if (!nesqp->hwqp.sq_vbase) {
-		nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n");
-		kfree(nespbl);
-		return -ENOMEM;
-	}
-
-	/* Now to get to sq.. we need to calculate how many */
-	/* PBL entries were used by the rq.. */
-	pbl += sq_pbl_entries;
-	nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32);
-	/* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
-	/*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
-
-	nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
-		  nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,
-		  nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);
-	spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-	if (!nesadapter->free_256pbl) {
-		pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
-				nespbl->pbl_pbase);
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-		kunmap(nesqp->page);
-		kfree(nespbl);
-		return -ENOMEM;
-	}
-	nesadapter->free_256pbl--;
-	spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-
-	nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase);
-	pblbuffer = nesqp->pbl_vbase;
-	if (!nesqp->pbl_vbase) {
-		/* memory allocated during nes_reg_user_mr() */
-		pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
-				    nespbl->pbl_pbase);
-		kfree(nespbl);
-		spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-		nesadapter->free_256pbl++;
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-		kunmap(nesqp->page);
-		return -ENOMEM;
-	}
-	memset(nesqp->pbl_vbase, 0, 256);
-	/* fill in the page address in the pbl buffer.. */
-	tpbl = pblbuffer + 16;
-	pbl = (__le64 *)nespbl->pbl_vbase;
-	while (sq_pbl_entries--)
-		*tpbl++ = *pbl++;
-	tpbl = pblbuffer;
-	while (rq_pbl_entries--)
-		*tpbl++ = *pbl++;
-
-	/* done with memory allocated during nes_reg_user_mr() */
-	pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
-			    nespbl->pbl_pbase);
-	kfree(nespbl);
-
-	nesqp->qp_mem_size =
-			max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256;     /* this is Q2 */
-	/* Round up to a multiple of a page */
-	nesqp->qp_mem_size += PAGE_SIZE - 1;
-	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
-
-	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
-			&nesqp->hwqp.q2_pbase);
-
-	if (!mem) {
-		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);
-		nesqp->pbl_vbase = NULL;
-		spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-		nesadapter->free_256pbl++;
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-		kunmap(nesqp->page);
-		return -ENOMEM;
-	}
-	nesqp->sq_kmapped = 1;
-	nesqp->hwqp.q2_vbase = mem;
-	mem += 256;
-	memset(nesqp->hwqp.q2_vbase, 0, 256);
-	nesqp->nesqp_context = mem;
-	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
-	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
-
-	return 0;
-}
-
-
-/**
- * nes_setup_mmap_qp
- */
-static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic,
-		int sq_size, int rq_size)
-{
-	void *mem;
-	struct nes_device *nesdev = nesvnic->nesdev;
-
-	nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) +
-			(sizeof(struct nes_hw_qp_wqe) * rq_size) +
-			max((u32)sizeof(struct nes_qp_context), ((u32)256)) +
-			256; /* this is Q2 */
-	/* Round up to a multiple of a page */
-	nesqp->qp_mem_size += PAGE_SIZE - 1;
-	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);
-
-	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,
-			&nesqp->hwqp.sq_pbase);
-	if (!mem)
-		return -ENOMEM;
-	nes_debug(NES_DBG_QP, "PCI consistent memory for "
-			"host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
-			mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size);
-
-	memset(mem, 0, nesqp->qp_mem_size);
-
-	nesqp->hwqp.sq_vbase = mem;
-	mem += sizeof(struct nes_hw_qp_wqe) * sq_size;
-
-	nesqp->hwqp.rq_vbase = mem;
-	nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase +
-			sizeof(struct nes_hw_qp_wqe) * sq_size;
-	mem += sizeof(struct nes_hw_qp_wqe) * rq_size;
-
-	nesqp->hwqp.q2_vbase = mem;
-	nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase +
-			sizeof(struct nes_hw_qp_wqe) * rq_size;
-	mem += 256;
-	memset(nesqp->hwqp.q2_vbase, 0, 256);
-
-	nesqp->nesqp_context = mem;
-	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;
-	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));
-	return 0;
-}
-
-
-/**
- * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
- */
-static void nes_free_qp_mem(struct nes_device *nesdev,
-		struct nes_qp *nesqp, int virt_wqs)
-{
-	unsigned long flags;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	if (!virt_wqs) {
-		pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,
-				nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);
-	}else {
-		spin_lock_irqsave(&nesadapter->pbl_lock, flags);
-		nesadapter->free_256pbl++;
-		spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
-		pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase);
-		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );
-		nesqp->pbl_vbase = NULL;
-		if (nesqp->sq_kmapped) {
-			nesqp->sq_kmapped = 0;
-			kunmap(nesqp->page);
-		}
-	}
-}
-
-
-/**
- * nes_create_qp
- */
-static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
-		struct ib_qp_init_attr *init_attr, struct ib_udata *udata)
-{
-	u64 u64temp= 0;
-	u64 u64nesqp = 0;
-	struct nes_pd *nespd = to_nespd(ibpd);
-	struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
-	struct nes_device *nesdev = nesvnic->nesdev;
-	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_qp *nesqp;
-	struct nes_cq *nescq;
-	struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context(
-		udata, struct nes_ucontext, ibucontext);
-	struct nes_hw_cqp_wqe *cqp_wqe;
-	struct nes_cqp_request *cqp_request;
-	struct nes_create_qp_req req;
-	struct nes_create_qp_resp uresp;
-	struct nes_pbl  *nespbl = NULL;
-	u32 qp_num = 0;
-	u32 opcode = 0;
-	/* u32 counter = 0; */
-	void *mem;
-	unsigned long flags;
-	int ret;
-	int err;
-	int virt_wqs = 0;
-	int sq_size;
-	int rq_size;
-	u8 sq_encoded_size;
-	u8 rq_encoded_size;
-	/* int counter; */
-
-	if (init_attr->create_flags)
-		return ERR_PTR(-EINVAL);
-
-	atomic_inc(&qps_created);
-	switch (init_attr->qp_type) {
-		case IB_QPT_RC:
-			if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) {
-				init_attr->cap.max_inline_data = 0;
-			} else {
-				init_attr->cap.max_inline_data = 64;
-			}
-			sq_size = init_attr->cap.max_send_wr;
-			rq_size = init_attr->cap.max_recv_wr;
-
-			/* check if the encoded sizes are OK or not... */
-			sq_encoded_size = nes_get_encoded_size(&sq_size);
-			rq_encoded_size = nes_get_encoded_size(&rq_size);
-
-			if ((!sq_encoded_size) || (!rq_encoded_size)) {
-				nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n",
-						rq_size, sq_size);
-				return ERR_PTR(-EINVAL);
-			}
-
-			init_attr->cap.max_send_wr = sq_size -2;
-			init_attr->cap.max_recv_wr = rq_size -1;
-			nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size);
-
-			ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps,
-					nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP);
-			if (ret) {
-				return ERR_PTR(ret);
-			}
-
-			/* Need 512 (actually now 1024)