xref: /illumos-gate/usr/src/uts/common/io/ib/clients/iser/iser_resource.c (revision 30e7468f8f41aa30ada067b2c1d5d284046514da)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/sunddi.h>
#include <sys/sysmacros.h>
#include <sys/ib/ibtl/ibti.h>
#include <sys/ib/ibtl/ibtl_types.h>

#include <sys/ib/clients/iser/iser.h>

/*
 * iser_resource.c
 *    Routines for allocating resources for iSER
 */

static iser_mr_t *iser_vmem_chunk_alloc(iser_hca_t *hca, ib_memlen_t chunksize,
    ibt_mr_flags_t mr_flags);

static void iser_vmem_chunk_free(iser_hca_t *hca, iser_mr_t *iser_mr);

static iser_mr_t *iser_reg_mem(iser_hca_t *hca, ib_vaddr_t vaddr,
    ib_memlen_t len, ibt_mr_flags_t mr_flags);

static void iser_dereg_mem(iser_hca_t *hca, iser_mr_t *mr);

static int iser_vmem_mr_compare(const void *void_mr1, const void *void_mr2);

/*
 * iser_init_hca_caches()
 * Invoked per HCA instance initialization, to establish HCA-wide
 * message and buffer kmem caches. Note we'll uniquify cache names
 * with the lower 32-bits of the HCA GUID.
 */
void
iser_init_hca_caches(iser_hca_t *hca)
{
	char name[ISER_CACHE_NAMELEN];

	(void) snprintf(name, ISER_CACHE_NAMELEN, "iser_msg_pool_%08x",
	    (uint32_t)(hca->hca_guid & 0xFFFFFFFF));
	hca->hca_msg_pool = iser_vmem_create(name, hca, ISER_MSG_MR_CHUNKSIZE,
	    ISER_MSG_POOL_MAX, ISER_MSG_MR_FLAGS);
	(void) snprintf(name, ISER_CACHE_NAMELEN, "iser_msg_cache_%08x",
	    (uint32_t)(hca->hca_guid & 0xFFFFFFFF));
	hca->iser_msg_cache = kmem_cache_create(name, sizeof (iser_msg_t),
	    0, &iser_msg_cache_constructor, &iser_msg_cache_destructor,
	    NULL, hca, NULL, KM_SLEEP);

	(void) snprintf(name, ISER_CACHE_NAMELEN, "iser_buf_pool_%08x",
	    (uint32_t)(hca->hca_guid & 0xFFFFFFFF));
	hca->hca_buf_pool = iser_vmem_create(name, hca, ISER_BUF_MR_CHUNKSIZE,
	    ISER_BUF_POOL_MAX, ISER_BUF_MR_FLAGS);
	(void) snprintf(name, ISER_CACHE_NAMELEN, "iser_buf_cache_%08x",
	    (uint32_t)(hca->hca_guid & 0xFFFFFFFF));
	hca->iser_buf_cache = kmem_cache_create(name, sizeof (iser_buf_t),
	    0, &iser_buf_cache_constructor, &iser_buf_cache_destructor,
	    NULL, hca, NULL, KM_SLEEP);
}

/*
 * iser_fini_hca_caches()
 * Invoked per HCA instance teardown, this routine cleans up the
 * message and buffer handle caches.
 */
void
iser_fini_hca_caches(iser_hca_t *hca)
{
	kmem_cache_destroy(hca->iser_buf_cache);
	iser_vmem_destroy(hca->hca_buf_pool);
	kmem_cache_destroy(hca->iser_msg_cache);
	iser_vmem_destroy(hca->hca_msg_pool);
}

/*
 * Allocate and initialize an iSER WR handle
 */
iser_wr_t *
iser_wr_get()
{
	iser_wr_t	*iser_wr;

	iser_wr = kmem_cache_alloc(iser_state->iser_wr_cache, KM_NOSLEEP);
	if (iser_wr != NULL) {
		iser_wr->iw_type = ISER_WR_UNDEFINED;
		iser_wr->iw_msg  = NULL;
		iser_wr->iw_buf  = NULL;
		iser_wr->iw_pdu  = NULL;
	}

	return (iser_wr);
}

/*
 * Free an iSER WR handle back to the global cache
 */
void
iser_wr_free(iser_wr_t *iser_wr)
{
	kmem_cache_free(iser_state->iser_wr_cache, iser_wr);
}

/*
 * iser_msg_cache_constructor()
 * Allocate and register memory for an iSER Control-type PDU message.
 * The cached objects will retain this memory registration in the HCA,
 * and thus provide a cache of pre-allocated and registered messages
 * for use in iSER.
 */
/* ARGSUSED */
int
iser_msg_cache_constructor(void *msg_void, void *arg, int flags)
{
	void		*memp = NULL;
	int		status;
	iser_msg_t	*msg = (iser_msg_t *)msg_void;
	iser_hca_t	*hca = (iser_hca_t *)arg;
	iser_mr_t	mr;

	memp = iser_vmem_alloc(hca->hca_msg_pool, ISER_MAX_CTRLPDU_LEN);
	if (memp == NULL) {
		ISER_LOG(CE_NOTE, "iser_msg_cache_constructor: "
		    "failed to allocate backing memory");
		return (DDI_FAILURE);
	}

	/* Fill in iser_mr for the memory we just allocated */
	status = iser_vmem_mr(hca->hca_msg_pool, memp,
	    ISER_MAX_CTRLPDU_LEN, &mr);
	if (status != IDM_STATUS_SUCCESS) {
		ISER_LOG(CE_NOTE, "iser_msg_cache_constructor: "
		    "couldn't find mr for %p", memp);
		iser_vmem_free(hca->hca_msg_pool, memp, ISER_MAX_CTRLPDU_LEN);
		return (DDI_FAILURE);
	}

	msg->msg_ds.ds_va	= (ib_vaddr_t)(uintptr_t)memp;
	msg->msg_ds.ds_key	= mr.is_mrlkey;

	/* Set a backpointer to this cache to save a lookup on free */
	msg->cache = hca->iser_msg_cache;

	return (DDI_SUCCESS);
}

/*
 * Deregister and free registered memory from an iser_msg_t handle.
 */
void
iser_msg_cache_destructor(void *mr, void *arg)
{
	iser_msg_t	*msg = (iser_msg_t *)mr;
	iser_hca_t	*hca = (iser_hca_t *)arg;
	uint8_t		*memp;

	memp = (uint8_t *)(uintptr_t)(ib_vaddr_t)msg->msg_ds.ds_va;
	iser_vmem_free(hca->hca_msg_pool, memp, ISER_MAX_CTRLPDU_LEN);
}

/*
 * Pull a msg handle off of hca's msg cache. If no object is available
 * on the cache, a new message buffer will be allocated and registered
 * with the HCA. Once freed, this message will not be unregistered, thus
 * building up a cache of pre-allocated and registered message buffers
 * over time.
 */
iser_msg_t *
iser_msg_get(iser_hca_t *hca, int num, int *ret)
{
	iser_msg_t	*tmp, *msg = NULL;
	int i;

	ASSERT(hca != NULL);

	/*
	 * Pull num number of message handles off the cache, linking
	 * them if more than one have been requested.
	 */
	for (i = 0; i < num; i++) {
		tmp = kmem_cache_alloc(hca->iser_msg_cache, KM_NOSLEEP);
		if (tmp == NULL) {
			ISER_LOG(CE_NOTE, "iser_msg_get: alloc failed, "
			    "requested (%d) allocated (%d)", num, i);
			break;
		}
		tmp->msg_ds.ds_len	= ISER_MAX_CTRLPDU_LEN;
		tmp->nextp = msg;
		msg = tmp;
	}

	if (ret != NULL) {
		*ret = i;
	}

	return (msg);
}

/*
 * Free this msg back to its cache, leaving the memory contained by
 * it registered for later re-use.
 */
void
iser_msg_free(iser_msg_t *msg)
{
	kmem_cache_free(msg->cache, msg);
}

/*
 * iser_buf_cache_constructor()
 * Allocate and register memory for an iSER RDMA operation. The cached
 * objects will retain this memory registration in the HCA, and thus
 * provide a cache of pre-allocated and registered messages for use in
 * iSER.
 */
/* ARGSUSED */
int
iser_buf_cache_constructor(void *mr, void *arg, int flags)
{
	uint8_t		*memp;
	idm_status_t	status;
	iser_buf_t	*iser_buf = (iser_buf_t *)mr;
	iser_hca_t	*hca = (iser_hca_t *)arg;

	/* Allocate an iser_mr handle for this buffer */
	iser_buf->iser_mr = kmem_zalloc(sizeof (iser_mr_t), KM_NOSLEEP);
	if (iser_buf->iser_mr == NULL) {
		ISER_LOG(CE_NOTE, "iser_buf_cache_constructor: "
		    "failed to allocate memory for iser_mr handle");
		return (DDI_FAILURE);
	}

	memp = iser_vmem_alloc(hca->hca_buf_pool, ISER_DEFAULT_BUFLEN);
	if (memp == NULL) {
		kmem_free(iser_buf->iser_mr, sizeof (iser_mr_t));
		return (DDI_FAILURE);
	}

	/* Fill in iser_mr for the memory we just allocated */
	status = iser_vmem_mr(hca->hca_buf_pool, memp, ISER_DEFAULT_BUFLEN,
	    iser_buf->iser_mr);

	if (status != IDM_STATUS_SUCCESS) {
		return (DDI_FAILURE);
	}

	/* Set buf pointer and len for later manipulation (if necessary) */
	iser_buf->buf		= (uint64_t *)(uintptr_t)memp;
	iser_buf->buflen	= ISER_DEFAULT_BUFLEN;

	/* Populate the SGE Vaddr and L_key for the xfer operation later */
	iser_buf->buf_ds.ds_va	= iser_buf->iser_mr->is_mrva;
	iser_buf->buf_ds.ds_key	= iser_buf->iser_mr->is_mrlkey;

	/* Set a backpointer to this cache to save a lookup on free */
	iser_buf->cache = hca->iser_buf_cache;

	gethrestime(&iser_buf->buf_constructed);

	return (DDI_SUCCESS);
}

/*
 * Deregister and free registered memory from an iser_buf_t handle.
 */
void
iser_buf_cache_destructor(void *mr, void *arg)
{
	iser_buf_t	*iser_buf = (iser_buf_t *)mr;
	iser_hca_t	*hca = (iser_hca_t *)arg;

	gethrestime(&iser_buf->buf_destructed);

	iser_vmem_free(hca->hca_buf_pool, iser_buf->buf, iser_buf->buflen);

	kmem_free(iser_buf->iser_mr, sizeof (iser_mr_t));
}

/*
 * Registration for initiator buffers
 */
int
iser_reg_rdma_mem(iser_hca_t *hca, idm_buf_t *idb)
{
	iser_mr_t	*iser_mr = NULL;

	ASSERT(idb != NULL);
	ASSERT(idb->idb_buflen > 0);

	iser_mr = iser_reg_mem(hca, (ib_vaddr_t)(uintptr_t)idb->idb_buf,
	    idb->idb_buflen, ISER_BUF_MR_FLAGS | IBT_MR_NOSLEEP);
	if (iser_mr == NULL) {
		ISER_LOG(CE_NOTE, "iser_reg_rdma_mem: failed to register "
		    "memory for idm_buf_t");
		return (DDI_FAILURE);
	}

	idb->idb_reg_private	= (void *)iser_mr;

	return (DDI_SUCCESS);
}

void
iser_dereg_rdma_mem(iser_hca_t *hca, idm_buf_t *idb)
{
	iser_mr_t	*mr;

	ASSERT(idb != NULL);
	mr = (iser_mr_t *)idb->idb_reg_private;

	iser_dereg_mem(hca, mr);
}

iser_vmem_mr_pool_t *
iser_vmem_create(const char *name, iser_hca_t *hca, ib_memlen_t chunksize,
    uint64_t max_total_size, ibt_mr_flags_t arena_mr_flags)
{
	iser_mr_t		*first_chunk;
	iser_vmem_mr_pool_t	*result;

	ASSERT(chunksize <= max_total_size);
	result = kmem_zalloc(sizeof (*result), KM_SLEEP);
	result->ivmp_hca = hca;
	result->ivmp_mr_flags = arena_mr_flags;
	result->ivmp_chunksize = chunksize;
	result->ivmp_max_total_size = max_total_size;
	mutex_init(&result->ivmp_mutex, NULL, MUTEX_DRIVER, NULL);
	avl_create(&result->ivmp_mr_list, iser_vmem_mr_compare,
	    sizeof (iser_mr_t), offsetof(iser_mr_t, is_avl_ln));

	first_chunk = iser_vmem_chunk_alloc(hca, chunksize,
	    arena_mr_flags | IBT_MR_SLEEP);

	avl_add(&result->ivmp_mr_list, first_chunk);
	result->ivmp_total_size += chunksize;

	result->ivmp_vmem = vmem_create(name,
	    (void *)(uintptr_t)first_chunk->is_mrva,
	    (size_t)first_chunk->is_mrlen, ISER_MR_QUANTSIZE,
	    NULL, NULL, NULL, 0, VM_SLEEP);

	return (result);
}

void
iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool)
{
	iser_mr_t	*chunk, *next_chunk;

	mutex_enter(&vmr_pool->ivmp_mutex);
	vmem_destroy(vmr_pool->ivmp_vmem);

	for (chunk = avl_first(&vmr_pool->ivmp_mr_list); chunk != NULL;
	    chunk = next_chunk) {
		next_chunk = AVL_NEXT(&vmr_pool->ivmp_mr_list, chunk);
		avl_remove(&vmr_pool->ivmp_mr_list, chunk);
		iser_vmem_chunk_free(vmr_pool->ivmp_hca, chunk);
	}
	mutex_exit(&vmr_pool->ivmp_mutex);

	avl_destroy(&vmr_pool->ivmp_mr_list);
	mutex_destroy(&vmr_pool->ivmp_mutex);

	kmem_free(vmr_pool, sizeof (*vmr_pool));
}

void *
iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size)
{
	void		*result;
	iser_mr_t	*next_chunk;
	ib_memlen_t	chunk_len;
	result = vmem_alloc(vmr_pool->ivmp_vmem, size,
	    VM_NOSLEEP | VM_FIRSTFIT);
	if (result == NULL) {
		mutex_enter(&vmr_pool->ivmp_mutex);
		chunk_len = vmr_pool->ivmp_chunksize;
		if ((vmr_pool->ivmp_total_size + chunk_len) >
		    vmr_pool->ivmp_max_total_size) {
			/*
			 * Don't go over the pool size limit.  We can allocate
			 * partial chunks so it's not always the case that
			 * current_size + chunk_size == max_total_size
			 */
			if (vmr_pool->ivmp_total_size >=
			    vmr_pool->ivmp_max_total_size) {
				mutex_exit(&vmr_pool->ivmp_mutex);
				return (NULL);
			} else {
				chunk_len = vmr_pool->ivmp_max_total_size -
				    vmr_pool->ivmp_total_size;
			}
		}
		next_chunk = iser_vmem_chunk_alloc(vmr_pool->ivmp_hca,
		    chunk_len, vmr_pool->ivmp_mr_flags | IBT_MR_NOSLEEP);
		if (next_chunk != NULL) {
			if (vmem_add(vmr_pool->ivmp_vmem,
			    (void *)(uintptr_t)next_chunk->is_mrva,
			    next_chunk->is_mrlen, VM_NOSLEEP) == NULL) {
				/* Free the chunk we just allocated */
				iser_vmem_chunk_free(vmr_pool->ivmp_hca,
				    next_chunk);
			} else {
				vmr_pool->ivmp_total_size +=
				    next_chunk->is_mrlen;
				avl_add(&vmr_pool->ivmp_mr_list, next_chunk);
			}

			result = vmem_alloc(vmr_pool->ivmp_vmem, size,
			    VM_NOSLEEP | VM_FIRSTFIT);
		}

		mutex_exit(&vmr_pool->ivmp_mutex);
	}

	return (result);
}


void
iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size)
{
	vmem_free(vmr_pool->ivmp_vmem, vaddr, size);
}

idm_status_t
iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size,
    iser_mr_t *mr)
{
	avl_index_t	where;
	ib_vaddr_t	mrva = (ib_vaddr_t)(uintptr_t)vaddr;
	iser_mr_t	search_chunk;
	iser_mr_t	*nearest_chunk;
	ib_vaddr_t	chunk_end;

	mutex_enter(&vmr_pool->ivmp_mutex);
	search_chunk.is_mrva = mrva;
	nearest_chunk = avl_find(&vmr_pool->ivmp_mr_list, &search_chunk,
	    &where);
	if (nearest_chunk == NULL) {
		nearest_chunk = avl_nearest(&vmr_pool->ivmp_mr_list, where,
		    AVL_BEFORE);
		if (nearest_chunk == NULL) {
			mutex_exit(&vmr_pool->ivmp_mutex);
			return (IDM_STATUS_FAIL);
		}
	}

	/* See if this chunk contains the specified address range */
	ASSERT(nearest_chunk->is_mrva <= mrva);
	chunk_end = nearest_chunk->is_mrva + nearest_chunk->is_mrlen;
	if (chunk_end >= mrva + size) {
		/* Yes, this chunk contains the address range */
		mr->is_mrhdl = nearest_chunk->is_mrhdl;
		mr->is_mrva = mrva;
		mr->is_mrlen = size;
		mr->is_mrlkey = nearest_chunk->is_mrlkey;
		mr->is_mrrkey = nearest_chunk->is_mrrkey;
		mutex_exit(&vmr_pool->ivmp_mutex);
		return (IDM_STATUS_SUCCESS);
	}
	mutex_exit(&vmr_pool->ivmp_mutex);

	return (IDM_STATUS_FAIL);
}

static iser_mr_t *
iser_vmem_chunk_alloc(iser_hca_t *hca, ib_memlen_t chunksize,
    ibt_mr_flags_t mr_flags)
{
	void		*chunk = NULL;
	iser_mr_t	*result = NULL;
	int		km_flags = 0;

	if (mr_flags & IBT_MR_NOSLEEP)
		km_flags |= KM_NOSLEEP;

	while ((chunk == NULL) && (chunksize >= ISER_MIN_CHUNKSIZE)) {
		chunk = kmem_alloc(chunksize, km_flags);
		if (chunk == NULL) {
			ISER_LOG(CE_NOTE, "iser_vmem_chunk_alloc: "
			    "chunk alloc of %d failed, trying %d",
			    (int)chunksize, (int)(chunksize / 2));
			chunksize /= 2;
		} else {
			ISER_LOG(CE_NOTE, "iser_vmem_chunk_alloc: "
			    "New chunk %p size %d", chunk, (int)chunksize);
		}
	}

	if (chunk != NULL) {
		result = iser_reg_mem(hca, (ib_vaddr_t)(uintptr_t)chunk,
		    chunksize, mr_flags);
		if (result == NULL) {
			ISER_LOG(CE_NOTE, "iser_vmem_chunk_alloc: "
			    "Chunk registration failed");
			kmem_free(chunk, chunksize);
		}
	}

	return (result);
}

static void
iser_vmem_chunk_free(iser_hca_t *hca, iser_mr_t *iser_mr)
{
	void		*chunk		= (void *)(uintptr_t)iser_mr->is_mrva;
	ib_memlen_t	chunksize	= iser_mr->is_mrlen;

	iser_dereg_mem(hca, iser_mr);

	kmem_free(chunk, chunksize);
}

iser_mr_t *
iser_reg_mem(iser_hca_t *hca, ib_vaddr_t vaddr, ib_memlen_t len,
    ibt_mr_flags_t mr_flags)
{
	iser_mr_t	*result = NULL;
	ibt_mr_attr_t   mr_attr;
	ibt_mr_desc_t	mr_desc;
	ibt_status_t	status;
	int		km_flags = 0;

	if (mr_flags & IBT_MR_NOSLEEP)
		mr_flags |= KM_NOSLEEP;

	result = (iser_mr_t *)kmem_zalloc(sizeof (iser_mr_t), km_flags);
	if (result == NULL) {
		ISER_LOG(CE_NOTE, "iser_reg_mem: failed to allocate "
		    "memory for iser_mr handle");
		return (NULL);
	}

	bzero(&mr_attr, sizeof (ibt_mr_attr_t));
	bzero(&mr_desc, sizeof (ibt_mr_desc_t));

	mr_attr.mr_vaddr	= vaddr;
	mr_attr.mr_len		= len;
	mr_attr.mr_as		= NULL;
	mr_attr.mr_flags	= mr_flags;

	status = ibt_register_mr(hca->hca_hdl, hca->hca_pdhdl, &mr_attr,
	    &result->is_mrhdl, &mr_desc);
	if (status != IBT_SUCCESS) {
		ISER_LOG(CE_NOTE, "iser_reg_mem: ibt_register_mr "
		    "failure (%d)", status);
		kmem_free(result, sizeof (iser_mr_t));
		return (NULL);
	}

	result->is_mrva		= mr_attr.mr_vaddr;
	result->is_mrlen	= mr_attr.mr_len;
	result->is_mrlkey	= mr_desc.md_lkey;
	result->is_mrrkey	= mr_desc.md_rkey;

	return (result);
}

void
iser_dereg_mem(iser_hca_t *hca, iser_mr_t *mr)
{
	ibt_deregister_mr(hca->hca_hdl, mr->is_mrhdl);
	kmem_free(mr, sizeof (iser_mr_t));
}

static int
iser_vmem_mr_compare(const void *void_mr1, const void *void_mr2)
{
	iser_mr_t *mr1 = (iser_mr_t *)void_mr1;
	iser_mr_t *mr2 = (iser_mr_t *)void_mr2;

	/* Sort memory chunks by their virtual address */
	if (mr1->is_mrva < mr2->is_mrva)
		return (-1);
	else if (mr1->is_mrva > mr2->is_mrva)
		return (1);

	return (0);
}