1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _ISER_RESOURCE_H 27 #define _ISER_RESOURCE_H 28 29 #ifdef __cplusplus 30 extern "C" { 31 #endif 32 33 #include <sys/types.h> 34 #include <sys/ib/ibtl/ibti.h> 35 #include <sys/ib/ibtl/ibtl_types.h> 36 #include <sys/iscsi_protocol.h> 37 38 #define ISER_CACHE_NAMELEN 31 /* KMEM_CACHE_NAMELEN */ 39 40 /* Default message lengths */ 41 #define ISER_MAX_CTRLPDU_LEN 0x4000 42 #define ISER_MAX_TEXTPDU_LEN 0x4000 43 44 /* Default data buffer length */ 45 #define ISER_DEFAULT_BUFLEN 0x20000 46 47 /* 48 * iser_resource.h 49 * Definitions and functions related to set up buffer allocation from 50 * IBT memory regions and managment of work requessts. 51 */ 52 53 struct iser_hca_s; 54 55 /* 56 * Memory regions 57 */ 58 typedef struct iser_mr_s { 59 ibt_mr_hdl_t is_mrhdl; 60 ib_vaddr_t is_mrva; 61 ib_memlen_t is_mrlen; 62 ibt_lkey_t is_mrlkey; 63 ibt_rkey_t is_mrrkey; 64 avl_node_t is_avl_ln; 65 } iser_mr_t; 66 67 typedef struct iser_vmem_mr_pool_s { 68 iser_hca_t *ivmp_hca; 69 ibt_mr_flags_t ivmp_mr_flags; 70 ib_memlen_t ivmp_chunksize; 71 vmem_t *ivmp_vmem; 72 uint64_t ivmp_total_size; 73 uint64_t ivmp_max_total_size; 74 avl_tree_t ivmp_mr_list; 75 kmutex_t ivmp_mutex; 76 } iser_vmem_mr_pool_t; 77 78 #define ISER_MR_QUANTSIZE 0x400 79 #define ISER_MIN_CHUNKSIZE 0x100000 /* 1MB */ 80 81 #ifdef _LP64 82 #define ISER_BUF_MR_CHUNKSIZE 0x8000000 /* 128MB */ 83 #define ISER_BUF_POOL_MAX 0x40000000 /* 1GB */ 84 #else 85 /* Memory is very limited on 32-bit kernels */ 86 #define ISER_BUF_MR_CHUNKSIZE 0x400000 /* 4MB */ 87 #define ISER_BUF_POOL_MAX 0x4000000 /* 64MB */ 88 #endif 89 #define ISER_BUF_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE | \ 90 IBT_MR_ENABLE_REMOTE_READ | IBT_MR_ENABLE_REMOTE_WRITE 91 #ifdef _LP64 92 #define ISER_MSG_MR_CHUNKSIZE 0x2000000 /* 32MB */ 93 #define ISER_MSG_POOL_MAX 0x10000000 /* 256MB */ 94 #else 95 #define ISER_MSG_MR_CHUNKSIZE 0x100000 /* 1MB */ 96 #define ISER_MSG_POOL_MAX 0x2000000 /* 32MB */ 97 #endif 98 #define ISER_MSG_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE 99 100 iser_vmem_mr_pool_t *iser_vmem_create(const char *name, iser_hca_t *hca, 101 ib_memlen_t chunksize, uint64_t max_total_size, 102 ibt_mr_flags_t arena_mr_flags); 103 void iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool); 104 void *iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size); 105 void iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size); 106 idm_status_t iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool, 107 void *vaddr, size_t size, iser_mr_t *mr); 108 109 /* 110 * iSER work request structure encodes an iSER Send Queue work request 111 * context, with pointers to relevant resources related to the work request. 112 * We hold a pointer to either an IDM PDU handle, an iSER message handle 113 * or an IDM buffer handle. These are allocated from a kmem_cache when 114 * we post send WR's, and freed back when the completion is polled. 115 */ 116 typedef enum { 117 ISER_WR_SEND, 118 ISER_WR_RDMAW, 119 ISER_WR_RDMAR, 120 ISER_WR_UNDEFINED 121 } iser_wr_type_t; 122 123 typedef struct iser_wr_s { 124 iser_wr_type_t iw_type; 125 struct iser_msg_s *iw_msg; 126 struct idm_buf_s *iw_buf; 127 struct idm_pdu_s *iw_pdu; 128 } iser_wr_t; 129 130 int iser_wr_cache_constructor(void *mr, void *arg, int flags); 131 void iser_wr_cache_destructor(void *mr, void *arg); 132 iser_wr_t *iser_wr_get(); 133 void iser_wr_free(iser_wr_t *iser_wr); 134 135 /* 136 * iSER message structure for iSCSI Control PDUs, constructor and 137 * destructor routines, and utility routines for allocating and 138 * freeing message handles. 139 */ 140 typedef struct iser_msg_s { 141 struct iser_msg_s *nextp; /* for building lists */ 142 kmem_cache_t *cache; /* back pointer for cleanup */ 143 ibt_wr_ds_t msg_ds; /* SGEs for hdr and text */ 144 ibt_mr_hdl_t mrhdl[2]; /* MR handles for each SGE */ 145 } iser_msg_t; 146 147 int iser_msg_cache_constructor(void *mr, void *arg, int flags); 148 void iser_msg_cache_destructor(void *mr, void *arg); 149 iser_msg_t *iser_msg_get(iser_hca_t *hca, int num, int *ret); 150 void iser_msg_free(iser_msg_t *msg); 151 152 /* 153 * iSER data buffer structure for iSER RDMA operations, constructor and 154 * destructor routines, and utility routines for allocating and freeing 155 * buffer handles. 156 */ 157 typedef struct iser_buf_s { 158 kmem_cache_t *cache; /* back pointer for cleanup */ 159 void *buf; /* buffer */ 160 uint64_t buflen; 161 iser_mr_t *iser_mr; /* MR handle for this buffer */ 162 ibt_wr_ds_t buf_ds; /* SGE for this buffer */ 163 ibt_send_wr_t buf_wr; /* DEBUG, copy of wr from request */ 164 ibt_wc_t buf_wc; /* DEBUG, copy of wc from completion */ 165 timespec_t buf_constructed; 166 timespec_t buf_destructed; 167 } iser_buf_t; 168 169 int iser_buf_cache_constructor(void *mr, void *arg, int flags); 170 void iser_buf_cache_destructor(void *mr, void *arg); 171 172 void iser_init_hca_caches(struct iser_hca_s *hca); 173 void iser_fini_hca_caches(struct iser_hca_s *hca); 174 175 /* Routines to register in-place memory passed on an existing idb */ 176 int iser_reg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); 177 void iser_dereg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); 178 179 #ifdef __cplusplus 180 } 181 #endif 182 183 #endif /* _ISER_RESOURCE_H */ 184