1 /* 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 3 */ 4 5 /* 6 * This file contains code imported from the OFED rds source file ib_ring.c 7 * Oracle elects to have and use the contents of ib_ring.c under and governed 8 * by the OpenIB.org BSD license (see below for full license text). However, 9 * the following notice accompanied the original version of this file: 10 */ 11 12 /* 13 * Copyright (c) 2006 Oracle. All rights reserved. 14 * 15 * This software is available to you under a choice of one of two 16 * licenses. You may choose to be licensed under the terms of the GNU 17 * General Public License (GPL) Version 2, available from the file 18 * COPYING in the main directory of this source tree, or the 19 * OpenIB.org BSD license below: 20 * 21 * Redistribution and use in source and binary forms, with or 22 * without modification, are permitted provided that the following 23 * conditions are met: 24 * 25 * - Redistributions of source code must retain the above 26 * copyright notice, this list of conditions and the following 27 * disclaimer. 28 * 29 * - Redistributions in binary form must reproduce the above 30 * copyright notice, this list of conditions and the following 31 * disclaimer in the documentation and/or other materials 32 * provided with the distribution. 33 * 34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 41 * SOFTWARE. 42 * 43 */ 44 #include <sys/rds.h> 45 46 #include <sys/ib/clients/rdsv3/rdsv3.h> 47 #include <sys/ib/clients/rdsv3/ib.h> 48 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 49 50 /* 51 * Locking for IB rings. 52 * We assume that allocation is always protected by a mutex 53 * in the caller (this is a valid assumption for the current 54 * implementation). 55 * 56 * Freeing always happens in an interrupt, and hence only 57 * races with allocations, but not with other free()s. 58 * 59 * The interaction between allocation and freeing is that 60 * the alloc code has to determine the number of free entries. 61 * To this end, we maintain two counters; an allocation counter 62 * and a free counter. Both are allowed to run freely, and wrap 63 * around. 64 * The number of used entries is always (alloc_ctr - free_ctr) % NR. 65 * 66 * The current implementation makes free_ctr atomic. When the 67 * caller finds an allocation fails, it should set an "alloc fail" 68 * bit and retry the allocation. The "alloc fail" bit essentially tells 69 * the CQ completion handlers to wake it up after freeing some 70 * more entries. 71 */ 72 73 void 74 rdsv3_ib_ring_init(struct rdsv3_ib_work_ring *ring, uint32_t nr) 75 { 76 (void) memset(ring, 0, sizeof (*ring)); 77 ring->w_nr = nr; 78 RDSV3_DPRINTF5("rdsv3_ib_ring_init", "ring %p nr %u", ring, ring->w_nr); 79 } 80 81 static inline uint32_t 82 __rdsv3_ib_ring_used(struct rdsv3_ib_work_ring *ring) 83 { 84 uint32_t diff; 85 86 /* This assumes that atomic_t has at least as many bits as uint32_t */ 87 diff = ring->w_alloc_ctr - (uint32_t)atomic_get(&ring->w_free_ctr); 88 ASSERT(diff <= ring->w_nr); 89 90 return (diff); 91 } 92 93 void 94 rdsv3_ib_ring_resize(struct rdsv3_ib_work_ring *ring, uint32_t nr) 95 { 96 /* 97 * We only ever get called from the connection setup code, 98 * prior to creating the QP. 99 */ 100 ASSERT(!__rdsv3_ib_ring_used(ring)); 101 ring->w_nr = nr; 102 } 103 104 static int 105 __rdsv3_ib_ring_empty(struct rdsv3_ib_work_ring *ring) 106 { 107 return (__rdsv3_ib_ring_used(ring) == 0); 108 } 109 110 uint32_t 111 rdsv3_ib_ring_alloc(struct rdsv3_ib_work_ring *ring, uint32_t val, 112 uint32_t *pos) 113 { 114 uint32_t ret = 0, avail; 115 116 avail = ring->w_nr - __rdsv3_ib_ring_used(ring); 117 118 RDSV3_DPRINTF5("rdsv3_ib_ring_alloc", 119 "ring %p val %u next %u free %u", ring, val, 120 ring->w_alloc_ptr, avail); 121 122 if (val && avail) { 123 ret = min(val, avail); 124 *pos = ring->w_alloc_ptr; 125 126 ring->w_alloc_ptr = (ring->w_alloc_ptr + ret) % ring->w_nr; 127 ring->w_alloc_ctr += ret; 128 } 129 130 return (ret); 131 } 132 133 void 134 rdsv3_ib_ring_free(struct rdsv3_ib_work_ring *ring, uint32_t val) 135 { 136 ring->w_free_ptr = (ring->w_free_ptr + val) % ring->w_nr; 137 atomic_add_32(&ring->w_free_ctr, val); 138 139 if (__rdsv3_ib_ring_empty(ring)) 140 rdsv3_wake_up(&ring->w_empty_wait); 141 } 142 143 void 144 rdsv3_ib_ring_unalloc(struct rdsv3_ib_work_ring *ring, uint32_t val) 145 { 146 ring->w_alloc_ptr = (ring->w_alloc_ptr - val) % ring->w_nr; 147 ring->w_alloc_ctr -= val; 148 } 149 150 int 151 rdsv3_ib_ring_empty(struct rdsv3_ib_work_ring *ring) 152 { 153 return (__rdsv3_ib_ring_empty(ring)); 154 } 155 156 int 157 rdsv3_ib_ring_low(struct rdsv3_ib_work_ring *ring) 158 { 159 return (__rdsv3_ib_ring_used(ring) <= (ring->w_nr >> 2)); 160 } 161 162 /* 163 * returns the oldest alloced ring entry. This will be the next one 164 * freed. This can't be called if there are none allocated. 165 */ 166 uint32_t 167 rdsv3_ib_ring_oldest(struct rdsv3_ib_work_ring *ring) 168 { 169 return (ring->w_free_ptr); 170 } 171 172 /* 173 * returns the number of completed work requests. 174 */ 175 176 uint32_t 177 rdsv3_ib_ring_completed(struct rdsv3_ib_work_ring *ring, 178 uint32_t wr_id, uint32_t oldest) 179 { 180 uint32_t ret; 181 182 if (oldest <= (unsigned long long)wr_id) 183 ret = (unsigned long long)wr_id - oldest + 1; 184 else 185 ret = ring->w_nr - oldest + (unsigned long long)wr_id + 1; 186 187 RDSV3_DPRINTF5("rdsv3_ib_ring_completed", 188 "ring %p ret %u wr_id %u oldest %u", ring, ret, wr_id, oldest); 189 return (ret); 190 } 191