1 2 #include "lm5710.h" 3 #include "bd_chain.h" 4 #include "command.h" 5 #include "context.h" 6 #include "lm_l4fp.h" 7 #include "lm_l4sp.h" 8 #include "mm_l4if.h" 9 #include "mm.h" 10 11 /* The maximum counter value for consumed count, if it exceeds this value we post it to firmware. 12 * FW holds 32bits for this counter. Therefore 100MB is OK (see L4 VBD spec) */ 13 #define MAX_GRQ_COUNTER 0x6400000 14 #define IS_OOO_CQE(__cmd) ((__cmd == CMP_OPCODE_TOE_GNI) \ 15 || (__cmd == CMP_OPCODE_TOE_GAIR) \ 16 || (__cmd == CMP_OPCODE_TOE_GAIL) \ 17 || (__cmd == CMP_OPCODE_TOE_GRI) \ 18 || (__cmd == CMP_OPCODE_TOE_GJ) \ 19 || (__cmd == CMP_OPCODE_TOE_DGI)) 20 21 typedef struct toe_rx_bd toe_rx_bd_t; 22 23 static u16_t lm_squeeze_rx_buffer_list( 24 struct _lm_device_t * pdev, 25 lm_tcp_state_t * tcp, 26 u16_t adjust_number, 27 lm_tcp_gen_buf_t ** unwanted_gen_buf 28 ); 29 30 static lm_status_t _lm_tcp_rx_post_buf( 31 struct _lm_device_t *pdev, 32 lm_tcp_state_t *tcp, 33 lm_tcp_buffer_t *tcp_buf, 34 lm_frag_list_t *frag_list 35 ); 36 37 static void lm_tcp_incr_consumed_gen( 38 struct _lm_device_t * pdev, 39 lm_tcp_state_t * tcp, 40 u32_t nbytes 41 ); 42 43 static void lm_tcp_return_gen_bufs( 44 struct _lm_device_t * pdev, 45 lm_tcp_state_t * tcp, 46 lm_tcp_gen_buf_t * gen_buf, 47 u32_t flags, 48 u8_t grq_idx 49 ); 50 51 static void lm_tcp_return_list_of_gen_bufs( 52 struct _lm_device_t * pdev, 53 lm_tcp_state_t * tcp, 54 d_list_t * gen_buf_list, 55 u32_t flags, 56 u8_t grq_idx 57 ); 58 59 static lm_isle_t * _lm_tcp_isle_get_free_list( 60 struct _lm_device_t * pdev, 61 u8_t grq_idx) 62 { 63 lm_isle_t * free_list = NULL; 64 lm_isle_t * isles_pool = pdev->toe_info.grqs[grq_idx].isles_pool; 65 u32_t isle_pool_idx; 66 u32_t isle_pool_size = pdev->params.l4_isles_pool_size; 67 DbgBreakIf(!isles_pool); 68 for (isle_pool_idx = 0; isle_pool_idx < isle_pool_size; isle_pool_idx++) { 69 if ((isles_pool[isle_pool_idx].isle_link.next == NULL) && (isles_pool[isle_pool_idx].isle_link.prev == NULL)) { 70 free_list = isles_pool + isle_pool_idx; 71 break; 72 } 73 } 74 DbgBreakIf(!free_list); 75 return free_list; 76 } 77 78 static lm_isle_t * _lm_tcp_isle_find( 79 struct _lm_device_t * pdev, 80 lm_tcp_state_t * tcp, 81 u8_t num_isle) 82 { 83 lm_isle_t * isle = NULL; 84 lm_tcp_con_rx_gen_info_t * gen_info; 85 u8_t isle_cnt, isle_idx; 86 87 DbgBreakIf(!(tcp && tcp->rx_con)); 88 gen_info = &tcp->rx_con->u.rx.gen_info; 89 isle_cnt = (u8_t)d_list_entry_cnt(&gen_info->isles_list); 90 DbgBreakIf(!isle_cnt); 91 DbgBreakIf(num_isle > isle_cnt); 92 if (num_isle == gen_info->current_isle_number) { 93 isle = gen_info->current_isle; 94 } else { 95 isle = (lm_isle_t*)gen_info->isles_list.head; 96 for (isle_idx = 1; isle_idx < num_isle; isle_idx++) { 97 isle = (lm_isle_t*)d_list_next_entry(&isle->isle_link); 98 } 99 gen_info->current_isle_number = num_isle; 100 gen_info->current_isle = isle; 101 } 102 return isle; 103 } 104 105 static u32_t _lm_tcp_isle_remove( 106 struct _lm_device_t * pdev, 107 lm_tcp_state_t * tcp, 108 u8_t grq_idx, 109 u8_t num_isle, 110 d_list_t * gen_buf_list) 111 { 112 u32_t nbytes = 0; 113 lm_isle_t * new_current_isle = NULL; 114 lm_isle_t * isle = NULL; 115 lm_tcp_con_rx_gen_info_t * gen_info; 116 u8_t isles_cnt; 117 u8_t new_current_isle_num; 118 119 DbgBreakIf(!(tcp && tcp->rx_con)); 120 gen_info = &tcp->rx_con->u.rx.gen_info; 121 isles_cnt = (u8_t)d_list_entry_cnt(&gen_info->isles_list); 122 DbgBreakIf(!(num_isle && (num_isle <= isles_cnt))); 123 isle = _lm_tcp_isle_find(pdev,tcp,num_isle); 124 125 // DbgBreakIf((isles_cnt > 1) && (num_isle == 1)); 126 if (isle->isle_link.next != NULL) { 127 new_current_isle = (lm_isle_t*)isle->isle_link.next; 128 new_current_isle_num = num_isle; 129 } else if (isle->isle_link.prev != NULL) { 130 new_current_isle = (lm_isle_t*)isle->isle_link.prev; 131 new_current_isle_num = num_isle - 1; 132 } else { 133 new_current_isle = NULL; 134 new_current_isle_num = 0; 135 } 136 137 #if defined(_NTDDK_) 138 #pragma prefast (push) 139 #pragma prefast (disable:6011) 140 #endif //_NTDDK_ 141 d_list_remove_entry(&gen_info->isles_list, &isle->isle_link); 142 #if defined(_NTDDK_) 143 #pragma prefast (pop) 144 #endif //_NTDDK_ 145 146 nbytes = isle->isle_nbytes; 147 d_list_add_tail(gen_buf_list, &isle->isle_gen_bufs_list_head); 148 d_list_init(&isle->isle_gen_bufs_list_head, NULL, NULL, 0); 149 if (new_current_isle_num) { 150 if (num_isle == 1) { 151 #if defined(_NTDDK_) 152 #pragma prefast (push) 153 #pragma prefast (disable:28182) 154 #endif //_NTDDK_ 155 d_list_remove_entry(&gen_info->isles_list, &new_current_isle->isle_link); 156 #if defined(_NTDDK_) 157 #pragma prefast (pop) 158 #endif //_NTDDK_ 159 d_list_add_tail(&isle->isle_gen_bufs_list_head, &new_current_isle->isle_gen_bufs_list_head); 160 d_list_push_head(&gen_info->isles_list, &isle->isle_link); 161 isle->isle_nbytes = new_current_isle->isle_nbytes; 162 #ifdef DEBUG_OOO_CQE 163 isle->dedicated_cid = new_current_isle->dedicated_cid; 164 isle->recent_ooo_combined_cqe = new_current_isle->recent_ooo_combined_cqe; 165 #endif 166 isle = new_current_isle; 167 new_current_isle = &gen_info->first_isle; 168 } 169 mm_mem_zero(&isle->isle_gen_bufs_list_head, sizeof(lm_isle_t) - sizeof(d_list_entry_t)); 170 isle->isle_link.next = isle->isle_link.prev = NULL; 171 } 172 gen_info->current_isle = new_current_isle; 173 gen_info->current_isle_number = new_current_isle_num; 174 return nbytes; 175 } 176 177 u32_t lm_tcp_rx_peninsula_to_rq(lm_device_t * pdev, lm_tcp_state_t * tcp, u32_t max_num_bytes_to_copy, u8_t sb_idx); 178 179 /* TODO: remove this temporary solution for solaris / linux compilation conflict, linux needs the 180 * first option, solaris the latter */ 181 #if defined(__LINUX) 182 #define TOE_RX_INIT_ZERO {{0}} 183 #else 184 #define TOE_RX_INIT_ZERO {0} 185 #endif 186 187 #define TOE_RX_DOORBELL(pdev,cid) do{\ 188 struct doorbell db = TOE_RX_INIT_ZERO;\ 189 db.header.data |= ((TOE_CONNECTION_TYPE << DOORBELL_HDR_T_CONN_TYPE_SHIFT) |\ 190 (DOORBELL_HDR_T_RX << DOORBELL_HDR_T_RX_SHIFT));\ 191 DOORBELL((pdev), (cid), *((u32_t *)&db));\ 192 } while(0) 193 194 static __inline void lm_tcp_rx_write_db( 195 lm_device_t *pdev, 196 lm_tcp_state_t *tcp 197 ) 198 { 199 lm_tcp_con_t *rx_con = tcp->rx_con; 200 volatile struct toe_rx_db_data *db_data = rx_con->db_data.rx; 201 202 if (!(rx_con->flags & TCP_RX_DB_BLOCKED)) { 203 db_data->bds_prod += rx_con->db_more_bds; /* nbds should be written before nbytes (FW assumption) */ 204 db_data->bytes_prod += rx_con->db_more_bytes; 205 206 DbgMessage(pdev, INFORMl4rx, 207 "_lm_tcp_rx_write_db: cid=%d, (nbytes+=%d, nbds+=%d)\n", 208 tcp->cid, rx_con->db_more_bytes, rx_con->db_more_bds); 209 TOE_RX_DOORBELL(pdev, tcp->cid); 210 } 211 212 /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */ 213 DbgBreakIf(S64_SUB(rx_con->bytes_post_cnt + rx_con->db_more_bytes, rx_con->bytes_comp_cnt) < 0); 214 rx_con->bytes_post_cnt += rx_con->db_more_bytes; 215 rx_con->buffer_post_cnt += rx_con->db_more_bufs; 216 rx_con->db_more_bytes = rx_con->db_more_bds = rx_con->db_more_bufs = 0; 217 rx_con->fp_db_cnt++; 218 } 219 220 /** Description 221 * This function is used to increase the window-size. Window is increased in 3 cases: 222 * 1. RQ-placed bytes 223 * 2. GRQ-Indicated succesfully (short/long loop, doensn't matter) 224 * 3. Window-update from NDIS (initial rcv window increased) 225 * 4. This function also takes into account dwa: delayed window algorithm, and updates the 226 * data structures accordingly, however, not all window-updates are part of the dwa algorithm, 227 * specifically, (3) therefore, we need to know if the update is dwa-aware or not. 228 */ 229 void lm_tcp_rx_post_sws ( 230 lm_device_t * pdev, 231 lm_tcp_state_t * tcp, 232 lm_tcp_con_t * rx_con, 233 u32_t nbytes, 234 u8_t op 235 ) 236 { 237 volatile struct toe_rx_db_data *db_data = rx_con->db_data.rx; 238 s32_t diff_to_fw; 239 240 switch (op) 241 { 242 case TCP_RX_POST_SWS_INC: 243 /*DbgMessage(pdev, FATAL, "lm_tcp_rx_post_sws() INC: OLD drv_rcv_win_right_edge=%d, nbytes=%d, NEW drv_rcv_win_right_edge=%d FW right_edge=%d \n", rx_con->u.rx.sws_info.drv_rcv_win_right_edge, nbytes, rx_con->u.rx.sws_info.drv_rcv_win_right_edge + nbytes, db_data->rcv_win_right_edge);*/ 244 if (rx_con->u.rx.sws_info.extra_bytes > nbytes) { 245 rx_con->u.rx.sws_info.extra_bytes -= nbytes; 246 nbytes = 0; 247 } else { 248 nbytes -= rx_con->u.rx.sws_info.extra_bytes; 249 rx_con->u.rx.sws_info.extra_bytes = 0; 250 rx_con->u.rx.sws_info.drv_rcv_win_right_edge += nbytes; 251 if (rx_con->u.rx.sws_info.drv_rcv_win_right_edge >= db_data->rcv_win_right_edge) { 252 RESET_FLAGS(tcp->rx_con->db_data.rx->flags, TOE_RX_DB_DATA_IGNORE_WND_UPDATES); 253 } 254 } 255 break; 256 case TCP_RX_POST_SWS_DEC: 257 if (rx_con->u.rx.sws_info.extra_bytes) { 258 rx_con->u.rx.sws_info.extra_bytes += nbytes; 259 nbytes = 0; 260 } 261 /*DbgMessage(pdev, FATAL, "lm_tcp_rx_post_sws() DEC: OLD drv_rcv_win_right_edge=%d, nbytes=%d, NEW drv_rcv_win_right_edge=%d\n", rx_con->u.rx.sws_info.drv_rcv_win_right_edge, nbytes, rx_con->u.rx.sws_info.drv_rcv_win_right_edge - nbytes);*/ 262 rx_con->u.rx.sws_info.drv_rcv_win_right_edge -= nbytes; 263 SET_FLAGS(db_data->flags, TOE_RX_DB_DATA_IGNORE_WND_UPDATES); 264 break; 265 case TCP_RX_POST_SWS_SET: 266 /*DbgMessage(pdev, FATAL, "lm_tcp_rx_post_sws() SET: nbytes=%d\n", nbytes);*/ 267 db_data->rcv_win_right_edge = nbytes; 268 rx_con->u.rx.sws_info.extra_bytes = 0;; 269 break; 270 default: 271 DbgBreakMsg("lm_tcp_rx_post_sws: Invalid operation\n"); 272 return; 273 } 274 275 /* note that diff_to_fw could be negative due to possibility of window-decrease in LH */ 276 diff_to_fw = S32_SUB(rx_con->u.rx.sws_info.drv_rcv_win_right_edge, db_data->rcv_win_right_edge); 277 278 /* If this update isn't dwa_aware, it's good to go... */ 279 280 //DbgMessage(pdev, WARNl4, "###lm_tcp_rx_post_sws cid=%d num_bytes=%d diff_to_fw=%d \n", tcp->cid, nbytes, diff_to_fw ); 281 /* we give the window only if diff_to_fw is larger than mss, which also means only in case it is negative... */ 282 if ( ((diff_to_fw >= (s32_t)rx_con->u.rx.sws_info.mss) || 283 (diff_to_fw >= (((s32_t)tcp->tcp_cached.initial_rcv_wnd) / 2)))) { 284 if (rx_con->u.rx.sws_info.timer_on) { 285 /* Vladz TBD: Cancel the timer */ 286 rx_con->u.rx.sws_info.timer_on = 0; 287 } 288 289 /* Ring the Advertise Window doorbell here */ 290 if (!(tcp->rx_con->flags & TCP_RX_DB_BLOCKED) && !(tcp->rx_con->flags & TCP_RX_POST_BLOCKED)) { 291 db_data->rcv_win_right_edge = rx_con->u.rx.sws_info.drv_rcv_win_right_edge; 292 DbgMessage(pdev, INFORMl4rx, 293 "_lm_tcp_adv_wnd_write_db: cid=%d, nbytes=%d\n", 294 tcp->cid, diff_to_fw); 295 TOE_RX_DOORBELL(pdev, tcp->cid); 296 } 297 } else { 298 if ( ! rx_con->u.rx.sws_info.timer_on ) { 299 /* Vladz TBD: schedule the timer here */ 300 rx_con->u.rx.sws_info.timer_on = 1; 301 } 302 } 303 } 304 305 static __inline toe_rx_bd_t * _lm_tcp_rx_set_bd ( 306 IN lm_frag_t * frag, 307 IN u16_t flags, 308 IN lm_bd_chain_t * rx_chain, 309 IN u32_t dbg_bytes_prod /* Used for synchronizing between fw and driver rq-available-bytes 310 * This is used only as a debug variable for asserting in the fw. */ 311 ) 312 { 313 struct toe_rx_bd * rx_bd; 314 315 /* hw limit: each bd can point to a buffer with max size of 64KB */ 316 DbgBreakIf(frag->size > TCP_MAX_SGE_SIZE || frag->size == 0); 317 rx_bd = (struct toe_rx_bd *)lm_toe_bd_chain_produce_bd(rx_chain); 318 rx_bd->addr_hi = frag->addr.as_u32.high; 319 rx_bd->addr_lo = frag->addr.as_u32.low; 320 rx_bd->flags = flags; 321 rx_bd->size = (u16_t)frag->size; 322 rx_bd->dbg_bytes_prod = dbg_bytes_prod; 323 DbgMessage(NULL, VERBOSEl4rx, "Setting Rx BD flags=0x%x, bd_addr=0x%p, size=%d\n", rx_bd->flags, rx_bd, frag->size); 324 return rx_bd; 325 } 326 327 328 /** Description 329 * function completes nbytes on a single tcp buffer and completes the buffer if it is 330 * completed. 331 * Assumptions: 332 * fp-lock is taken. 333 * It is only called from lm_tcp_rx_post_buf!!! 334 */ 335 static void lm_tcp_complete_tcp_buf( 336 lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con, lm_tcp_buffer_t * tcp_buf, u32_t completed_bytes) 337 { 338 s_list_t completed_bufs; 339 s_list_entry_t * entry; 340 341 DbgBreakIf(completed_bytes > tcp_buf->more_to_comp); 342 tcp_buf->more_to_comp -= completed_bytes; 343 con->app_buf_bytes_acc_comp += completed_bytes; 344 345 if(tcp_buf->more_to_comp == 0 && GET_FLAGS(tcp_buf->flags, TCP_BUF_FLAG_L4_POST_END)) { 346 tcp_buf->app_buf_xferred = con->app_buf_bytes_acc_comp; 347 DbgBreakIf(tcp_buf->app_buf_xferred > tcp_buf->app_buf_size); /* this may be partial completion */ 348 con->app_buf_bytes_acc_comp = 0; 349 if (GET_FLAGS(con->flags, TCP_POST_COMPLETE_SPLIT)) { 350 RESET_FLAGS(con->flags, TCP_POST_COMPLETE_SPLIT); 351 } 352 } else { 353 tcp_buf->app_buf_xferred = 0; 354 } 355 356 if (tcp_buf->more_to_comp == 0) { 357 /* should have nothing in the active tb list except this buffer, if we're completing this buffer, 358 * it means that we had something in the peninsula, this means that at the end of the DPC there was 359 * nothing in the active-tb-list, and between DPCs all posted buffers 'occupied' bytes from the peninsula 360 * and were completed to the client. This means that there can be no RQ completions during the DPC that 361 * will try to access the active tb list w/o a lock 362 */ 363 DbgBreakIf(s_list_entry_cnt(&con->active_tb_list) != 1); 364 lm_bd_chain_bds_consumed(&con->bd_chain, tcp_buf->bd_used); 365 366 con->buffer_completed_cnt ++; 367 DbgMessage(pdev, VERBOSEl4fp, 368 "cid=%d, completing tcp buf towards mm from post-flow, actual_completed_bytes=%d\n", 369 tcp->cid, tcp_buf->size); 370 entry = s_list_pop_head(&con->active_tb_list); 371 DbgBreakIf(con->rq_nbytes < tcp_buf->size); 372 con->rq_nbytes -= tcp_buf->size; 373 s_list_init(&completed_bufs, entry, entry, 1); 374 con->rq_completion_calls++; 375 mm_tcp_complete_bufs(pdev, tcp, con, &completed_bufs, LM_STATUS_SUCCESS); 376 } 377 } 378 379 380 381 void lm_tcp_rx_cmp_process( 382 struct _lm_device_t * pdev, 383 lm_tcp_state_t * tcp, 384 u32_t completed_bytes, 385 u8_t push 386 ) 387 { 388 lm_tcp_con_t *rx_con; 389 u32_t actual_bytes_completed; 390 MM_INIT_TCP_LOCK_HANDLE(); 391 392 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_cmp_process, completed_bytes=%d, push=%d cid=%d\n", completed_bytes, push, tcp->cid); 393 DbgBreakIf(!(completed_bytes || push)); /* otherwise there is no point for this function to be called */ 394 395 rx_con = tcp->rx_con; 396 DbgBreakIf(! rx_con); 397 398 if (!(rx_con->flags & TCP_DEFERRED_PROCESSING)) { 399 mm_acquire_tcp_lock(pdev, rx_con); 400 } 401 DbgBreakIf(rx_con->flags & TCP_RX_COMP_BLOCKED); 402 403 /* RQ completions can't arrive while we have something in the peninsula (peninsula must either be completed or copied 404 * to the app-buffer before) An RQ_SKP within the dpc will always take care of previous RQs waiting to be copied to. */ 405 DbgBreakIf(!d_list_is_empty(&rx_con->u.rx.gen_info.peninsula_list)); 406 DbgBreakIf(!d_list_is_empty(&rx_con->u.rx.gen_info.dpc_peninsula_list)); 407 408 actual_bytes_completed = lm_tcp_complete_nbytes(pdev, tcp, rx_con, completed_bytes , push); 409 410 rx_con->bytes_comp_cnt += actual_bytes_completed; 411 DbgBreakIf(S64_SUB(rx_con->bytes_post_cnt, rx_con->bytes_comp_cnt) < 0); 412 DbgMessage(pdev, VERBOSEl4rx, "lm_tcp_rx_comp, after comp: pending=%d, active_bufs=%d\n", 413 S64_SUB(rx_con->bytes_post_cnt, rx_con->bytes_comp_cnt), 414 s_list_entry_cnt(&rx_con->active_tb_list)); 415 416 if ( completed_bytes ) { 417 /* Vladz: TBD 418 lm_neigh_update_nic_reachability_time(tcp->path->neigh) */ 419 } 420 if (!(rx_con->flags & TCP_DEFERRED_PROCESSING)) { 421 mm_release_tcp_lock(pdev, rx_con); 422 } 423 } /* lm_tcp_rx_comp */ 424 425 426 void lm_tcp_rx_skp_process( 427 struct _lm_device_t * pdev, 428 lm_tcp_state_t * tcp, 429 u32_t bytes_skipped, 430 u8_t sb_idx 431 ) 432 { 433 lm_tcp_con_t *rx_con; 434 u32_t comp_bytes; 435 MM_INIT_TCP_LOCK_HANDLE(); 436 437 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_skp_process, bytes_skipped=%d, cid=%d\n", bytes_skipped, tcp->cid); 438 439 if (bytes_skipped == 0) { 440 /* nothing to do here - occurs on special fw case, where there is GRQ->RQ processing with no GRQ and no RQ, 441 * this will usually happen at the beginning or in special cases of the connection */ 442 return; 443 } 444 445 rx_con = tcp->rx_con; 446 DbgBreakIf(! rx_con); 447 448 if (!GET_FLAGS(rx_con->flags, TCP_DEFERRED_PROCESSING)) { 449 mm_acquire_tcp_lock(pdev, rx_con); 450 } 451 DbgBreakIf(GET_FLAGS(rx_con->flags, TCP_RX_COMP_BLOCKED)); 452 453 comp_bytes = min(bytes_skipped, tcp->rx_con->u.rx.skp_bytes_copied); 454 if (comp_bytes) { 455 tcp->rx_con->bytes_comp_cnt += comp_bytes; 456 /* complete nbytes on buffers (dpc-flow ) */ 457 lm_tcp_complete_nbytes(pdev, tcp, tcp->rx_con, comp_bytes, /* push=*/ 0); 458 bytes_skipped -= comp_bytes; 459 tcp->rx_con->u.rx.skp_bytes_copied -= comp_bytes; 460 } 461 462 /* We know for sure, that all the application buffers we are about to access have already been posted 463 * before the dpc, and therefore are valid in the active_tb_list. 464 * TBA Michals: bypass FW 465 */ 466 if (bytes_skipped) { 467 DbgBreakIf(!d_list_is_empty(&rx_con->u.rx.gen_info.peninsula_list)); 468 DbgBreakIfAll(d_list_is_empty(&rx_con->u.rx.gen_info.dpc_peninsula_list)); 469 DbgBreakIf(((lm_tcp_gen_buf_t *)d_list_peek_head(&rx_con->u.rx.gen_info.dpc_peninsula_list))->placed_bytes == 0); 470 rx_con->u.rx.gen_info.bytes_copied_cnt_in_process += lm_tcp_rx_peninsula_to_rq(pdev, tcp, bytes_skipped,sb_idx); 471 } 472 473 if (!GET_FLAGS(rx_con->flags, TCP_DEFERRED_PROCESSING)) { 474 mm_release_tcp_lock(pdev, rx_con); 475 } 476 } /* lm_tcp_rx_skp */ 477 478 void lm_tcp_rx_delete_isle( 479 struct _lm_device_t * pdev, 480 lm_tcp_state_t * tcp, 481 u8_t sb_idx, 482 u8_t num_isle, 483 u32_t num_of_isles) 484 { 485 lm_tcp_con_t * rx_con = tcp->rx_con; 486 lm_tcp_con_rx_gen_info_t * gen_info; 487 d_list_t removed_list; 488 u32_t isle_nbytes; 489 490 491 492 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_delete_isle cid=%d isle=%d\n", tcp->cid, num_isle); 493 gen_info = &rx_con->u.rx.gen_info; 494 d_list_init(&removed_list, NULL, NULL, 0); 495 496 while (num_of_isles) { 497 isle_nbytes = _lm_tcp_isle_remove(pdev, tcp, sb_idx, num_isle + (num_of_isles - 1), &removed_list); 498 pdev->toe_info.grqs[sb_idx].number_of_isles_delta--; 499 DbgBreakIf(isle_nbytes > gen_info->isle_nbytes); 500 gen_info->isle_nbytes -= isle_nbytes; 501 num_of_isles--; 502 } 503 504 pdev->toe_info.grqs[sb_idx].gen_bufs_in_isles_delta -= (s32_t)d_list_entry_cnt(&removed_list); 505 if (!d_list_is_empty(&removed_list)) { 506 lm_tcp_return_list_of_gen_bufs(pdev,tcp ,&removed_list, MM_TCP_RGB_COLLECT_GEN_BUFS, sb_idx); 507 tcp->rx_con->droped_non_empty_isles++; 508 } else { 509 DbgBreak(); 510 tcp->rx_con->droped_empty_isles++; 511 } 512 rx_con->dpc_info.dpc_flags &= ~(LM_TCP_DPC_TOO_BIG_ISLE | LM_TCP_DPC_TOO_MANY_ISLES); 513 return; 514 } 515 516 u8_t lm_toe_is_rx_completion(lm_device_t *pdev, u8_t drv_toe_rss_id) 517 { 518 u8_t result = FALSE; 519 lm_tcp_rcq_t *rcq = NULL; 520 521 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.rcqs) > drv_toe_rss_id)); 522 523 rcq = &pdev->toe_info.rcqs[drv_toe_rss_id]; 524 525 if ( rcq->hw_con_idx_ptr && 526 *rcq->hw_con_idx_ptr != lm_bd_chain_cons_idx(&rcq->bd_chain) ) 527 { 528 result = TRUE; 529 } 530 DbgMessage(pdev, INFORMl4int, "lm_toe_is_rx_completion(): result is:%s\n", result? "TRUE" : "FALSE"); 531 532 return result; 533 } 534 535 /** Description 536 * checks if the processing of a certain RCQ is suspended 537 */ 538 u8_t lm_toe_is_rcq_suspended(lm_device_t *pdev, u8_t drv_toe_rss_id) 539 { 540 u8_t result = FALSE; 541 lm_tcp_rcq_t *rcq = NULL; 542 543 if (drv_toe_rss_id < MAX_L4_RX_CHAIN) 544 { 545 rcq = &pdev->toe_info.rcqs[drv_toe_rss_id]; 546 if (rcq->suspend_processing) { 547 result = TRUE; 548 } 549 } 550 DbgMessage(pdev, INFORMl4int, "lm_toe_is_rcq_suspended(): sb_idx:%d, result is:%s\n", drv_toe_rss_id, result?"TRUE":"FALSE"); 551 return result; 552 } 553 554 555 /** Description 556 * Increment consumed generic counter for a connection. 557 * To avoid rollover in the FW if the counter exceeds a maximum threshold, the driver should 558 * not wait for application buffers and post 'receive window update' doorbell immediately. 559 * The FW holds 32bits for this counter. Therefore a threshold of 100MB is OK. 560 */ 561 static void lm_tcp_incr_consumed_gen( 562 struct _lm_device_t * pdev, 563 lm_tcp_state_t * tcp, 564 u32_t nbytes 565 ) 566 { 567 volatile struct toe_rx_db_data *db_data = tcp->rx_con->db_data.rx; 568 569 db_data->consumed_grq_bytes += nbytes; 570 571 /* theres no need to increased the consumed_cnt in two stages (one in the driver and one for FW db_data) 572 * we can always directly increase FW db_data, we need to decide whether we need to give a doorbell, basically 573 * we have two cases where doorbells are given: (1) buffer posted and bypasses fw (2) indication succeeded in which case 574 * window will also be increased, however, the window isn't always increased: if it's smaller than MSS, so, if we 575 * increase the consumed count by something smaller than mss - we'll give the doorbell here... */ 576 577 if (nbytes < tcp->rx_con->u.rx.sws_info.mss) { 578 if (!(tcp->rx_con->flags & TCP_RX_DB_BLOCKED)) { 579 TOE_RX_DOORBELL(pdev, tcp->cid); 580 } 581 } 582 } 583 584 /** Description 585 * Copies as many bytes as possible from the peninsula to the single tcp buffer received 586 * updates the peninsula. 587 * This function can be called from two flows: 588 * 1. Post of a buffer 589 * 2. Completion of a dpc. 590 * We need to know which flow it is called from to know which peninsula list to use: 591 * dpc_peninsula_list / peninsula_list. 592 * Post ALWAYS uses the peninsula_list, since it doesn't know about the dpc_peninsula 593 * Completion ALWAYS uses the dpc_peninsula_list, and in this case peninsula_list MUST be empty 594 * this is because there can be buffers in the active_tb_list ONLY if peninsula_list is empty. 595 * 596 * first_buf_offset refers to the peninsula we're dealing with, at the end of the dpc the dpc_peninsula 597 * is copied to the peninsula, therefore first_buf_offset will still be valid. copying from post means that 598 * there is something in the peninsula which means theres nothing in the active_tb_list ==> won't be a copy from 599 * dpc. Copying from dpc means theres something in the active-tb-list ==> nothing in the peninsula ==> won't be called 600 * from post, mutual exclusion exists between the post/dpc of copying, therefore we can have only one first_buffer_offset 601 * all other accesses (indication) are done under a lock. 602 * param: dpc - indicates if this is called from the dpc or not (post) 603 * Assumptions: 604 * tcp_buf->more_to_comp is initialized 605 * tcp_buf->size is initialized 606 * num_bufs_complete is initialized by caller (could differ from zero) 607 * Returns 608 * the actual number of bytes copied 609 * num_bufs_complete is the number of buffers that were completely copied to the pool and can be 610 * returned to the pool. 611 */ 612 static u32_t lm_tcp_rx_peninsula_to_rq_copy( 613 lm_device_t * pdev, 614 lm_tcp_state_t * tcp, 615 lm_tcp_buffer_t * tcp_buf, 616 d_list_t * return_list, 617 u32_t max_num_bytes_to_copy, 618 u8_t dpc) 619 { 620 lm_tcp_gen_buf_t * curr_gen_buf; 621 lm_tcp_con_rx_gen_info_t * gen_info; 622 d_list_t * peninsula; 623 u32_t tcp_offset; 624 u32_t ncopy; 625 u32_t bytes_left; 626 u32_t bytes_copied = 0; 627 628 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_peninsula_to_rq_copy tcp_buf = 0x%x cid=%d\n", *((u32_t *)&tcp_buf), tcp->cid); 629 630 gen_info = &tcp->rx_con->u.rx.gen_info; 631 632 if (dpc) { 633 peninsula = &gen_info->dpc_peninsula_list; 634 } else { 635 peninsula = &gen_info->peninsula_list; 636 } 637 638 curr_gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_head(peninsula); 639 tcp_offset = tcp_buf->size - tcp_buf->more_to_comp; 640 bytes_left = min(tcp_buf->more_to_comp, max_num_bytes_to_copy); /* copy to buffer only what's aloud...*/ 641 642 /* start copying as much as possible from peninsula to tcp buffer */ 643 while (bytes_left && curr_gen_buf && curr_gen_buf->placed_bytes) { 644 ncopy = curr_gen_buf->placed_bytes - gen_info->first_buf_offset; 645 if (ncopy > bytes_left) { 646 ncopy = bytes_left; 647 } 648 if (mm_tcp_copy_to_tcp_buf(pdev, tcp, tcp_buf, 649 curr_gen_buf->buf_virt + gen_info->first_buf_offset, /* start of data in generic buffer */ 650 tcp_offset, ncopy) != ncopy) 651 { 652 gen_info->copy_gen_buf_dmae_cnt++; 653 654 /* If this is generic buffer that has the free_when_done flag on it means it's non-cached memory and not physical 655 * memory -> so, we can't try and dmae to it... not likely to happen... */ 656 if (!GET_FLAGS(curr_gen_buf->flags, GEN_FLAG_FREE_WHEN_DONE)) { 657 if (mm_tcp_rx_peninsula_to_rq_copy_dmae(pdev, 658 tcp, 659 curr_gen_buf->buf_phys, 660 gen_info->first_buf_offset, /* start of data in generic buffer */ 661 tcp_buf, 662 tcp_offset, 663 ncopy) != ncopy) 664 { 665 DbgBreakMsg("Unable To Copy"); 666 gen_info->copy_gen_buf_fail_cnt++; 667 668 break; 669 } 670 } else { 671 DbgBreakMsg("Unable To Copy"); 672 gen_info->copy_gen_buf_fail_cnt++; 673 674 break; 675 } 676 } 677 678 /* update peninsula */ 679 bytes_copied += ncopy; 680 681 gen_info->first_buf_offset += (u16_t)ncopy; 682 683 /* done with the generic buffer? - return it to the pool */ 684 if (curr_gen_buf->placed_bytes == gen_info->first_buf_offset) { 685 curr_gen_buf = (lm_tcp_gen_buf_t *)d_list_pop_head(peninsula); 686 d_list_push_tail(return_list, &curr_gen_buf->link); 687 gen_info->first_buf_offset = 0; 688 gen_info->num_buffers_copied_grq++; 689 curr_gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_head(peninsula); 690 } 691 692 /* update tcp buf stuff */ 693 bytes_left -= ncopy; 694 tcp_offset += ncopy; 695 } 696 697 if (dpc) { 698 gen_info->dpc_peninsula_nbytes -= bytes_copied; 699 } else { 700 gen_info->peninsula_nbytes -= bytes_copied; 701 } 702 703 /* return the number of bytes actually copied */ 704 return bytes_copied; 705 } 706 707 /** Description 708 * function copies data from the peninsula to tcp buffers already placed in the 709 * active_tb_list. The function completes the buffers if a tcp buffer from active_tb_list 710 * was partially/fully filled. This case simulates a call to lm_tcp_rx_comp 711 * (i.e. a completion received from firmware) 712 * Assumptions: 713 */ 714 u32_t lm_tcp_rx_peninsula_to_rq(lm_device_t * pdev, lm_tcp_state_t * tcp, u32_t max_num_bytes_to_copy, u8_t sb_idx) 715 { 716 lm_tcp_buffer_t * curr_tcp_buf; 717 lm_tcp_con_rx_gen_info_t * gen_info; 718 d_list_t return_list; 719 u32_t copied_bytes = 0, currently_copied = 0; 720 721 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_peninsula_to_rq cid=%d\n", tcp->cid); 722 723 gen_info = &tcp->rx_con->u.rx.gen_info; 724 725 DbgBreakIf(gen_info->peninsula_blocked == TRUE); /* terminate was already called - no copying should be done */ 726 727 /* Copy data from dpc_peninsula to tcp buffer[s] */ 728 d_list_init(&return_list, NULL, NULL, 0); 729 730 curr_tcp_buf = lm_tcp_next_entry_dpc_active_list(tcp->rx_con); 731 732 /* TBA Michals: FW Bypass First check if we can copy to bypass buffers */ 733 734 /* Copy the number of bytes received in SKP */ 735 while (max_num_bytes_to_copy && gen_info->dpc_peninsula_nbytes && curr_tcp_buf) { 736 currently_copied = lm_tcp_rx_peninsula_to_rq_copy(pdev, tcp, curr_tcp_buf, &return_list, max_num_bytes_to_copy, TRUE); 737 curr_tcp_buf = (lm_tcp_buffer_t *)s_list_next_entry(&curr_tcp_buf->link); 738 DbgBreakIf(max_num_bytes_to_copy < currently_copied); 739 max_num_bytes_to_copy -= currently_copied; 740 copied_bytes += currently_copied; 741 } 742 743 if (!d_list_is_empty(&return_list)) { 744 745 lm_tcp_return_list_of_gen_bufs(pdev,tcp , &return_list, 746 (sb_idx != NON_EXISTENT_SB_IDX) ? MM_TCP_RGB_COLLECT_GEN_BUFS : 0, sb_idx); 747 } 748 749 /* If we've copied to a buffer in the active_tb_list we need to complete it since fw knows 750 * the driver has the bytes and the driver will take care of copying them and completing them. 751 * this path simulates a call to lm_tcp_rx_comp (buffers taken from active_tb_list) */ 752 /* Note that pending bytes here could reach a negative value if a partial 753 * application buffer was posted and the doorbell hasn't been given yet, however, 754 * once the doorbell is given for the application buffer the pending bytes will reach a non-negative 755 * value (>=0) */ 756 tcp->rx_con->bytes_comp_cnt += copied_bytes; 757 /* complete nbytes on buffers (dpc-flow ) */ 758 lm_tcp_complete_nbytes(pdev, tcp, tcp->rx_con, copied_bytes, /* push=*/ 0); 759 760 DbgMessage(pdev, VERBOSEl4rx, "lm_tcp_rx_peninsula_to_rq copied %d bytes cid=%d\n", copied_bytes, tcp->cid); 761 return copied_bytes; 762 } 763 764 /** Description 765 * determines whether or not we can indicate. 766 * Rules: 767 * - Indication is not blocked 768 * - we are not in the middle of completion a split-buffer 769 * we can only indicate after an entire buffer has been completed/copied to. 770 * we determine this by the app_buf_bytes_acc_comp. This is to avoid the 771 * following data integrity race: 772 * application buffer: app_start, app_end 773 * app_start is posted, peninsula copied to app_start, app_start completed to 774 * fw then the rest is indicated. fw receives app_end, fw thinks peninsula was 775 * copied to buffer, application buffer misses data... 776 * - our active_tb_list is empty... we HAVE to make sure to 777 * always indicate after we've fully utilized our RQ 778 * buffers... 779 */ 780 static __inline u8_t _lm_tcp_ok_to_indicate(lm_tcp_con_t * rx_con) 781 { 782 return (!(rx_con->flags & TCP_RX_IND_BLOCKED) && (rx_con->app_buf_bytes_acc_comp == 0) && 783 (s_list_is_empty(&rx_con->active_tb_list))); 784 } 785 786 /** Description 787 * GA: add a buffer to the peninsula - nbytes represents the number of bytes in the previous buffer. 788 * GR: release a buffer from the peninsula - nbytes represents the number of bytes in the current buffer. 789 * Assumption: 790 * GR can only be called on a buffer that had been added using GA before 791 */ 792 void lm_tcp_rx_gen_peninsula_process(lm_device_t * pdev, lm_tcp_state_t * tcp, u32_t nbytes, lm_tcp_gen_buf_t * gen_buf) 793 { 794 lm_tcp_con_t * rx_con = tcp->rx_con; 795 lm_tcp_con_rx_gen_info_t * gen_info; 796 lm_tcp_gen_buf_t * last_gen_buf; 797 798 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_gen_peninsula_process, nbytes=%d, cid=%d add=%s\n", nbytes, tcp->cid, 799 (gen_buf)? "TRUE" : "FALSE"); 800 801 DbgBreakIf(rx_con->flags & TCP_RX_COMP_BLOCKED); 802 803 gen_info = &rx_con->u.rx.gen_info; 804 805 /* update the previous buffer OR current buffer if this is a release operation. This function is always called 806 * from within a DPC and updates the dpc_peninsula */ 807 if (nbytes) { 808 gen_info->dpc_peninsula_nbytes += nbytes; 809 last_gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_tail(&gen_info->dpc_peninsula_list); 810 DbgBreakIfAll(last_gen_buf == NULL); 811 DbgBreakIfAll(last_gen_buf->placed_bytes != 0); 812 DbgBreakIfAll(nbytes > LM_TCP_GEN_BUF_SIZE(pdev)); 813 last_gen_buf->placed_bytes = (u16_t)nbytes; 814 } 815 816 if (gen_buf /* add */) { 817 DbgBreakIf(SIG(gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 818 DbgBreakIf(END_SIG(gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 819 820 d_list_push_tail(&gen_info->dpc_peninsula_list, &gen_buf->link); 821 } 822 823 } 824 825 void lm_tcp_rx_gen_isle_create(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_gen_buf_t * gen_buf, u8_t sb_idx, u8_t isle_num) 826 { 827 lm_isle_t * current_isle = NULL; 828 lm_isle_t * next_isle = NULL; 829 lm_tcp_con_t * rx_con = tcp->rx_con; 830 lm_tcp_con_rx_gen_info_t * gen_info; 831 u8_t isles_cnt; 832 d_list_entry_t * isle_entry_prev = NULL; 833 d_list_entry_t * isle_entry_next = NULL; 834 835 gen_info = &rx_con->u.rx.gen_info; 836 isles_cnt = (u8_t)d_list_entry_cnt(&gen_info->isles_list); 837 if (isles_cnt) { 838 DbgBreakIf(isles_cnt == T_TCP_MAX_ISLES_PER_CONNECTION_TOE); 839 current_isle = _lm_tcp_isle_get_free_list(pdev, sb_idx); 840 DbgBreakIf(!current_isle); 841 #ifdef DEBUG_OOO_CQE 842 DbgBreakIf(current_isle->dedicated_cid != 0); 843 current_isle->dedicated_cid = tcp->cid; 844 #endif 845 } else { 846 current_isle = &gen_info->first_isle; 847 } 848 849 d_list_push_head(¤t_isle->isle_gen_bufs_list_head, &gen_buf->link); 850 current_isle->isle_nbytes = 0; 851 if (isle_num == 1) { 852 if (current_isle != &gen_info->first_isle) { 853 *current_isle = gen_info->first_isle; 854 d_list_init(&gen_info->first_isle.isle_gen_bufs_list_head, NULL, NULL, 0); 855 d_list_push_head(&gen_info->first_isle.isle_gen_bufs_list_head, &gen_buf->link); 856 gen_info->first_isle.isle_nbytes = 0; 857 isle_entry_prev = &gen_info->first_isle.isle_link; 858 isle_entry_next = gen_info->first_isle.isle_link.next; 859 } 860 } else if (isle_num <= isles_cnt) { 861 next_isle = _lm_tcp_isle_find(pdev,tcp,isle_num); 862 isle_entry_prev = next_isle->isle_link.prev; 863 isle_entry_next = &next_isle->isle_link; 864 } else if (isle_num == (isles_cnt + 1)) { 865 isle_entry_next = NULL; 866 isle_entry_prev = gen_info->isles_list.tail; 867 } else { 868 DbgBreak(); 869 } 870 871 d_list_insert_entry(&gen_info->isles_list, isle_entry_prev, isle_entry_next, ¤t_isle->isle_link); 872 if (isle_num == 1) { 873 current_isle = &gen_info->first_isle; 874 } 875 #ifdef DEBUG_OOO_CQE 876 SET_DEBUG_OOO_INFO(current_isle, CMP_OPCODE_TOE_GNI, 0); 877 #endif 878 gen_info->current_isle = current_isle; 879 gen_info->current_isle_number = isle_num; 880 pdev->toe_info.grqs[sb_idx].number_of_isles_delta++; 881 if (isles_cnt == gen_info->max_number_of_isles) { 882 gen_info->max_number_of_isles++; 883 } 884 } 885 886 void lm_tcp_rx_gen_isle_right_process(lm_device_t * pdev, lm_tcp_state_t * tcp, u32_t nbytes, lm_tcp_gen_buf_t * gen_buf, u8_t sb_idx, u8_t isle_num) 887 { 888 lm_tcp_con_t * rx_con = tcp->rx_con; 889 lm_tcp_con_rx_gen_info_t * gen_info; 890 lm_tcp_gen_buf_t * last_gen_buf; 891 lm_isle_t * requested_isle; 892 893 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_gen_isle_process nbytes = %d cid=%d\n", nbytes, tcp->cid); 894 895 gen_info = &rx_con->u.rx.gen_info; 896 requested_isle = _lm_tcp_isle_find(pdev,tcp,isle_num); 897 DbgBreakIf(!requested_isle); 898 899 /* update the previous buffer */ 900 last_gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_tail(&requested_isle->isle_gen_bufs_list_head); 901 DbgBreakIf(last_gen_buf == NULL); 902 if (nbytes) { 903 gen_info->isle_nbytes += nbytes; 904 requested_isle->isle_nbytes += nbytes; 905 DbgBreakIf(last_gen_buf->placed_bytes != 0); 906 DbgBreakIf(nbytes > 0xffff); 907 last_gen_buf->placed_bytes = (u16_t)nbytes; 908 } else { 909 DbgBreakIf(gen_buf == NULL); 910 DbgBreakIf(last_gen_buf->placed_bytes == 0); 911 } 912 913 if (gen_buf) { 914 DbgBreakIf(SIG(gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 915 DbgBreakIf(END_SIG(gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 916 917 d_list_push_tail(&requested_isle->isle_gen_bufs_list_head, &gen_buf->link); 918 pdev->toe_info.grqs[sb_idx].gen_bufs_in_isles_delta++; 919 if (pdev->params.l4_max_gen_bufs_in_isle 920 && (d_list_entry_cnt(&requested_isle->isle_gen_bufs_list_head) > pdev->params.l4_max_gen_bufs_in_isle)) { 921 if (pdev->params.l4_limit_isles & L4_LI_NOTIFY) { 922 DbgBreak(); 923 } 924 if (pdev->params.l4_limit_isles & L4_LI_MAX_GEN_BUFS_IN_ISLE) { 925 rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_TOO_BIG_ISLE; 926 } 927 } 928 #ifdef DEBUG_OOO_CQE 929 SET_DEBUG_OOO_INFO(requested_isle, CMP_OPCODE_TOE_GAIR, nbytes); 930 } else { 931 SET_DEBUG_OOO_INFO(requested_isle, CMP_OPCODE_TOE_GRI, nbytes); 932 #endif 933 } 934 } 935 936 void lm_tcp_rx_gen_isle_left_process(lm_device_t * pdev, lm_tcp_state_t * tcp, u32_t nbytes, lm_tcp_gen_buf_t * gen_buf, u8_t sb_idx, u8_t isle_num) 937 { 938 lm_tcp_con_t * rx_con = tcp->rx_con; 939 lm_tcp_con_rx_gen_info_t * gen_info; 940 lm_tcp_gen_buf_t * last_gen_buf; 941 lm_isle_t * requested_isle; 942 943 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_gen_isle_process nbytes = %d cid=%d\n", nbytes, tcp->cid); 944 945 gen_info = &rx_con->u.rx.gen_info; 946 requested_isle = _lm_tcp_isle_find(pdev,tcp,isle_num); 947 DbgBreakIf(!requested_isle); 948 949 if (nbytes) { 950 DbgBreakIf(!gen_info->wait_for_isle_left); 951 DbgBreakIf(gen_buf != NULL); 952 gen_info->wait_for_isle_left = FALSE; 953 gen_info->isle_nbytes += nbytes; 954 requested_isle->isle_nbytes += nbytes; 955 #if defined(_NTDDK_) 956 #pragma prefast (push) 957 #pragma prefast (disable:28182) // If nbytes is larger that zero than ((returned_list_of_gen_bufs))->head is not NULL. 958 #endif //_NTDDK_ 959 last_gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_head(&requested_isle->isle_gen_bufs_list_head); 960 DbgBreakIf(last_gen_buf->placed_bytes); 961 last_gen_buf->placed_bytes = (u16_t)nbytes; 962 #if defined(_NTDDK_) 963 #pragma prefast (pop) 964 #endif //_NTDDK_ 965 } else { 966 DbgBreakIf(gen_info->wait_for_isle_left); 967 DbgBreakIf(gen_buf == NULL); 968 DbgBreakIf(SIG(gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 969 DbgBreakIf(END_SIG(gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 970 gen_info->wait_for_isle_left = TRUE; 971 d_list_push_head(&requested_isle->isle_gen_bufs_list_head, &gen_buf->link); 972 pdev->toe_info.grqs[sb_idx].gen_bufs_in_isles_delta++; 973 } 974 #ifdef DEBUG_OOO_CQE 975 SET_DEBUG_OOO_INFO(requested_isle, CMP_OPCODE_TOE_GAIL, nbytes); 976 #endif 977 } 978 979 void lm_tcp_rx_gen_join_process(lm_device_t * pdev, lm_tcp_state_t * tcp, u8_t sb_idx, u8_t isle_num) 980 { 981 lm_tcp_con_t * rx_con = tcp->rx_con; 982 lm_tcp_con_rx_gen_info_t * gen_info; 983 lm_isle_t * start_isle; 984 d_list_t gen_buf_list; 985 u32_t isle_nbytes; 986 DbgMessage(pdev, VERBOSEl4rx, "##lm_tcp_rx_gen_join_process cid=%d\n", tcp->cid); 987 988 gen_info = &rx_con->u.rx.gen_info; 989 990 991 if (!isle_num) { 992 /* break if peninsula list isn't empty and the last buffer in list isn't released yet */ 993 DbgBreakIf(d_list_entry_cnt(&gen_info->dpc_peninsula_list) && 994 ((lm_tcp_gen_buf_t *)(d_list_peek_tail(&gen_info->dpc_peninsula_list)))->placed_bytes == 0); 995 d_list_init(&gen_buf_list, NULL, NULL, 0); 996 isle_nbytes = _lm_tcp_isle_remove(pdev, tcp, sb_idx, 1, &gen_buf_list); 997 // DbgBreakIf(!(isle_nbytes && d_list_entry_cnt(&gen_buf_list))); 998 if (d_list_entry_cnt(&gen_buf_list) > 1) { 999 DbgBreakIf(((lm_tcp_gen_buf_t *)(d_list_peek_head(&gen_buf_list)))->placed_bytes == 0); 1000 } 1001 pdev->toe_info.grqs[sb_idx].gen_bufs_in_isles_delta -= (s32_t)d_list_entry_cnt(&gen_buf_list); 1002 pdev->toe_info.grqs[sb_idx].number_of_isles_delta--; 1003 1004 if (!d_list_is_empty(&gen_buf_list)) { 1005 d_list_add_tail(&gen_info->dpc_peninsula_list, &gen_buf_list); 1006 } 1007 gen_info->dpc_peninsula_nbytes += isle_nbytes; 1008 gen_info->isle_nbytes -= isle_nbytes; 1009 } else { 1010 start_isle = _lm_tcp_isle_find(pdev,tcp,isle_num); 1011 d_list_init(&gen_buf_list, NULL, NULL, 0); 1012 isle_nbytes = _lm_tcp_isle_remove(pdev, tcp, sb_idx, isle_num + 1, &gen_buf_list); 1013 // DbgBreakIf(!(isle_nbytes && d_list_entry_cnt(&gen_buf_list))); 1014 pdev->toe_info.grqs[sb_idx].number_of_isles_delta--; 1015 if (d_list_entry_cnt(&gen_buf_list) > 1) { 1016 DbgBreakIf(((lm_tcp_gen_buf_t *)(d_list_peek_head(&gen_buf_list)))->placed_bytes == 0); 1017 } 1018 DbgBreakIf(((lm_tcp_gen_buf_t *)(d_list_peek_tail(&start_isle->isle_gen_bufs_list_head)))->placed_bytes == 0); 1019 if (!d_list_is_empty(&gen_buf_list)) { 1020 d_list_add_tail(&start_isle->isle_gen_bufs_list_head, &gen_buf_list); 1021 } 1022 start_isle->isle_nbytes += isle_nbytes; 1023 #ifdef DEBUG_OOO_CQE 1024 SET_DEBUG_OOO_INFO(start_isle,CMP_OPCODE_TOE_GJ,0); 1025 #endif 1026 } 1027 rx_con->dpc_info.dpc_flags &= ~(LM_TCP_DPC_TOO_BIG_ISLE | LM_TCP_DPC_TOO_MANY_ISLES); 1028 1029 } 1030 1031 static __inline lm_tcp_gen_buf_t * lm_tcp_rx_next_grq_buf(lm_device_t * pdev, u8_t sb_idx) 1032 { 1033 lm_tcp_gen_buf_t * gen_buf; 1034 1035 /* 11/12/2008 - TODO: Enhance locking acquisition method, 1036 * TBD: aggragate cons, and active_gen_list updates */ 1037 MM_ACQUIRE_TOE_GRQ_LOCK_DPC(pdev, sb_idx); 1038 1039 /* Get the generic buffer for this completion */ 1040 gen_buf = (lm_tcp_gen_buf_t *)d_list_pop_head(&pdev->toe_info.grqs[sb_idx].active_gen_list); 1041 if (ERR_IF(gen_buf == NULL)) { 1042 DbgBreakMsg("Received a fw GA/GAI without any generic buffers\n"); 1043 return NULL; 1044 } 1045 DbgBreakIf(!gen_buf); 1046 DbgBreakIf(SIG(gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 1047 DbgBreakIf(END_SIG(gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 1048 1049 /* each generic buffer is represented by ONE bd on the bd-chain */ 1050 lm_bd_chain_bds_consumed(&pdev->toe_info.grqs[sb_idx].bd_chain, 1); 1051 1052 MM_RELEASE_TOE_GRQ_LOCK_DPC(pdev, sb_idx); 1053 1054 return gen_buf; 1055 } 1056 1057 /** Description 1058 * completes the fast-path operations for a certain connection 1059 * Assumption: 1060 * fp-rx lock is taken 1061 * This function is mutual exclusive: there can only be one thread running it at a time. 1062 */ 1063 void lm_tcp_rx_complete_tcp_fp(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con) 1064 { 1065 lm_tcp_buffer_t * curr_tcp_buf; 1066 u32_t add_sws_bytes = 0; 1067 1068 if (con->dpc_info.dpc_comp_blocked) { 1069 /* we will no longer receive a "skp" */ 1070 SET_FLAGS(con->flags, TCP_POST_NO_SKP); /* so that new posts complete immediately... */ 1071 /* complete any outstanding skp bytes... */ 1072 if (tcp->rx_con->u.rx.skp_bytes_copied) { 1073 /* now we can complete these bytes that have already been copied... */ 1074 tcp->rx_con->bytes_comp_cnt += tcp->rx_con->u.rx.skp_bytes_copied; 1075 /* complete nbytes on buffers (dpc-flow ) */ 1076 lm_tcp_complete_nbytes(pdev, tcp, tcp->rx_con, tcp->rx_con->u.rx.skp_bytes_copied, /* push=*/ 0); 1077 tcp->rx_con->u.rx.skp_bytes_copied = 0; 1078 } 1079 } 1080 1081 /* TBA Michals FW BYPASS...copy here */ 1082 if (!d_list_is_empty(&con->u.rx.gen_info.dpc_peninsula_list)) { 1083 /* only copy if this is the end... otherwise, we will wait for that SKP... */ 1084 if (lm_tcp_next_entry_dpc_active_list(con) && con->u.rx.gen_info.dpc_peninsula_nbytes && con->dpc_info.dpc_comp_blocked) { 1085 /* couldn't have been posted buffers if peninsula exists... */ 1086 DbgBreakIf(!d_list_is_empty(&con->u.rx.gen_info.peninsula_list)); 1087 con->u.rx.gen_info.bytes_copied_cnt_in_comp += lm_tcp_rx_peninsula_to_rq(pdev, tcp, 0xffffffff,NON_EXISTENT_SB_IDX); 1088 } 1089 1090 /* check if we still have something in the peninsula after the copying AND our active tb list is empty... otherwise, it's intended 1091 * for that and we'll wait for the next RQ_SKP in the next DPC. UNLESS, we've got completion block, in which case RQ_SKP won't make it 1092 * way ever... */ 1093 curr_tcp_buf = lm_tcp_next_entry_dpc_active_list(con); 1094 DbgBreakIf(!d_list_is_empty(&con->u.rx.gen_info.dpc_peninsula_list) && curr_tcp_buf && con->dpc_info.dpc_comp_blocked); 1095 if (!d_list_is_empty(&con->u.rx.gen_info.dpc_peninsula_list) && !curr_tcp_buf) { 1096 d_list_add_tail(&con->u.rx.gen_info.peninsula_list, &con->u.rx.gen_info.dpc_peninsula_list); 1097 con->u.rx.gen_info.peninsula_nbytes += con->u.rx.gen_info.dpc_peninsula_nbytes; 1098 con->u.rx.gen_info.dpc_peninsula_nbytes = 0; 1099 1100 /* we want to leave any non-released buffer in the dpc_peninsula (so that we don't access the list w/o a lock) */ 1101 if (((lm_tcp_gen_buf_t *)d_list_peek_tail(&con->u.rx.gen_info.peninsula_list))->placed_bytes == 0) { 1102 lm_tcp_gen_buf_t * gen_buf; 1103 gen_buf = (lm_tcp_gen_buf_t *)d_list_pop_tail(&con->u.rx.gen_info.peninsula_list); 1104 if CHK_NULL(gen_buf) 1105 { 1106 DbgBreakIfAll( !gen_buf ) ; 1107 return; 1108 } 1109 d_list_init(&con->u.rx.gen_info.dpc_peninsula_list, &gen_buf->link, &gen_buf->link, 1); 1110 } else { 1111 d_list_clear(&con->u.rx.gen_info.dpc_peninsula_list); 1112 } 1113 1114 } 1115 } 1116 1117 /**** Client completing : may result in lock-release *****/ 1118 /* during lock-release, due to this function being called from service_deferred, more 1119 * cqes can be processed. We don't want to mix. This function is mutually exclusive, so 1120 * any processing makes it's way to being completed by calling this function. 1121 * the following define a "fast-path completion" 1122 * (i) RQ buffers to be completed 1123 * defined by dpc_completed_tail and are collected during lm_tcp_complete_bufs BEFORE lock 1124 * is released, so no more buffer processing can make it's way into this buffer completion. 1125 * (ii) GRQ buffers to be indicated 1126 * Are taken from peninsula, and not dpc_peninsula, so no NEW generic buffers can make their 1127 * way to this indication 1128 * (iii) Fin to be indicated 1129 * determined by the flags, since dpc_flags CAN be modified during processing we copy 1130 * them to a snapshot_flags parameter, which is initialized in this function only, so no fin 1131 * can can make its way in while we release the lock. 1132 * (iv) Remainders for sp 1133 * all sp operations are logged in dpc_flags. for the same reason as (iii) no sp commands can 1134 * make their way in during this fp-completion, all sp-processing after will relate to this point in time. 1135 */ 1136 /* NDC is the only fp flag: determining that we should complete all the processed cqes. Therefore, we can 1137 * turn it off here. We should turn it off, since if no sp flags are on, the sp-complete function shouldn't be called 1138 */ 1139 // RESET_FLAGS(con->dpc_info.dpc_flags, LM_TCP_DPC_NDC); 1140 con->dpc_info.snapshot_flags = con->dpc_info.dpc_flags; 1141 con->dpc_info.dpc_flags = 0; 1142 1143 /* compensate fw-window with the rq-placed bytes */ 1144 if (con->dpc_info.dpc_rq_placed_bytes) { 1145 add_sws_bytes += con->dpc_info.dpc_rq_placed_bytes; 1146 con->dpc_info.dpc_rq_placed_bytes = 0; 1147 } 1148 1149 1150 /* check if we completed a buffer that as a result unblocks the um from posting more (a split buffer that 1151 * was placed on the last bd). If this occured - we should not have any other RQs!!! */ 1152 if (con->dpc_info.dpc_unblock_post) { 1153 RESET_FLAGS(con->flags, TCP_POST_DELAYED); 1154 con->dpc_info.dpc_unblock_post = 0; 1155 } 1156 1157 /* NOTE: AFTER THIS STAGE DO NOT ACCESS DPC-INFO ANYMORE - for deferred cqes issue */ 1158 1159 /* complete buffers to client */ 1160 if (con->dpc_info.dpc_completed_tail != NULL) { 1161 lm_tcp_complete_bufs(pdev,tcp,con); 1162 } 1163 1164 /* Is there something left to indicate? */ 1165 if (!d_list_is_empty(&con->u.rx.gen_info.peninsula_list) && _lm_tcp_ok_to_indicate(con)) { 1166 mm_tcp_rx_indicate_gen(pdev,tcp); 1167 add_sws_bytes += tcp->rx_con->u.rx.gen_info.add_sws_bytes; /* any bytes we need to update will be aggregated here during indicate */ 1168 tcp->rx_con->u.rx.gen_info.add_sws_bytes = 0; 1169 } 1170 1171 if (add_sws_bytes) { 1172 lm_tcp_rx_post_sws(pdev, tcp, con, add_sws_bytes, TCP_RX_POST_SWS_INC); 1173 } 1174 1175 } 1176 1177 1178 /** Description 1179 * processes a single cqe. 1180 */ 1181 void lm_tcp_rx_process_cqe( 1182 lm_device_t * pdev, 1183 struct toe_rx_cqe * cqe, 1184 lm_tcp_state_t * tcp, 1185 u8_t sb_idx) 1186 { 1187 u32_t nbytes; 1188 u8_t cmd; 1189 u8_t isle_num = 0; 1190 1191 cmd = ((cqe->params1 & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT); 1192 1193 1194 /* Check that the cqe nbytes make sense, we could have got here by chance... */ 1195 /* update completion has a different usage for nbyts which is a sequence -so any number is valid*/ 1196 if(IS_OOO_CQE(cmd)) { 1197 nbytes = (cqe->data.ooo_params.ooo_params & TOE_RX_CQE_OOO_PARAMS_NBYTES) >> TOE_RX_CQE_OOO_PARAMS_NBYTES_SHIFT; 1198 isle_num = (cqe->data.ooo_params.ooo_params & TOE_RX_CQE_OOO_PARAMS_ISLE_NUM) >> TOE_RX_CQE_OOO_PARAMS_ISLE_NUM_SHIFT; 1199 if (((isle_num == 0) && (cmd != CMP_OPCODE_TOE_GJ)) || (isle_num > T_TCP_MAX_ISLES_PER_CONNECTION_TOE)) { 1200 DbgMessage(pdev, FATAL, "Isle number %d is not valid for OOO CQE %d\n", isle_num, cmd); 1201 DbgBreak(); 1202 } 1203 } else if (cmd == RAMROD_OPCODE_TOE_UPDATE) { 1204 nbytes = cqe->data.raw_data; 1205 } else { 1206 nbytes = (cqe->data.in_order_params.in_order_params & TOE_RX_CQE_IN_ORDER_PARAMS_NBYTES) >> TOE_RX_CQE_IN_ORDER_PARAMS_NBYTES_SHIFT; 1207 DbgBreakIfAll(nbytes & 0xc0000000); /* two upper bits on show a completion larger than 1GB - a bit odd...*/ 1208 DbgBreakIf(nbytes && tcp->rx_con->dpc_info.dpc_comp_blocked); 1209 } 1210 if (pdev->toe_info.archipelago.l4_decrease_archipelago 1211 && d_list_entry_cnt(&tcp->rx_con->u.rx.gen_info.first_isle.isle_gen_bufs_list_head)) { 1212 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_TOO_MANY_ISLES; 1213 } 1214 switch(cmd) 1215 { 1216 case CMP_OPCODE_TOE_SRC_ERR: 1217 DbgMessage(pdev, FATAL, "ERROR: NO SEARCHER ENTRY!\n"); 1218 DbgBreakIfAll(TRUE); 1219 return; 1220 case CMP_OPCODE_TOE_GA: 1221 //DbgMessage(pdev, WARN, "GenericAdd cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1222 lm_tcp_rx_gen_peninsula_process(pdev, tcp, nbytes, 1223 lm_tcp_rx_next_grq_buf(pdev, sb_idx)); 1224 return; 1225 case CMP_OPCODE_TOE_GNI: 1226 //DbgMessage(pdev, WARN, "GenericCreateIsle cid=%d isle_num=%d!\n", tcp->cid, isle_num); 1227 DbgBreakIf(nbytes); 1228 lm_tcp_rx_gen_isle_create(pdev, tcp, 1229 lm_tcp_rx_next_grq_buf(pdev, sb_idx), sb_idx, isle_num); 1230 return; 1231 case CMP_OPCODE_TOE_GAIR: 1232 //DbgMessage(pdev, WARN, "GenericAddIsleR cid=%d isle_num=%d nbytes=%d!\n", tcp->cid, isle_num, nbytes); 1233 lm_tcp_rx_gen_isle_right_process(pdev, tcp, nbytes, 1234 lm_tcp_rx_next_grq_buf(pdev, sb_idx), sb_idx, isle_num); 1235 return; 1236 case CMP_OPCODE_TOE_GAIL: 1237 DbgMessage(pdev, WARN, "GenericAddIsleL cid=%d isle_num=%d nbytes=%d!\n", tcp->cid, isle_num, nbytes); 1238 if (nbytes) 1239 { 1240 lm_tcp_rx_gen_isle_left_process(pdev, tcp, nbytes, 1241 NULL, sb_idx, isle_num); 1242 } 1243 else 1244 { 1245 lm_tcp_rx_gen_isle_left_process(pdev, tcp, 0, 1246 lm_tcp_rx_next_grq_buf(pdev, sb_idx), sb_idx, isle_num); 1247 } 1248 return; 1249 case CMP_OPCODE_TOE_GRI: 1250 // DbgMessage(pdev, WARN, "GenericReleaseIsle cid=%d isle_num=%d nbytes=%d!\n", tcp->cid, isle_num, nbytes); 1251 lm_tcp_rx_gen_isle_right_process(pdev, tcp, nbytes, NULL, sb_idx, isle_num); 1252 return; 1253 case CMP_OPCODE_TOE_GR: 1254 //DbgMessage(pdev, WARN, "GenericRelease cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1255 lm_tcp_rx_gen_peninsula_process(pdev, tcp, nbytes, NULL); 1256 return; 1257 case CMP_OPCODE_TOE_GJ: 1258 //DbgMessage(pdev, WARN, "GenericJoin cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1259 lm_tcp_rx_gen_join_process(pdev, tcp, sb_idx, isle_num); 1260 return; 1261 case CMP_OPCODE_TOE_CMP: 1262 //DbgMessage(pdev, WARN, "Cmp(push) cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1263 /* Add fast path handler here */ 1264 lm_tcp_rx_cmp_process(pdev, tcp, nbytes, 1); 1265 return; 1266 case CMP_OPCODE_TOE_REL: 1267 //DbgMessage(pdev, WARN, "Rel(nopush) cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1268 lm_tcp_rx_cmp_process(pdev, tcp, nbytes, 0); 1269 return; 1270 case CMP_OPCODE_TOE_SKP: 1271 //DbgMessage(pdev, WARN, "Skp cid=%d nbytes=%d!\n", tcp->cid, cqe->nbytes); 1272 lm_tcp_rx_skp_process(pdev, tcp, nbytes, sb_idx); 1273 return; 1274 case CMP_OPCODE_TOE_DGI: 1275 DbgMessage(pdev, WARN, "Delete Isle cid=%d!\n", tcp->cid); 1276 lm_tcp_rx_delete_isle(pdev, tcp, sb_idx, isle_num, nbytes); 1277 return; 1278 } 1279 1280 /* for the rest of the commands, if we have nbytes, we need to complete them (generic/app) */ 1281 /* unless it's an update completion, in which case the nbytes has a different meaning. */ 1282 if ((cmd != RAMROD_OPCODE_TOE_UPDATE) && nbytes) { 1283 lm_tcp_gen_buf_t * gen_buf; 1284 gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_tail(&tcp->rx_con->u.rx.gen_info.dpc_peninsula_list); 1285 if(gen_buf && (gen_buf->placed_bytes == 0)) { 1286 lm_tcp_rx_gen_peninsula_process(pdev, tcp, nbytes, NULL); 1287 } else { 1288 /* if we're here - we will no longer see a RQ_SKP so, let's simulate one...note if we didn't get nbytes here.. we still need 1289 * to take care of this later if it's a blocking completion the skip will have to be everything in the peninsula 1290 * we can access skp_bytes here lockless, because the only time it will be accessed in post is if there is something in the peninsula, if we got a RQ_SKP here, there can't be...*/ 1291 DbgBreakIf(!d_list_is_empty(&tcp->rx_con->u.rx.gen_info.peninsula_list)); 1292 DbgBreakIf(tcp->rx_con->rq_nbytes <= tcp->rx_con->u.rx.gen_info.dpc_peninsula_nbytes+tcp->rx_con->u.rx.skp_bytes_copied); // we got a RQ completion here... so peninsula CAN;T cover RQ!!! 1293 lm_tcp_rx_skp_process(pdev, tcp, tcp->rx_con->u.rx.gen_info.dpc_peninsula_nbytes+tcp->rx_con->u.rx.skp_bytes_copied, sb_idx); 1294 1295 /* We give push=1 here, this will seperate between 'received' data and 'aborted' bufs. we won't 1296 * have any buffers left that need to be aborted that have partial completed data on them */ 1297 lm_tcp_rx_cmp_process(pdev, tcp, nbytes, 2 /* push as result of sp-completion*/); 1298 } 1299 } 1300 1301 switch (cmd) { 1302 case CMP_OPCODE_TOE_FIN_RCV: 1303 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_RECV; 1304 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_FIN_RECEIVED */ 1305 return; 1306 case CMP_OPCODE_TOE_FIN_UPL: 1307 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_RECV_UPL; 1308 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_FIN_RECEIVED + Request to upload the connection */ 1309 return; 1310 case CMP_OPCODE_TOE_RST_RCV: 1311 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RESET_RECV; 1312 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_RST_RECEIVED */ 1313 return; 1314 case RAMROD_OPCODE_TOE_UPDATE: 1315 DbgBreakIf( (tcp->hdr.status != STATE_STATUS_NORMAL) && (tcp->hdr.status != STATE_STATUS_ABORTED)); 1316 DbgBreakIf(tcp->sp_request == NULL); 1317 DbgBreakIf((tcp->sp_request->type != SP_REQUEST_UPDATE_NEIGH) && 1318 (tcp->sp_request->type != SP_REQUEST_UPDATE_PATH) && 1319 (tcp->sp_request->type != SP_REQUEST_UPDATE_TCP) && 1320 (tcp->sp_request->type != SP_REQUEST_UPDATE_PATH_RELINK)); 1321 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1322 1323 /*DbgMessage(pdev, FATAL, "lm_tcp_rx_process_cqe() RAMROD_OPCODE_TOE_UPDATE: IGNORE_WND_UPDATES=%d, cqe->nbytes=%d\n", GET_FLAGS(tcp->rx_con->db_data.rx->flags, TOE_RX_DB_DATA_IGNORE_WND_UPDATES), cqe->nbytes);*/ 1324 1325 if ((tcp->sp_request->type == SP_REQUEST_UPDATE_TCP) && (GET_FLAGS(tcp->rx_con->db_data.rx->flags, TOE_RX_DB_DATA_IGNORE_WND_UPDATES))) 1326 { 1327 tcp->rx_con->dpc_info.dpc_fw_wnd_after_dec = nbytes; 1328 } 1329 return; 1330 case CMP_OPCODE_TOE_URG: 1331 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_URG; 1332 return; 1333 case CMP_OPCODE_TOE_MAX_RT: 1334 DbgMessage(pdev, WARNl4, "lm_tcp_rx_process_cqe: CMP_OPCODE_TOE_MAX_RT cid=%d\n", tcp->cid); 1335 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RT_TO; 1336 return; 1337 case CMP_OPCODE_TOE_RT_TO: 1338 DbgMessage(pdev, WARNl4, "lm_tcp_rx_process_cqe: CMP_OPCODE_TOE_RT_TO cid=%d\n", tcp->cid); 1339 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RT_TO; 1340 return; 1341 case CMP_OPCODE_TOE_KA_TO: 1342 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_KA_TO; 1343 return; 1344 case CMP_OPCODE_TOE_DBT_RE: 1345 /* LH Inbox specification: Black Hole detection (RFC 2923) 1346 * TCP Chimney target MUST upload the connection if the TCPDoubtReachabilityRetransmissions threshold is hit. 1347 * SPARTA test scripts and tests that will fail if not implemented: All tests in Tcp_BlackholeDetection.wsf, we cause 1348 * the upload by giving L4_UPLOAD_REASON_UPLOAD_REQUEST (same as Teton) */ 1349 DbgMessage(pdev, INFORMl4, "lm_tcp_rx_process_cqe: RCQE CMP_OPCODE_TOE_DBT_RE, cid=%d\n", tcp->cid); 1350 DbgMessage(pdev, WARNl4, "lm_tcp_rx_process_cqe: RCQE CMP_OPCODE_TOE_DBT_RE, cid=%d IGNORING!!!\n", tcp->cid); 1351 /* We add this here only for windows and not ediag */ 1352 #if (!defined(DOS)) && (!defined(__LINUX)) 1353 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_DBT_RE; 1354 #endif 1355 return; 1356 case CMP_OPCODE_TOE_SYN: 1357 case CMP_OPCODE_TOE_FW2_TO: 1358 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_UPLD_CLOSE; 1359 return; 1360 case CMP_OPCODE_TOE_2WY_CLS: 1361 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_UPLD_CLOSE; 1362 return; 1363 case CMP_OPCODE_TOE_OPT_ERR: 1364 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_OPT_ERR; 1365 return; 1366 case RAMROD_OPCODE_TOE_QUERY: 1367 DbgBreakIf(! tcp->sp_request ); 1368 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_QUERY); 1369 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1370 return; 1371 case RAMROD_OPCODE_TOE_SEARCHER_DELETE: 1372 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE_OFFLOAD); 1373 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1374 return; 1375 case RAMROD_OPCODE_TOE_RESET_SEND: 1376 DbgBreakIf(! tcp->sp_request); 1377 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_ABORTIVE_DISCONNECT); 1378 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1379 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_RST_REQ_COMPLETED */ 1380 return; 1381 case RAMROD_OPCODE_TOE_INVALIDATE: 1382 DbgBreakIf(! tcp->sp_request); 1383 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INVALIDATE); 1384 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1385 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_INV_REQ_COMPLETED */ 1386 return; 1387 case RAMROD_OPCODE_TOE_TERMINATE: 1388 DbgBreakIf(! tcp->sp_request); 1389 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE1_OFFLOAD); 1390 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1391 tcp->rx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_TRM_REQ_COMPLETED */ 1392 return; 1393 case RAMROD_OPCODE_TOE_EMPTY_RAMROD: 1394 DbgBreakIf(nbytes); 1395 DbgBreakIf(! tcp->sp_request ); 1396 DbgBreakIf((tcp->sp_request->type != SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT) && 1397 (tcp->sp_request->type != SP_REQUEST_PENDING_REMOTE_DISCONNECT) && 1398 (tcp->sp_request->type != SP_REQUEST_PENDING_TX_RST)); 1399 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1400 return; 1401 case RAMROD_OPCODE_TOE_INITIATE_OFFLOAD: 1402 DbgBreakIf(nbytes); 1403 DbgBreakIf(! tcp->sp_request ); 1404 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INITIATE_OFFLOAD); 1405 1406 /* 13/08/08 NirV: bugbug, temp workaround for dpc watch dog bug, 1407 * complete ofld request here - assumption: tcp lock is NOT taken by caller */ 1408 lm_tcp_comp_initiate_offload_request(pdev, tcp, LM_STATUS_SUCCESS); 1409 lm_sq_complete(pdev, CMD_PRIORITY_NORMAL, RAMROD_OPCODE_TOE_INITIATE_OFFLOAD, tcp->ulp_type, tcp->cid); 1410 1411 return; 1412 case CMP_OPCODE_TOE_LCN_ERR: 1413 DbgBreakIf(! tcp->sp_request ); 1414 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INITIATE_OFFLOAD); 1415 tcp->rx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 1416 return; 1417 default: 1418 DbgMessage(pdev, FATAL, "unexpected rx cqe opcode=%d\n", cmd); 1419 DbgBreakIfAll(TRUE); 1420 } 1421 } 1422 1423 u8_t lm_tcp_rx_process_cqes(lm_device_t *pdev, u8_t drv_toe_rss_id, s_list_t * connections) 1424 { 1425 lm_tcp_rcq_t *rcq; 1426 lm_tcp_grq_t *grq; 1427 struct toe_rx_cqe *cqe, *hist_cqe; 1428 lm_tcp_state_t *tcp = NULL; 1429 u32_t cid; 1430 u32_t avg_dpc_cnt; 1431 u16_t cq_new_idx; 1432 u16_t cq_old_idx; 1433 u16_t num_to_reproduce = 0; 1434 u8_t defer_cqe; 1435 u8_t process_rss_upd_later = FALSE; 1436 MM_INIT_TCP_LOCK_HANDLE(); 1437 1438 DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_rx_process_cqes START\n"); 1439 1440 rcq = &pdev->toe_info.rcqs[drv_toe_rss_id]; 1441 grq = &pdev->toe_info.grqs[drv_toe_rss_id]; 1442 cq_new_idx = *(rcq->hw_con_idx_ptr); 1443 cq_old_idx = lm_bd_chain_cons_idx(&rcq->bd_chain); 1444 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) < 0); 1445 1446 /* save statistics */ 1447 rcq->num_cqes_last_dpc = S16_SUB(cq_new_idx, cq_old_idx); 1448 DbgMessage(pdev, VERBOSEl4int, "###lm_tcp_rx_process_cqes num_cqes=%d\n", rcq->num_cqes_last_dpc); 1449 1450 if (rcq->num_cqes_last_dpc) { /* Exclude zeroed value from statistics*/ 1451 if(rcq->max_cqes_per_dpc < rcq->num_cqes_last_dpc) { 1452 rcq->max_cqes_per_dpc = rcq->num_cqes_last_dpc; 1453 } 1454 /* we don't want to wrap around...*/ 1455 if ((rcq->sum_cqes_last_x_dpcs + rcq->num_cqes_last_dpc) < rcq->sum_cqes_last_x_dpcs) { 1456 rcq->avg_dpc_cnt = 0; 1457 rcq->sum_cqes_last_x_dpcs = 0; 1458 } 1459 rcq->sum_cqes_last_x_dpcs += rcq->num_cqes_last_dpc; 1460 rcq->avg_dpc_cnt++; 1461 avg_dpc_cnt = rcq->avg_dpc_cnt; 1462 if (avg_dpc_cnt) { /*Prevent division by 0*/ 1463 rcq->avg_cqes_per_dpc = rcq->sum_cqes_last_x_dpcs / avg_dpc_cnt; 1464 } else { 1465 rcq->sum_cqes_last_x_dpcs = 0; 1466 } 1467 } 1468 1469 1470 /* if we are suspended, we need to check if we can resume processing */ 1471 if (rcq->suspend_processing == TRUE) { 1472 lm_tcp_rss_update_suspend_rcq(pdev, rcq); 1473 if (rcq->suspend_processing == TRUE) { 1474 /* skip the consumption loop */ 1475 cq_new_idx = cq_old_idx; 1476 DbgMessage(pdev, VERBOSEl4int, "lm_tcp_rx_process_cqes(): rcq suspended - idx:%d\n", drv_toe_rss_id); 1477 } 1478 } 1479 1480 while(cq_old_idx != cq_new_idx) { 1481 u32_t update_stats_type; 1482 u8_t opcode; 1483 1484 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0); 1485 1486 /* get next consumed cqe */ 1487 cqe = lm_toe_bd_chain_consume_bd(&rcq->bd_chain); 1488 update_stats_type = cqe->data.raw_data; 1489 DbgBreakIf(!cqe); 1490 num_to_reproduce++; 1491 1492 /* get cid and opcode from cqe */ 1493 cid = SW_CID(((cqe->params1 & TOE_RX_CQE_CID) >> TOE_RX_CQE_CID_SHIFT)); 1494 opcode = (cqe->params1 & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT; 1495 1496 if (opcode == RAMROD_OPCODE_TOE_RSS_UPDATE) { 1497 1498 /* update the saved consumer */ 1499 cq_old_idx = lm_bd_chain_cons_idx(&rcq->bd_chain); 1500 1501 /* rss update ramrod */ 1502 DbgMessage(pdev, INFORMl4int, "lm_tcp_rx_process_cqes(): calling lm_tcp_rss_update_ramrod_comp - drv_toe_rss_id:%d\n", drv_toe_rss_id); 1503 if (num_to_reproduce > 1) { 1504 process_rss_upd_later = TRUE; 1505 lm_tcp_rss_update_ramrod_comp(pdev, rcq, cid, update_stats_type, FALSE); 1506 break; 1507 } 1508 lm_tcp_rss_update_ramrod_comp(pdev, rcq, cid, update_stats_type, TRUE); 1509 1510 /* suspend further RCQ processing (if needed) */ 1511 if (rcq->suspend_processing == TRUE) 1512 break; 1513 else 1514 continue; 1515 1516 } 1517 1518 if (cid < MAX_ETH_REG_CONS) { 1519 /* toe init ramrod */ 1520 DbgBreakIf(((cqe->params1 & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT) 1521 != RAMROD_OPCODE_TOE_INIT); 1522 lm_tcp_init_ramrod_comp(pdev); 1523 cq_old_idx = lm_bd_chain_cons_idx(&rcq->bd_chain); 1524 DbgBreakIf(cq_old_idx != cq_new_idx); 1525 /* We need to update the slow-path ring. This is usually done in the lm_tcp_rx_complete_sp_cqes, 1526 * but we won't get there since this completion is not associated with a connection. USUALLY we 1527 * have to update the sp-ring only AFTER we've written the CQ producer, this is to promise that there 1528 * will always be an empty entry for another ramrod completion, but in this case we're safe, since only 1529 * one CQE is occupied anyway */ 1530 lm_sq_complete(pdev, CMD_PRIORITY_NORMAL, RAMROD_OPCODE_TOE_INIT, TOE_CONNECTION_TYPE, LM_SW_LEADING_RSS_CID(pdev)); 1531 break; 1532 } 1533 1534 tcp = lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, cid); 1535 DbgBreakIf(!tcp); 1536 /* save cqe in history_cqes */ 1537 hist_cqe = (struct toe_rx_cqe *)lm_tcp_qe_buffer_next_cqe_override(&tcp->rx_con->history_cqes); 1538 *hist_cqe = *cqe; 1539 1540 /* ASSUMPTION: if COMP_DEFERRED changes from FALSE to TRUE, the change occurs only in DPC 1541 * o/w it can only change from TRUE to FALSE. 1542 * 1543 * Read flag w/o lock. Flag may change by the time we call rx_defer_cqe 1544 * Need to check again under lock. We want to avoid acquiring the lock every DPC */ 1545 defer_cqe = ((tcp->rx_con->flags & TCP_RX_COMP_DEFERRED) == TCP_RX_COMP_DEFERRED); 1546 if (defer_cqe) { 1547 /* if we're deferring completions - just store the cqe and continue to the next one 1548 * Assumptions: ALL commands need to be deferred, we aren't expecting any command on 1549 * L4 that we should pay attention to for this connection ( only one outstanding sp at a time ) */ 1550 /* Return if we are still deferred (may have changed since initial check was w/o a lock */ 1551 mm_acquire_tcp_lock(pdev, tcp->rx_con); 1552 /* check again under lock if we're deferred */ 1553 defer_cqe = ((tcp->rx_con->flags & TCP_RX_COMP_DEFERRED) == TCP_RX_COMP_DEFERRED); 1554 if (defer_cqe) { 1555 tcp->rx_con->flags |= TCP_DEFERRED_PROCESSING; 1556 1557 /* 13/08/08 NirV: bugbug, temp workaround for dpc watch dog bug, 1558 * release the tcp lock if cqe is offload complete */ 1559 if (((cqe->params1 & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT) == RAMROD_OPCODE_TOE_INITIATE_OFFLOAD) 1560 { 1561 mm_release_tcp_lock(pdev, tcp->rx_con); 1562 } 1563 1564 lm_tcp_rx_process_cqe(pdev,cqe,tcp,drv_toe_rss_id); 1565 } 1566 1567 /* 13/08/08 NirV: bugbug, temp workaround for dpc watch dog bug, 1568 * release the tcp lock if cqe is not offload complete (was released earlier) */ 1569 if (((cqe->params1 & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT) != RAMROD_OPCODE_TOE_INITIATE_OFFLOAD) 1570 { 1571 mm_release_tcp_lock(pdev, tcp->rx_con); 1572 } 1573 } 1574 1575 if (!defer_cqe) { 1576 /* connections will always be initialized to a dummy, so once a tcp connection is added to the 1577 * list, it's link will be initialized to point to another link other than NULL */ 1578 if (s_list_next_entry(&tcp->rx_con->dpc_info.link) == NULL) { 1579 s_list_push_head(connections, &tcp->rx_con->dpc_info.link); 1580 } 1581 lm_tcp_rx_process_cqe(pdev, cqe, tcp, drv_toe_rss_id); 1582 } 1583 1584 cq_old_idx = lm_bd_chain_cons_idx(&rcq->bd_chain); 1585 } 1586 1587 /* We may have nothing to reproduce if we were called from a sw_dpc */ 1588 if (num_to_reproduce) { 1589 lm_toe_bd_chain_bds_produced(&rcq->bd_chain, num_to_reproduce); 1590 1591 /* GilR 5/13/2006 - TBA - save some stats? */ 1592 1593 /* notify the fw of the prod of the RCQ */ 1594 LM_INTMEM_WRITE16(pdev, USTORM_TOE_CQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)), 1595 lm_bd_chain_prod_idx(&rcq->bd_chain), BAR_USTRORM_INTMEM); 1596 1597 if (pdev->params.enable_dynamic_hc[HC_INDEX_TOE_RX_CQ_CONS]) { 1598 u32_t l4_quasi_byte_counter; 1599 u16_t prod_idx_diff = lm_bd_chain_prod_idx(&rcq->bd_chain) - rcq->bd_chain.bds_per_page * rcq->bd_chain.page_cnt; 1600 l4_quasi_byte_counter = prod_idx_diff; 1601 l4_quasi_byte_counter <<= 16; 1602 //fIXME 1603 LM_INTMEM_WRITE32(pdev, rcq->hc_sb_info.iro_dhc_offset, l4_quasi_byte_counter, BAR_CSTRORM_INTMEM); 1604 } 1605 } 1606 DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_rx_process_cqes END\n"); 1607 return process_rss_upd_later; 1608 } 1609 1610 /** Description 1611 * compensate the grq 1612 * Assumption: 1613 * called under the GRQ LOCK 1614 */ 1615 void lm_tcp_rx_compensate_grq(lm_device_t * pdev, u8_t drv_toe_rss_id) 1616 { 1617 d_list_t * collected_gen_bufs_list = &pdev->toe_info.grqs[drv_toe_rss_id].aux_gen_list; 1618 1619 MM_ACQUIRE_TOE_GRQ_LOCK_DPC(pdev, drv_toe_rss_id); 1620 if (lm_tcp_rx_fill_grq(pdev, drv_toe_rss_id, collected_gen_bufs_list,FILL_GRQ_FULL)) { 1621 DbgMessage(pdev, INFORMl4rx, "lm_toe_service_rx_intr: Updating GRQ producer\n"); 1622 /* notify the fw of the prod of the GRQ */ 1623 LM_INTMEM_WRITE16(pdev, USTORM_TOE_GRQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)), 1624 lm_bd_chain_prod_idx(&pdev->toe_info.grqs[drv_toe_rss_id].bd_chain), BAR_USTRORM_INTMEM); 1625 } 1626 /* check if occupancy is above threshold */ 1627 if (pdev->toe_info.grqs[drv_toe_rss_id].bd_chain.capacity - pdev->toe_info.grqs[drv_toe_rss_id].bd_chain.bd_left < GRQ_XON_TH) { 1628 pdev->toe_info.grqs[drv_toe_rss_id].grq_compensate_on_alloc = TRUE; 1629 } else { 1630 pdev->toe_info.grqs[drv_toe_rss_id].grq_compensate_on_alloc = FALSE; 1631 } 1632 1633 MM_RELEASE_TOE_GRQ_LOCK_DPC(pdev, drv_toe_rss_id); 1634 1635 if (!d_list_is_empty(collected_gen_bufs_list)) { 1636 mm_tcp_return_list_of_gen_bufs(pdev,collected_gen_bufs_list,0, NON_EXISTENT_SB_IDX); 1637 d_list_clear(collected_gen_bufs_list); 1638 } 1639 } 1640 1641 static __inline void lm_tcp_rx_lock_grq(lm_device_t *pdev, u8_t drv_toe_rss_id) 1642 { 1643 /* If we've asked for compensation on allocation (which is only set from within a dpc) 1644 * there is a risk of the grq being accessed from a different context (the alloc context) 1645 * therefore, we cancel this option. Needs to be under lock in case alloc context is already 1646 * compensating */ 1647 if (pdev->toe_info.grqs[drv_toe_rss_id].grq_compensate_on_alloc) { 1648 MM_ACQUIRE_TOE_GRQ_LOCK_DPC(pdev, drv_toe_rss_id); 1649 pdev->toe_info.grqs[drv_toe_rss_id].grq_compensate_on_alloc = FALSE; 1650 MM_RELEASE_TOE_GRQ_LOCK_DPC(pdev, drv_toe_rss_id); 1651 } 1652 } 1653 1654 void lm_toe_service_rx_intr(lm_device_t *pdev, u8_t drv_toe_rss_id) 1655 { 1656 s_list_t connections; 1657 s_list_entry_t dummy; 1658 lm_tcp_con_t * con; 1659 lm_tcp_state_t * tcp; 1660 u32_t dbg_loop_cnt = 0; 1661 u8_t process_rss_upd; 1662 1663 MM_INIT_TCP_LOCK_HANDLE(); 1664 1665 DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_rx_intr START\n"); 1666 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.rcqs) > drv_toe_rss_id)); 1667 1668 /* lock the grq from external access: i.e.. allocation compensation */ 1669 lm_tcp_rx_lock_grq(pdev, drv_toe_rss_id); 1670 1671 while (TRUE) { 1672 dbg_loop_cnt++; 1673 s_list_clear(&connections); 1674 s_list_push_head(&connections, &dummy); 1675 /* process the cqes and initialize connections with all the connections that appeared 1676 * in the DPC */ 1677 process_rss_upd = lm_tcp_rx_process_cqes(pdev,drv_toe_rss_id,&connections); 1678 1679 /* Compensate the GRQ with generic buffers from the pool : process_cqes takes buffers from the grq */ 1680 lm_tcp_rx_compensate_grq(pdev,drv_toe_rss_id); 1681 1682 /* FP: traverse the connections. remember to ignore the last one */ 1683 con = (lm_tcp_con_t *)s_list_peek_head(&connections); 1684 tcp = con->tcp_state; 1685 while (s_list_next_entry(&con->dpc_info.link) != NULL) { 1686 mm_acquire_tcp_lock(pdev, con); 1687 lm_tcp_rx_complete_tcp_fp(pdev, con->tcp_state, con); 1688 mm_release_tcp_lock(pdev, con); 1689 con = (lm_tcp_con_t *)s_list_next_entry(&con->dpc_info.link); 1690 tcp = con->tcp_state; 1691 } 1692 1693 /* SP : traverse the connections. remember to ignore the last one */ 1694 con = (lm_tcp_con_t *)s_list_pop_head(&connections); 1695 s_list_next_entry(&con->dpc_info.link) = NULL; 1696 tcp = con->tcp_state; 1697 while (s_list_entry_cnt(&connections) > 0) { 1698 /* we access snapshot and not dpc, since once the dpc_flags were copied 1699 * to snapshot they were zeroized */ 1700 if (con->dpc_info.snapshot_flags) { 1701 lm_tcp_rx_complete_tcp_sp(pdev, tcp, con); 1702 } 1703 con = (lm_tcp_con_t *)s_list_pop_head(&connections); 1704 s_list_next_entry(&con->dpc_info.link) = NULL; 1705 tcp = con->tcp_state; 1706 } 1707 1708 if (process_rss_upd) { 1709 lm_tcp_rss_update_suspend_rcq(pdev,&pdev->toe_info.rcqs[drv_toe_rss_id]); 1710 if (!pdev->toe_info.rcqs[drv_toe_rss_id].suspend_processing) { 1711 pdev->toe_info.rcqs[drv_toe_rss_id].rss_update_processing_continued++; 1712 continue; 1713 } 1714 } 1715 break; 1716 } 1717 if (pdev->toe_info.rcqs[drv_toe_rss_id].rss_update_processing_max_continued < dbg_loop_cnt) { 1718 pdev->toe_info.rcqs[drv_toe_rss_id].rss_update_processing_max_continued = dbg_loop_cnt; 1719 } 1720 1721 if (pdev->toe_info.grqs[drv_toe_rss_id].number_of_isles_delta || pdev->toe_info.grqs[drv_toe_rss_id].gen_bufs_in_isles_delta) { 1722 MM_ACQUIRE_ISLES_CONTROL_LOCK_DPC(pdev); 1723 lm_tcp_update_isles_cnts(pdev, pdev->toe_info.grqs[drv_toe_rss_id].number_of_isles_delta, 1724 pdev->toe_info.grqs[drv_toe_rss_id].gen_bufs_in_isles_delta); 1725 MM_RELEASE_ISLES_CONTROL_LOCK_DPC(pdev); 1726 pdev->toe_info.grqs[drv_toe_rss_id].number_of_isles_delta = pdev->toe_info.grqs[drv_toe_rss_id].gen_bufs_in_isles_delta = 0; 1727 } 1728 1729 DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_rx_intr END\n"); 1730 } 1731 1732 /** Description: 1733 * Post a single tcp buffer to the Rx bd chain 1734 * Assumptions: 1735 * - caller initiated tcp_buf->flags field with BUFFER_START/BUFFER_END/PUSH appropriately 1736 * Returns: 1737 * - SUCCESS - tcp buf was successfully attached to the bd chain 1738 * - RESOURCE - not enough available BDs on bd chain for given tcp buf 1739 * - CONNECTION_CLOSED - whenever connection's flag are marked as 'POST BLOCKED' */ 1740 lm_status_t lm_tcp_rx_post_buf( 1741 struct _lm_device_t *pdev, 1742 lm_tcp_state_t *tcp, 1743 lm_tcp_buffer_t *tcp_buf, 1744 lm_frag_list_t *frag_list 1745 ) 1746 { 1747 lm_tcp_con_t * rx_con; 1748 lm_tcp_con_rx_gen_info_t * gen_info; 1749 lm_status_t lm_stat = LM_STATUS_SUCCESS; 1750 d_list_t return_list; /* buffers to return to pool in case of copying to buffer */ 1751 u32_t copied_bytes = 0; 1752 u32_t add_sws_bytes = 0; 1753 u8_t split_buffer = FALSE; 1754 1755 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_post_buf cid=%d\n", tcp->cid); 1756 DbgBreakIf(!(pdev && tcp)); 1757 DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid))); 1758 /* (tcp_buf==NULL <=> frag_list==NULL) && (frag_list!= NULL => frag_list->cnt != 0) */ 1759 DbgBreakIf( ( ! ( ( (!tcp_buf) && (!frag_list) ) || (tcp_buf && frag_list) ) ) || 1760 ( frag_list && (frag_list->cnt == 0) ) ); 1761 1762 rx_con = tcp->rx_con; 1763 if ( GET_FLAGS(rx_con->flags, TCP_RX_POST_BLOCKED) ) { 1764 // DbgBreakIf(!tcp_buf); /* (lm_tcp_rx_post_buf design guides VBD doc) */ 1765 if (!tcp_buf) { 1766 tcp->rx_con->zb_rx_post_blocked++; 1767 return LM_STATUS_SUCCESS; 1768 } else { 1769 tcp->rx_con->rx_post_blocked++; 1770 return LM_STATUS_CONNECTION_CLOSED; 1771 } 1772 } 1773 1774 /* TCP_POST_DELAYED is turned on when the lm can not process new buffers for some reason, but not permanently 1775 * Assumption: UM will eventually try to repost this buffer... */ 1776 if ( GET_FLAGS(rx_con->flags, TCP_POST_DELAYED)) { 1777 return LM_STATUS_FAILURE; 1778 } 1779 1780 RESET_FLAGS(rx_con->flags, TCP_INDICATE_REJECTED); 1781 1782 /* set tcp_buf fields */ 1783 if (tcp_buf) { 1784 /* check bd chain availability */ 1785 if(lm_bd_chain_avail_bds(&rx_con->bd_chain) < frag_list->cnt) { 1786 DbgBreakIf(s_list_is_empty(&rx_con->active_tb_list)); 1787 /* Check if the last placed BD was part of a split buffer (no end flag) if so, mark is at special split-end 1788 * and give a doorbell as if it was with END. Also, block UM from giving us more buffers until we've completed 1789 * this one (See L4 VBD Spec for more details on "Large Application Buffers" */ 1790 if (!(GET_FLAGS(rx_con->u.rx.last_rx_bd->flags , TOE_RX_BD_END))) { 1791 SET_FLAGS(rx_con->u.rx.last_rx_bd->flags, (TOE_RX_BD_END | TOE_RX_BD_SPLIT)); 1792 /* Mark the last buffer in active-tb-list as 'special' so that we know when we complete it that we can 1793 * unblock UM... */ 1794 tcp_buf = (lm_tcp_buffer_t *)s_list_peek_tail(&rx_con->active_tb_list); 1795 SET_FLAGS(tcp_buf->flags, TCP_BUF_FLAG_L4_SPLIT); 1796 SET_FLAGS(rx_con->flags, TCP_POST_DELAYED); 1797 lm_tcp_rx_write_db(pdev, tcp); 1798 } 1799 DbgMessage(pdev, INFORMl4rx, "post rx buf failed, rx chain is full (cid=%d, avail bds=%d, buf nfrags=%d)\n", 1800 tcp->cid, lm_bd_chain_avail_bds(&rx_con->bd_chain), frag_list->cnt); 1801 return LM_STATUS_RESOURCE; 1802 } 1803 1804 tcp_buf->size = tcp_buf->more_to_comp = (u32_t)frag_list->size; 1805 tcp_buf->bd_used = 0; /* will be modified if buffer will be posted */ 1806 DbgBreakIf(!(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START ? 1807 rx_con->app_buf_bytes_acc_post == 0 : 1808 rx_con->app_buf_bytes_acc_post > 0)); 1809 rx_con->app_buf_bytes_acc_post += tcp_buf->size; 1810 1811 /* special care in case of last tcp buffer of an application buffer */ 1812 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) { 1813 tcp_buf->app_buf_xferred = 0; /* just for safety */ 1814 tcp_buf->app_buf_size = rx_con->app_buf_bytes_acc_post; 1815 rx_con->app_buf_bytes_acc_post = 0; 1816 } 1817 split_buffer = !(GET_FLAGS(tcp_buf->flags, TCP_BUF_FLAG_L4_POST_START) && GET_FLAGS(tcp_buf->flags, TCP_BUF_FLAG_L4_POST_END)); 1818 } else { 1819 /* zero-byte request */ 1820 rx_con->u.rx.rx_zero_byte_recv_reqs++; 1821 } 1822 1823 /* we could be in the middle of completing a split-buffer... this is in case the previous split buffer was posted partially and we got a 1824 * cmp with push... need to complete it here. */ 1825 if (GET_FLAGS(rx_con->flags, TCP_POST_COMPLETE_SPLIT)) { 1826 DbgBreakIf(!split_buffer); /* we can only be in this state if we're completing split buffers... */ 1827 rx_con->bytes_push_skip_cnt += tcp_buf->more_to_comp; /* how many bytes did we skip? */ 1828 tcp_buf->more_to_comp = 0; 1829 rx_con->partially_completed_buf_cnt++; 1830 /* complete buffer */ 1831 s_list_push_tail(&(tcp->rx_con->active_tb_list), &(tcp_buf->link)); 1832 rx_con->rq_nbytes += tcp_buf->size; 1833 rx_con->buffer_skip_post_cnt++; 1834 lm_tcp_complete_tcp_buf(pdev, tcp, rx_con,tcp_buf,0); 1835 return LM_STATUS_SUCCESS; 1836 } 1837 1838 gen_info = &rx_con->u.rx.gen_info; 1839 1840 if ( gen_info->peninsula_nbytes ) { 1841 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_post_buf WITH GENERIC, cid=%d, tcp_buf=%p, buf_size=%d, buf_flags=%d, peninsula_nbytes=%d\n", 1842 tcp->cid, tcp_buf, frag_list ? frag_list->size : 0, tcp_buf ? tcp_buf->flags : 0, rx_con->u.rx.gen_info.peninsula_nbytes); 1843 if (tcp_buf) { 1844 d_list_init(&return_list, NULL, NULL, 0); 1845 copied_bytes = lm_tcp_rx_peninsula_to_rq_copy(pdev,tcp,tcp_buf,&return_list, 0xffffffff, FALSE); 1846 gen_info->bytes_copied_cnt_in_post += copied_bytes; 1847 if (!d_list_is_empty(&return_list)) { 1848 lm_tcp_return_list_of_gen_bufs(pdev,tcp,&return_list, MM_TCP_RGB_COMPENSATE_GRQS, NON_EXISTENT_SB_IDX); 1849 } 1850 if ((copied_bytes == tcp_buf->size) && !split_buffer && s_list_is_empty(&rx_con->active_tb_list)) { 1851 /* consumed_cnt: our way of telling fw we bypassed it */ 1852 lm_tcp_incr_consumed_gen(pdev, tcp, tcp_buf->size); 1853 /* simulate a _lm_tcp_rx_post_buf for lm_tcp_complete_bufs */ 1854 s_list_push_tail(&(tcp->rx_con->active_tb_list), &(tcp_buf->link)); 1855 rx_con->rq_nbytes += tcp_buf->size; 1856 rx_con->buffer_skip_post_cnt++; 1857 rx_con->bytes_skip_post_cnt += copied_bytes; 1858 /* If we copied some bytes to the RQ, we can now compensate FW-Window with these copied bytes. */ 1859 add_sws_bytes += copied_bytes; 1860 /* this function completes nbytes on the tcp buf and may complete the buffer if more_to_comp = 0*/ 1861 lm_tcp_complete_tcp_buf(pdev, tcp, rx_con,tcp_buf,copied_bytes); 1862 } else { 1863 /* will be posted and therefore get a SKP at some stage. */ 1864 if (!GET_FLAGS(rx_con->flags, TCP_POST_NO_SKP)) { 1865 rx_con->u.rx.skp_bytes_copied += copied_bytes; 1866 } 1867 lm_stat = _lm_tcp_rx_post_buf(pdev, tcp, tcp_buf, frag_list); 1868 DbgBreakIf(lm_stat != LM_STATUS_SUCCESS); 1869 if (copied_bytes && GET_FLAGS(rx_con->flags, TCP_POST_NO_SKP)) { 1870 lm_tcp_rx_write_db(pdev, tcp); /* for the case of split buffer in which bytes/bds are accumulated in bd_more* fields. bd_more* fields must be cleaned at this phase */ 1871 rx_con->bytes_comp_cnt += copied_bytes; 1872 /* If we copied some bytes to the RQ, we can now compensate FW-Window with these copied bytes. */ 1873 add_sws_bytes += copied_bytes; 1874 /* this function completes nbytes on the tcp buf and may complete the buffer if more_to_comp = 0*/ 1875 lm_tcp_complete_tcp_buf(pdev, tcp, rx_con,tcp_buf,copied_bytes); 1876 } 1877 } 1878 } 1879 /* if we have something to indicate after copying and it's ok to indicate... - indicate it */ 1880 if (gen_info->peninsula_nbytes && _lm_tcp_ok_to_indicate(rx_con)) { 1881 DbgBreakIf(frag_list && (frag_list->size != copied_bytes)); /* can't have bytes left with free space in tcp buf */ 1882 mm_tcp_rx_indicate_gen(pdev, tcp); 1883 add_sws_bytes += gen_info->add_sws_bytes; /* any bytes we need to update will be aggregated here during indicate */ 1884 gen_info->add_sws_bytes = 0; 1885 1886 } 1887 } else if (tcp_buf) { 1888 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_post_buf NO COPY, cid=%d, tcp_buf=%p, buf_size=%d, buf_flags=%d, peninsula_nbytes=%d\n", 1889 tcp->cid, tcp_buf, frag_list->size, tcp_buf->flags, rx_con->u.rx.gen_info.peninsula_nbytes); 1890 lm_stat = _lm_tcp_rx_post_buf(pdev, tcp, tcp_buf, frag_list); 1891 DbgBreakIf(lm_stat != LM_STATUS_SUCCESS); 1892 } 1893 1894 if (add_sws_bytes) { 1895 lm_tcp_rx_post_sws(pdev, tcp, rx_con, add_sws_bytes, TCP_RX_POST_SWS_INC); 1896 } 1897 1898 1899 return lm_stat; 1900 } 1901 1902 1903 /* Assumptions: 1904 * - caller initiated appropriately the following fields: 1905 * - tcp_buf->flags 1906 * - tcp_buf->size, tcp_buf->more_to_comp 1907 * - tcp_buf->app_buf_size, tcp_buf->app_buf_xferred 1908 * - caller verified that there is enough availabe BDs in the BD chain for the given buffer */ 1909 static lm_status_t _lm_tcp_rx_post_buf( 1910 struct _lm_device_t *pdev, 1911 lm_tcp_state_t *tcp, 1912 lm_tcp_buffer_t *tcp_buf, 1913 lm_frag_list_t *frag_list 1914 ) 1915 { 1916 lm_tcp_con_t *rx_con = tcp->rx_con; 1917 lm_bd_chain_t * rx_chain; 1918 u16_t old_prod, new_prod; 1919 struct toe_rx_bd * rx_bd; 1920 lm_frag_t * frag = frag_list->frag_arr; 1921 u32_t dbg_buf_size = 0; 1922 u32_t bd_bytes_prod; /* Each bd is initialized with a cyclic counter of bytes prod until that bd. */ 1923 u16_t flags = 0; 1924 u32_t i; 1925 1926 /* Number of fragments of entire application buffer can't be bigger 1927 * than size of the BD chain (entire application buffer since we can't 1928 * post partial application buffer to the FW , db_more_bds however includes the "next" bd, so we need 1929 * to take that into consideration as well */ 1930 DbgBreakIfAll( (rx_con->db_more_bds + frag_list->cnt) > (u32_t)(rx_con->bd_chain.capacity + rx_con->bd_chain.page_cnt)); 1931 1932 rx_chain = &rx_con->bd_chain; 1933 DbgBreakIf(lm_bd_chain_avail_bds(rx_chain) < frag_list->cnt); 1934 1935 old_prod = lm_bd_chain_prod_idx(rx_chain); 1936 1937 /* First BD should have the START flag */ 1938 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START) { 1939 flags = TOE_RX_BD_START; 1940 } 1941 1942 /* Set NO_PUSH flag if needed */ 1943 if ( tcp_buf->flags & TCP_BUF_FLAG_L4_RX_NO_PUSH ) { 1944 flags |= TOE_RX_BD_NO_PUSH; 1945 } 1946 if (tcp_buf->flags & TCP_BUF_FLAG_L4_PARTIAL_FILLED) { 1947 if (!rx_con->partially_filled_buf_sent && !rx_con->rq_completion_calls) { 1948 SET_FLAGS(rx_con->db_data.rx->flags, TOE_RX_DB_DATA_PARTIAL_FILLED_BUF); 1949 } else { 1950 RESET_FLAGS(tcp_buf->flags, TCP_BUF_FLAG_L4_PARTIAL_FILLED); 1951 } 1952 rx_con->partially_filled_buf_sent++; 1953 } 1954 /* Attach the first frag to the BD-chain */ 1955 bd_bytes_prod = rx_con->db_data.rx->bytes_prod + rx_con->db_more_bytes; 1956 rx_bd = _lm_tcp_rx_set_bd(frag, flags, rx_chain, bd_bytes_prod); 1957 bd_bytes_prod += frag->size; 1958 dbg_buf_size += frag->size; 1959 flags &= ~TOE_RX_BD_START; 1960 frag++; 1961 1962 /* "attach" the frags to the bd chain */ 1963 for(i = 1; i < frag_list->cnt; i++, frag++) { 1964 rx_bd = _lm_tcp_rx_set_bd(frag, flags, rx_chain, bd_bytes_prod); 1965 dbg_buf_size += frag->size; 1966 bd_bytes_prod += frag->size; 1967 } 1968 tcp->rx_con->u.rx.last_rx_bd = rx_bd; 1969 1970 /* The last BD must have the END flag */ 1971 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) { 1972 rx_bd->flags |= TOE_RX_BD_END; 1973 DbgMessage(NULL, VERBOSEl4rx, "Setting Rx last BD flags=0x%x\n", rx_bd->flags); 1974 } 1975 1976 DbgBreakIf(frag_list->cnt > TCP_MAX_SGL_SIZE); 1977 tcp_buf->bd_used = frag_list->cnt & TCP_MAX_SGL_SIZE; 1978 DbgBreakIf(tcp_buf->size != dbg_buf_size); 1979 1980 /* Perpare data for a DoorBell */ 1981 rx_con->db_more_bytes += tcp_buf->size; 1982 new_prod = lm_bd_chain_prod_idx(rx_chain); 1983 DbgBreakIf(S16_SUB(new_prod, old_prod) < tcp_buf->bd_used); 1984 rx_con->db_more_bds += S16_SUB(new_prod, old_prod); 1985 rx_con->db_more_bufs++; 1986 1987 /* Enqueue the buffer to the active_tb_list */ 1988 s_list_push_tail(&(rx_con->active_tb_list), &(tcp_buf->link)); 1989 rx_con->rq_nbytes += tcp_buf->size; 1990 1991 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) { 1992 lm_tcp_rx_write_db(pdev, tcp); 1993 } 1994 1995 1996 return LM_STATUS_SUCCESS; 1997 } 1998 1999 static lm_status_t _lm_tcp_rx_get_buffered_data( 2000 struct _lm_device_t * pdev, 2001 lm_tcp_state_t * tcp, 2002 lm_frag_list_t ** frag_list, /* if *frag_list is NULL, the rx con pre-allocaed will be used */ 2003 lm_tcp_gen_buf_t ** gen_buf 2004 ) 2005 { 2006 lm_tcp_con_t * rx_con = tcp->rx_con; 2007 lm_tcp_con_rx_gen_info_t * gen_info; 2008 lm_tcp_gen_buf_t * head_of_indication; 2009 d_list_t indicate_list; 2010 d_list_entry_t * entry; 2011 lm_tcp_gen_buf_t * curr_gen_buf; 2012 u32_t gen_offset, i; 2013 u32_t num_bufs_to_indicate; 2014 u32_t ind_nbufs=0, ind_nbytes=0; 2015 u8_t dont_send_to_system_more_then_rwin; 2016 DbgMessage(pdev, VERBOSEl4rx, "###_lm_tcp_rx_get_buffered_data cid=%d\n", tcp->cid); 2017 2018 gen_info = &rx_con->u.rx.gen_info; 2019 2020 2021 if ((u16_t)tcp->tcp_cached.rcv_indication_size != 0) { 2022 DbgBreakMsg("MichalS rcv_indication_size != 0 not implemented\n"); 2023 /* MichalS TBA: RcvIndicationSize > 0 will change following block quite a lot */ 2024 } 2025 2026 num_bufs_to_indicate = d_list_entry_cnt(&gen_info->peninsula_list); 2027 2028 /* The buffers in peninsula_list are ALWAYS released, unreleased buffers are in the dpc_peninsula_list. */ 2029 DbgBreakIf(((lm_tcp_gen_buf_t *)d_list_peek_tail(&gen_info->peninsula_list))->placed_bytes == 0); 2030 2031 if (*frag_list == NULL) { 2032 *frag_list = gen_info->frag_list; 2033 (*frag_list)->cnt = gen_info->max_frag_count; 2034 } 2035 2036 if (num_bufs_to_indicate > (*frag_list)->cnt) { 2037 DbgMessage(pdev, WARNl4rx, "_lm_tcp_rx_get_buffered_data: number of buffers to indicate[%d] is larger than frag_cnt[%d] cid=%d\n", 2038 num_bufs_to_indicate, (*frag_list)->cnt, tcp->cid); 2039 num_bufs_to_indicate = gen_info->max_frag_count; 2040 gen_info->num_non_full_indications++; 2041 } 2042 d_list_init(&indicate_list, NULL, NULL, 0); 2043 dont_send_to_system_more_then_rwin = (u8_t)gen_info->dont_send_to_system_more_then_rwin; 2044 while (num_bufs_to_indicate--) { 2045 entry = d_list_pop_head(&gen_info->peninsula_list); 2046 DbgBreakIf(entry == NULL); 2047 if (dont_send_to_system_more_then_rwin) { 2048 if ((ind_nbytes + ((lm_tcp_gen_buf_t *)entry)->placed_bytes) 2049 > tcp->tcp_cached.initial_rcv_wnd) { 2050 if (ind_nbytes) { 2051 d_list_push_head(&gen_info->peninsula_list, entry); 2052 break; 2053 } else { 2054 dont_send_to_system_more_then_rwin = FALSE; 2055 } 2056 } 2057 } 2058 d_list_push_tail(&indicate_list, entry); 2059 ind_nbufs ++; 2060 ind_nbytes += ((lm_tcp_gen_buf_t *)entry)->placed_bytes; 2061 } 2062 2063 ind_nbytes -= gen_info->first_buf_offset; 2064 2065 head_of_indication = (lm_tcp_gen_buf_t *)d_list_peek_head(&indicate_list); 2066 2067 if CHK_NULL(head_of_indication) 2068 { 2069 DbgBreakIfAll( !head_of_indication ) ; 2070 return LM_STATUS_FAILURE ; 2071 } 2072 2073 head_of_indication->tcp = tcp; 2074 head_of_indication->ind_nbufs = ind_nbufs; 2075 head_of_indication->ind_bytes = ind_nbytes; 2076 DbgBreakIf(gen_info->peninsula_nbytes < ind_nbytes); 2077 gen_info->peninsula_nbytes -= ind_nbytes; 2078 2079 /* initialize frag list */ 2080 (*frag_list)->cnt = ind_nbufs; 2081 (*frag_list)->size = ind_nbytes; 2082 curr_gen_buf = head_of_indication; 2083 2084 gen_offset = gen_info->first_buf_offset; 2085 for (i = 0; i < (*frag_list)->cnt; i++ ) { 2086 (*frag_list)->frag_arr[i].addr.as_ptr = curr_gen_buf->buf_virt + gen_offset; 2087 (*frag_list)->frag_arr[i].size = curr_gen_buf->placed_bytes - gen_offset; 2088 curr_gen_buf = NEXT_GEN_BUF(curr_gen_buf); 2089 gen_offset = 0; /* only first buffer can have an offset */ 2090 /* we don't touch gen_info->first_buf_offset - this is handled in lm_tcp_rx_buffered_data_indicated */ 2091 } 2092 *gen_buf = head_of_indication; 2093 DbgMessage(pdev, VERBOSEl4rx, "###_lm_tcp_rx_get_buffered_data ind_bytes = %d\n", (*frag_list)->size); 2094 2095 mm_atomic_inc(&pdev->toe_info.stats.total_indicated); 2096 return LM_STATUS_SUCCESS; 2097 } 2098 2099 lm_status_t lm_tcp_rx_get_buffered_data_from_terminate ( 2100 struct _lm_device_t * pdev, 2101 lm_tcp_state_t * tcp, 2102 lm_frag_list_t ** frag_list, 2103 lm_tcp_gen_buf_t ** gen_buf 2104 ) 2105 { 2106 lm_tcp_con_t * rx_con = tcp->rx_con; 2107 lm_tcp_con_rx_gen_info_t * gen_info; 2108 u16_t buff_cnt; 2109 lm_tcp_gen_buf_t * unwanted_gen_buf = NULL; 2110 lm_tcp_gen_buf_t * temp_gen_buf = NULL; 2111 lm_status_t lm_status = LM_STATUS_SUCCESS; 2112 2113 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_get_buffered_data_from_terminate cid=%d\n", tcp->cid); 2114 2115 gen_info = &rx_con->u.rx.gen_info; 2116 2117 /* make sure ALL the peninsula is released */ 2118 DbgBreakIf(!d_list_is_empty(&gen_info->peninsula_list) && 2119 (((lm_tcp_gen_buf_t *)d_list_peek_tail(&gen_info->peninsula_list))->placed_bytes == 0)); 2120 2121 *frag_list = NULL; 2122 if (gen_info->peninsula_nbytes == 0) { 2123 return LM_STATUS_SUCCESS; 2124 } 2125 2126 /* DbgBreakIf(gen_info->peninsula_nbytes > tcp->tcp_cached.initial_rcv_wnd);*/ 2127 gen_info->dont_send_to_system_more_then_rwin = FALSE; 2128 if ((buff_cnt = (u16_t)d_list_entry_cnt(&gen_info->peninsula_list)) > gen_info->max_frag_count) { 2129 lm_bd_chain_t *bd_chain = &tcp->rx_con->bd_chain; 2130 u16_t possible_frag_count, decreased_count; 2131 possible_frag_count = (/*bd_chain->page_cnt**/ 2132 LM_PAGE_SIZE - sizeof(lm_frag_list_t)) / sizeof(lm_frag_t); 2133 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2134 "###lm_tcp_rx_get_buffered_data_from_terminate cid=%d: peninsula_list cnt (%d) > max frag_count (%d)\n", 2135 tcp->cid, buff_cnt, gen_info->max_frag_count); 2136 2137 if (possible_frag_count > gen_info->max_frag_count) { 2138 /* This solution is ugly: 2139 since there will not be any further buffered data indications to the client, we must be able to 2140 indicate all the buffered data now. But the preallocated frag list in the rx con is too short! 2141 So instead of the pre-allocated frag list we need to use a larger memory. Our options: 2142 1. allocate memory here and release it later. 2143 2. use other pre-allocated memory that is not in use anymore (e.g. the bd chain) [chosen solution] 2144 In any case both solutions may fail: memory allocation can fail and the other pre-allocated memory 2145 might also be too short. the fallback from this is: 2146 - don't indicate anything and release the peninsula (NOT IMPLEMENTED) 2147 DbgBreakIfAll((u16_t)(sizeof(lm_frag_list_t) + sizeof(lm_frag_t)*buff_cnt) > bd_chain->page_cnt*LM_PAGE_SIZE); */ 2148 if (possible_frag_count < buff_cnt) { 2149 decreased_count = possible_frag_count; 2150 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2151 "###lm_tcp_rx_get_buffered_data_from_terminate cid=%d: peninsula_list cnt (%d) > aux.frag_cnt (%d)\n", 2152 tcp->cid, buff_cnt, possible_frag_count); 2153 } else { 2154 decreased_count = 0; 2155 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2156 "###lm_tcp_rx_get_buffered_data_from_terminate cid=%d: aux.frag_cnt (%d) is enough for %d buffs\n", 2157 tcp->cid, possible_frag_count, buff_cnt); 2158 } 2159 *frag_list = (lm_frag_list_t*)bd_chain->bd_chain_virt; 2160 (*frag_list)->cnt = possible_frag_count; 2161 (*frag_list)->size = 0; 2162 } else { 2163 decreased_count = (u16_t)gen_info->max_frag_count; 2164 } 2165 if (decreased_count) { 2166 u16_t returned_buff_cnt = lm_squeeze_rx_buffer_list(pdev, tcp, decreased_count, &unwanted_gen_buf); 2167 if (decreased_count < returned_buff_cnt) { 2168 lm_frag_list_t* new_frag_list; 2169 u32_t mem_size_for_new_frag_list = returned_buff_cnt * sizeof(lm_frag_t) + sizeof(lm_frag_list_t); 2170 // new_frag_list = (lm_frag_list_t*)mm_alloc_mem(pdev, mem_size_for_new_frag_list, LM_RESOURCE_NDIS); 2171 new_frag_list = (lm_frag_list_t*)mm_rt_alloc_mem(pdev, mem_size_for_new_frag_list, LM_RESOURCE_NDIS); 2172 2173 if (new_frag_list != NULL) { 2174 tcp->type_of_aux_memory = TCP_CON_AUX_RT_MEM; 2175 tcp->aux_memory = new_frag_list; 2176 tcp->aux_mem_size = mem_size_for_new_frag_list; 2177 *frag_list = new_frag_list; 2178 (*frag_list)->cnt = returned_buff_cnt; 2179 (*frag_list)->size = 0; 2180 tcp->aux_mem_flag = TCP_CON_AUX_RT_MEM_SUCCSESS_ALLOCATION; 2181 } else { 2182 /* No way. Let's send up only part of data. Data distortion is unavoidable. 2183 TODO: prevent data distortion by termination the connection itself at least */ 2184 lm_status = LM_STATUS_RESOURCE; 2185 tcp->aux_mem_flag = TCP_CON_AUX_RT_MEM_FAILED_ALLOCATION; 2186 /* Get rid of whatever remains in the peninsula...add it to unwanted... */ 2187 if (unwanted_gen_buf) 2188 { 2189 temp_gen_buf = (lm_tcp_gen_buf_t*)d_list_peek_tail(&gen_info->peninsula_list); 2190 if (temp_gen_buf) 2191 { 2192 temp_gen_buf->link.next = &(unwanted_gen_buf->link); 2193 unwanted_gen_buf->link.prev = &(temp_gen_buf->link); 2194 unwanted_gen_buf = (lm_tcp_gen_buf_t*)d_list_peek_head(&gen_info->peninsula_list); 2195 } 2196 } 2197 else 2198 { 2199 unwanted_gen_buf = (lm_tcp_gen_buf_t*)d_list_peek_head(&gen_info->peninsula_list); 2200 } 2201 d_list_clear(&gen_info->peninsula_list); 2202 2203 } 2204 } 2205 } 2206 } 2207 if (lm_status == LM_STATUS_SUCCESS) 2208 { 2209 _lm_tcp_rx_get_buffered_data(pdev, tcp, frag_list, gen_buf); 2210 2211 /* for cleaness: lm_tcp_rx_buffered_data_indicated will not be called 2212 * indication is 'succesfull' */ 2213 gen_info->num_bytes_indicated += (u32_t)(*frag_list)->size; 2214 gen_info->first_buf_offset = 0; 2215 gen_info->num_buffers_indicated += (*gen_buf)->ind_nbufs; 2216 } 2217 2218 gen_info->peninsula_blocked = TRUE; 2219 2220 if (unwanted_gen_buf) { 2221 lm_tcp_return_gen_bufs(pdev, tcp, unwanted_gen_buf,MM_TCP_RGB_COMPENSATE_GRQS, NON_EXISTENT_SB_IDX); 2222 } 2223 2224 if (*gen_buf) { 2225 /* with data taken from terminate, we can always act as in 'short-loop' since the bytes for 2226 * this connection won't increase the window anyway... */ 2227 (*gen_buf)->flags &= ~GEN_FLAG_SWS_UPDATE; 2228 } 2229 2230 return lm_status; 2231 } 2232 2233 lm_status_t lm_tcp_rx_get_buffered_data( 2234 struct _lm_device_t * pdev, 2235 lm_tcp_state_t * tcp, 2236 lm_frag_list_t ** frag_list, 2237 lm_tcp_gen_buf_t ** gen_buf 2238 ) 2239 { 2240 lm_tcp_con_t * rx_con = tcp->rx_con; 2241 lm_tcp_con_rx_gen_info_t * gen_info; 2242 lm_status_t lm_status; 2243 2244 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_get_buffered_data cid=%d\n", tcp->cid); 2245 gen_info = &rx_con->u.rx.gen_info; 2246 2247 DbgBreakIf(gen_info->peninsula_blocked == TRUE); /* terminate was already called */ 2248 2249 if (gen_info->peninsula_nbytes == 0 || (rx_con->flags & TCP_RX_IND_BLOCKED)) { 2250 return LM_STATUS_FAILURE; 2251 } 2252 2253 *frag_list = NULL; 2254 lm_status = _lm_tcp_rx_get_buffered_data(pdev, tcp, frag_list, gen_buf); 2255 if (*gen_buf) { 2256 if (gen_info->update_window_mode == LM_TOE_UPDATE_MODE_LONG_LOOP) { 2257 gen_info->pending_indicated_bytes += (*gen_buf)->ind_bytes; 2258 /* We need to increase the number of pending return indications here, since once we return 2259 * we are basically pending for the return of this specific indication. There are two cases 2260 * that require decreasing the pending return indications. The first is if the indication failed 2261 * the second is if it succeeded AND the buffers returned... */ 2262 gen_info->pending_return_indications++; 2263 (*gen_buf)->flags |= GEN_FLAG_SWS_UPDATE; 2264 } else { 2265 (*gen_buf)->flags &= ~GEN_FLAG_SWS_UPDATE; 2266 } 2267 } 2268 2269 return LM_STATUS_SUCCESS; 2270 } 2271 2272 void lm_tcp_rx_buffered_data_indicated( 2273 struct _lm_device_t * pdev, 2274 lm_tcp_state_t * tcp, 2275 u32_t accepted_bytes, 2276 lm_tcp_gen_buf_t * gen_buf /* head of indications generic buffer NULL if indication succeeded */ 2277 ) 2278 { 2279 lm_tcp_con_rx_gen_info_t * gen_info = &tcp->rx_con->u.rx.gen_info; 2280 2281 DbgMessage(pdev, VERBOSEl4rx , "###lm_tcp_rx_buffered_data_indicated accepted_bytes = %d cid=%d\n", accepted_bytes, tcp->cid); 2282 2283 DbgBreakIf(gen_info->peninsula_blocked == TRUE); /* terminate was already called */ 2284 2285 lm_tcp_incr_consumed_gen(pdev, tcp, accepted_bytes); 2286 gen_info->num_bytes_indicated += accepted_bytes; 2287 2288 if (gen_buf == NULL) { /* succesfull indication */ 2289 gen_info->first_buf_offset = 0; 2290 if (gen_info->update_window_mode == LM_TOE_UPDATE_MODE_SHORT_LOOP) { 2291 gen_info->add_sws_bytes += accepted_bytes; 2292 } 2293 gen_info->num_success_indicates++; 2294 gen_info->bytes_indicated_accepted += accepted_bytes; 2295 tcp->rx_con->u.rx.zero_byte_posted_during_ind = FALSE; 2296 } else { /* complete rejection / partial success, gen_buf remains in our control */ 2297 /* indication failed */ 2298 lm_tcp_gen_buf_t * curr_gen_buf, * ret_buf; 2299 d_list_t return_to_pool_list; 2300 d_list_t return_to_peninsula_list; 2301 u32_t nbytes; 2302 DbgBreakIf(accepted_bytes > gen_buf->ind_bytes); 2303 gen_info->peninsula_nbytes += gen_buf->ind_bytes - accepted_bytes; 2304 2305 gen_info->num_failed_indicates++; 2306 gen_info->bytes_indicated_accepted+= accepted_bytes; 2307 gen_info->bytes_indicated_rejected+= gen_buf->ind_bytes - accepted_bytes; 2308 2309 DbgMessage(pdev, INFORMl4rx, "GENERIC: %s Indication for cid=%d accepted_bytes=%d\n", 2310 (accepted_bytes == 0)? "Rejected" : "Partial", tcp->cid, accepted_bytes); 2311 2312 d_list_init(&return_to_pool_list, NULL, NULL, 0); 2313 d_list_init(&return_to_peninsula_list, NULL, NULL, 0); 2314 2315 DbgBreakIf(gen_buf->tcp->rx_con->flags & TCP_INDICATE_REJECTED); 2316 if (tcp->rx_con->u.rx.zero_byte_posted_during_ind) { 2317 tcp->rx_con->u.rx.zero_byte_posted_during_ind = FALSE; 2318 } else { 2319 gen_buf->tcp->rx_con->flags |= TCP_INDICATE_REJECTED; 2320 } 2321 2322 curr_gen_buf = gen_buf; 2323 2324 /* indicated bytes are in fact 'freed up' space: so we can make the sws_bytes larger, 2325 * this is always true here luxury-mode or not */ 2326 gen_info->add_sws_bytes += accepted_bytes; 2327 2328 /* buffer was returned to us so it is no longer pending return...if we increased the 'pending' we have 2329 * to decrease */ 2330 if (gen_buf->flags & GEN_FLAG_SWS_UPDATE) { 2331 gen_info->pending_return_indications--; 2332 gen_info->pending_indicated_bytes-=gen_buf->ind_bytes; 2333 } 2334 mm_atomic_inc(&pdev->toe_info.stats.total_indicated_returned); /* stats */ 2335 2336 /* return buffers that were fully indicated to the generic pool, ones that we're not, to the peninsula */ 2337 while (accepted_bytes) { 2338 nbytes = ((lm_tcp_gen_buf_t *)curr_gen_buf)->placed_bytes - gen_info->first_buf_offset; 2339 if (accepted_bytes >= nbytes) { 2340 /* the buffer was completely accepted */ 2341 accepted_bytes -= nbytes; 2342 ret_buf = curr_gen_buf; 2343 curr_gen_buf = NEXT_GEN_BUF(curr_gen_buf); 2344 d_list_push_tail(&return_to_pool_list, &ret_buf->link); 2345 gen_info->num_buffers_indicated++; 2346 gen_info->first_buf_offset = 0; 2347 } else { 2348 gen_info->first_buf_offset += (u16_t)accepted_bytes; 2349 accepted_bytes = 0; 2350 } 2351 } 2352 2353 /* is there anything to return to the peninsula ? (i.e. return_head moved) */ 2354 while (curr_gen_buf) { 2355 curr_gen_buf->ind_bytes = 0; 2356 curr_gen_buf->ind_nbufs = 0; 2357 ret_buf = curr_gen_buf; 2358 curr_gen_buf = NEXT_GEN_BUF(curr_gen_buf); 2359 gen_info->bufs_indicated_rejected++; 2360 d_list_push_tail(&return_to_peninsula_list, &ret_buf->link); 2361 } 2362 2363 if (!d_list_is_empty(&return_to_pool_list)) { 2364 lm_tcp_return_list_of_gen_bufs(pdev, tcp, &return_to_pool_list, MM_TCP_RGB_COMPENSATE_GRQS, NON_EXISTENT_SB_IDX); 2365 } 2366 2367 /* There must be at least something to return to the peninsula since this was partial indication */ 2368 DbgBreakIf(d_list_is_empty(&return_to_peninsula_list)); 2369 /* re-insert generic buffers to the peninsula. 2370 * we need to re-insert the buffers to the head of the peninsula */ 2371 d_list_add_head(&gen_info->peninsula_list, &return_to_peninsula_list); 2372 2373 } 2374 2375 } 2376 2377 /** Description 2378 * returns the buffers to the generic pool 2379 */ 2380 void lm_tcp_return_gen_bufs(struct _lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_gen_buf_t * gen_buf,u32_t flags, u8_t grq_idx) 2381 { 2382 lm_tcp_gen_buf_t * curr_gen_buf = gen_buf; 2383 2384 #if DBG 2385 gen_buf->ind_nbufs = 0; /* for debugging purposes will count how many buffers are in our list */ 2386 while (curr_gen_buf) { 2387 DbgBreakIf(SIG(curr_gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 2388 DbgBreakIf(END_SIG(curr_gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 2389 /* We increase the bytes for both pool-buffers, and buffered-data buffers because when the OS 2390 * gives posted buffers the window is smaller */ 2391 curr_gen_buf = NEXT_GEN_BUF(curr_gen_buf); 2392 gen_buf->ind_nbufs++; 2393 } 2394 #endif 2395 2396 mm_tcp_return_gen_bufs(pdev, gen_buf,flags,grq_idx); 2397 } 2398 2399 /** Description 2400 * returns the buffers to the generic pool 2401 */ 2402 void lm_tcp_return_list_of_gen_bufs(struct _lm_device_t * pdev, lm_tcp_state_t * tcp, d_list_t * gen_buf_list,u32_t flags, u8_t grq_idx) 2403 { 2404 lm_tcp_gen_buf_t * gen_buf = (lm_tcp_gen_buf_t *)d_list_peek_head(gen_buf_list); 2405 lm_tcp_gen_buf_t * curr_gen_buf = gen_buf; 2406 2407 #if DBG 2408 gen_buf->ind_nbufs = 0; /* for debugging purposes will count how many buffers are in our list */ 2409 while (curr_gen_buf) { 2410 DbgBreakIf(SIG(curr_gen_buf->buf_virt) != L4GEN_BUFFER_SIG); 2411 DbgBreakIf(END_SIG(curr_gen_buf->buf_virt, LM_TCP_GEN_BUF_SIZE(pdev)) != L4GEN_BUFFER_SIG_END); 2412 /* We increase the bytes for both pool-buffers, and buffered-data buffers because when the OS 2413 * gives posted buffers the window is smaller */ 2414 curr_gen_buf = NEXT_GEN_BUF(curr_gen_buf); 2415 gen_buf->ind_nbufs++; 2416 } 2417 #endif 2418 2419 mm_tcp_return_list_of_gen_bufs(pdev, gen_buf_list,flags,grq_idx); 2420 } 2421 2422 void lm_tcp_rx_indication_returned(struct _lm_device_t *pdev, lm_tcp_state_t * tcp, lm_tcp_gen_buf_t * gen_buf) 2423 { 2424 DbgMessage(pdev, VERBOSEl4rx, "###lm_tcp_rx_con_indication_returned cid=%d\n", tcp->cid); 2425 2426 DbgBreakIf(tcp != gen_buf->tcp); 2427 DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid))); 2428 2429 /* TBA fix in case of RcvIndicateSize > 0 */ 2430 DbgBreakIf(gen_buf->refcnt != 0); 2431 2432 tcp->rx_con->u.rx.gen_info.pending_return_indications--; 2433 tcp->rx_con->u.rx.gen_info.pending_indicated_bytes -= gen_buf->ind_bytes; 2434 2435 /* Update the sws bytes according to the ind number of bytes this function is only called if in fact 2436 * this is a buffer that is marked as an 'update buffer' otherwise this function isn't called. */ 2437 DbgBreakIfAll(!(gen_buf->flags & GEN_FLAG_SWS_UPDATE)); 2438 lm_tcp_rx_post_sws(pdev, tcp, tcp->rx_con, gen_buf->ind_bytes, TCP_RX_POST_SWS_INC); 2439 lm_tcp_return_gen_bufs(pdev, tcp, gen_buf, MM_TCP_RGB_COMPENSATE_GRQS, NON_EXISTENT_SB_IDX); 2440 } 2441 2442 u8_t lm_tcp_is_tcp_dead(struct _lm_device_t * pdev, lm_tcp_state_t * tcp, u8_t op) 2443 { 2444 UNREFERENCED_PARAMETER_(pdev); 2445 2446 if(op == TCP_IS_DEAD_OP_UPLD_COMP) { 2447 DbgBreakIf(tcp->hdr.status != STATE_STATUS_UPLOAD_PENDING); 2448 tcp->hdr.status = STATE_STATUS_UPLOAD_DONE; 2449 } 2450 if (GET_FLAGS(tcp->rx_con->flags, TCP_COMP_DEFERRED)) { 2451 /* we can't kill the connection here! it's still being handled by deferred function which will 2452 * access it... killing will be done from that context... */ 2453 return FALSE; 2454 } 2455 if (tcp->rx_con->u.rx.gen_info.pending_return_indications == 0) { 2456 /* If the function is called from offload completion flow, we might have completions on the RCQ 2457 that we haven't processed yet so haven't completed / indicated bufs, 2458 so there are bytes in the peninsula and this state is legal */ 2459 DbgBreakIf(!(tcp->rx_con->flags & TCP_RX_IND_BLOCKED) && 2460 (tcp->rx_con->u.rx.gen_info.peninsula_nbytes != 0) && 2461 (op != TCP_IS_DEAD_OP_OFLD_COMP_DFRD)); 2462 if (tcp->hdr.status == STATE_STATUS_UPLOAD_DONE) { 2463 return TRUE; 2464 } 2465 } 2466 return FALSE; 2467 } 2468 2469 lm_status_t lm_tcp_con_status(struct _lm_device_t * pdev, lm_tcp_con_t * rx_con) 2470 { 2471 UNREFERENCED_PARAMETER_(pdev); 2472 2473 if (rx_con->flags & TCP_RX_POST_BLOCKED) { 2474 return LM_STATUS_CONNECTION_CLOSED; 2475 } 2476 return LM_STATUS_SUCCESS; 2477 } 2478 2479 u32_t lm_tcp_calc_gen_buf_size(struct _lm_device_t * pdev) 2480 { 2481 u32_t gen_buf_size = 0; 2482 u32_t const chain_idx = LM_SW_LEADING_RSS_CID(pdev); 2483 2484 /* determine size of buffer: in steps of pages, larger than the minimum and 2485 * the mtu */ 2486 if(CHK_NULL(pdev) || 2487 ERR_IF((ARRSIZE(pdev->params.l2_cli_con_params) <= chain_idx) || 2488 (CHIP_IS_E1H(pdev) && (chain_idx >= ETH_MAX_RX_CLIENTS_E1H)) || /* TODO E2 add IS_E2*/ 2489 (CHIP_IS_E1(pdev) && (chain_idx >= ETH_MAX_RX_CLIENTS_E1)) )) 2490 { 2491 DbgBreakIf(1); 2492 return 0; 2493 } 2494 2495 if (pdev->params.l4_gen_buf_size < pdev->params.l2_cli_con_params[chain_idx].mtu) 2496 { 2497 gen_buf_size = pdev->params.l2_cli_con_params[chain_idx].mtu; 2498 2499 } 2500 else 2501 { 2502 gen_buf_size = pdev->params.l4_gen_buf_size; 2503 } 2504 /* bring to page-size boundary */ 2505 gen_buf_size = (gen_buf_size + (LM_PAGE_SIZE-1)) & ~(LM_PAGE_SIZE-1); 2506 2507 return gen_buf_size; 2508 } 2509 2510 u16_t lm_squeeze_rx_buffer_list( 2511 struct _lm_device_t * pdev, 2512 lm_tcp_state_t * tcp, 2513 u16_t adjust_number, 2514 lm_tcp_gen_buf_t ** unwanted_gen_buf 2515 ) 2516 { 2517 u32_t gen_buff_size = lm_tcp_calc_gen_buf_size(pdev); 2518 lm_tcp_con_t * rx_con = tcp->rx_con; 2519 lm_tcp_con_rx_gen_info_t * gen_info = &rx_con->u.rx.gen_info; 2520 d_list_t unwanted_list = {0}; 2521 lm_tcp_gen_buf_t * gen_buf_copy_to = NULL; 2522 lm_tcp_gen_buf_t * gen_buf_copy_from = NULL, *next_buffer = NULL; 2523 u16_t free_bytes_to_copy = 0, bytes_to_copy = 0, gen_buf_offset = 0; 2524 u8_t force_buffer_division = FALSE; 2525 u16_t buffers_number = (u16_t)d_list_entry_cnt(&gen_info->peninsula_list); 2526 2527 *unwanted_gen_buf = NULL; 2528 2529 if ((adjust_number * gen_buff_size) >= gen_info->peninsula_nbytes) { 2530 d_list_init(&unwanted_list, NULL, NULL, 0); 2531 gen_buf_copy_to = (lm_tcp_gen_buf_t*)d_list_peek_head(&gen_info->peninsula_list); 2532 next_buffer = NEXT_GEN_BUF(gen_buf_copy_to); 2533 free_bytes_to_copy = gen_buff_size - gen_buf_copy_to->placed_bytes; 2534 while (buffers_number > adjust_number) { 2535 gen_buf_copy_from = next_buffer; 2536 if (gen_buf_copy_from != NULL) { 2537 next_buffer = NEXT_GEN_BUF(gen_buf_copy_from); 2538 bytes_to_copy = gen_buf_copy_from->placed_bytes; 2539 if (bytes_to_copy <= free_bytes_to_copy) { 2540 mm_memcpy(gen_buf_copy_to->buf_virt + gen_buf_copy_to->placed_bytes, 2541 gen_buf_copy_from->buf_virt, bytes_to_copy); 2542 free_bytes_to_copy -= bytes_to_copy; 2543 gen_buf_copy_to->placed_bytes += bytes_to_copy; 2544 d_list_remove_entry(&gen_info->peninsula_list, &gen_buf_copy_from->link); 2545 d_list_push_tail(&unwanted_list, &gen_buf_copy_from->link); 2546 buffers_number--; 2547 continue; 2548 } else { 2549 if (force_buffer_division) { 2550 if (free_bytes_to_copy) { 2551 mm_memcpy(gen_buf_copy_to->buf_virt + gen_buf_copy_to->placed_bytes, 2552 gen_buf_copy_from->buf_virt, free_bytes_to_copy); 2553 gen_buf_copy_to->placed_bytes += free_bytes_to_copy; 2554 mm_memcpy(gen_buf_copy_from->buf_virt, 2555 gen_buf_copy_from->buf_virt + free_bytes_to_copy, bytes_to_copy - free_bytes_to_copy); 2556 gen_buf_copy_from->placed_bytes -= free_bytes_to_copy; 2557 } 2558 } 2559 gen_buf_copy_to = gen_buf_copy_from; 2560 next_buffer = NEXT_GEN_BUF(gen_buf_copy_from); 2561 free_bytes_to_copy = gen_buff_size - gen_buf_copy_to->placed_bytes; 2562 continue; 2563 } 2564 } else { 2565 if (!force_buffer_division) { 2566 force_buffer_division = TRUE; 2567 gen_buf_copy_to = (lm_tcp_gen_buf_t*)d_list_peek_head(&gen_info->peninsula_list); 2568 next_buffer = NEXT_GEN_BUF(gen_buf_copy_to); 2569 gen_buf_offset = gen_info->first_buf_offset; 2570 if (gen_buf_offset) { 2571 /* move to start of buffer*/ 2572 mm_memcpy(gen_buf_copy_to->buf_virt, 2573 gen_buf_copy_to->buf_virt + gen_buf_offset, gen_buf_copy_to->placed_bytes - gen_buf_offset); 2574 gen_buf_copy_to->placed_bytes -= gen_buf_offset; 2575 gen_buf_offset = gen_info->first_buf_offset = 0; 2576 } 2577 free_bytes_to_copy = gen_buff_size - gen_buf_copy_to->placed_bytes; 2578 continue; 2579 } else { 2580 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2581 "###lm_squeeze_rx_buffer_list cid=%d: peninsula_list cnt (%d) is still more frag_count (%d)\n", 2582 tcp->cid, buffers_number, adjust_number); 2583 break; 2584 } 2585 } 2586 } 2587 *unwanted_gen_buf = (lm_tcp_gen_buf_t*)d_list_peek_head(&unwanted_list); 2588 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2589 "###lm_squeeze_rx_buffer_list cid=%d(%d,%d,%d): peninsula_list cnt is decreased till %d\n", 2590 tcp->cid, tcp->tcp_cached.initial_rcv_wnd, tcp->tcp_cached.rcv_indication_size, gen_buff_size, buffers_number); 2591 } else { 2592 DbgMessage(pdev, WARNl4rx | WARNl4sp, 2593 "###lm_squeeze_rx_buffer_list cid=%d(%d,%d): could not replace %dB (%d bufs) into %d frags of %dB each\n", 2594 tcp->cid, tcp->tcp_cached.initial_rcv_wnd, tcp->tcp_cached.rcv_indication_size, 2595 gen_info->peninsula_nbytes, buffers_number, adjust_number, gen_buff_size); 2596 } 2597 return buffers_number; 2598 } 2599 2600 void lm_tcp_rx_clear_isles(struct _lm_device_t * pdev, lm_tcp_state_t * tcp_state, d_list_t * isles_list) 2601 { 2602 lm_tcp_con_rx_gen_info_t * gen_info; 2603 u8_t isle_cnt; 2604 2605 DbgBreakIf(!(tcp_state && tcp_state->rx_con)); 2606 gen_info = &tcp_state->rx_con->u.rx.gen_info; 2607 while ((isle_cnt = (u8_t)d_list_entry_cnt(&gen_info->isles_list))) { 2608 d_list_t aux_isles_list; 2609 d_list_init(&aux_isles_list, NULL, NULL, 0); 2610 _lm_tcp_isle_remove(pdev, tcp_state, NON_EXISTENT_SB_IDX, isle_cnt, &aux_isles_list); 2611 if (!d_list_is_empty(&aux_isles_list)) { 2612 d_list_add_head(isles_list, &aux_isles_list); 2613 } 2614 } 2615 return; 2616 } 2617 2618