1 2 #include "lm5710.h" 3 #include "bd_chain.h" 4 #include "command.h" 5 #include "context.h" 6 #include "lm_l4fp.h" 7 #include "lm_l4sp.h" 8 #include "mm_l4if.h" 9 10 11 /* TODO: remove this temporary solution for solaris / linux compilation conflict, linux needs the 12 * first option, solaris the latter */ 13 #if defined(__LINUX) 14 #define TOE_TX_INIT_ZERO {{0}} 15 #else 16 #define TOE_TX_INIT_ZERO {0} 17 #endif 18 19 #define TOE_TX_DOORBELL(pdev,cid) do{\ 20 struct doorbell db = TOE_TX_INIT_ZERO;\ 21 db.header.data |= (TOE_CONNECTION_TYPE << DOORBELL_HDR_T_CONN_TYPE_SHIFT);\ 22 DOORBELL((pdev), (cid), *((u32_t *)&db));\ 23 } while(0) 24 25 static __inline void _lm_tcp_tx_write_db( 26 lm_device_t * pdev, 27 lm_tcp_con_t * tx_con, 28 u32_t cid, 29 u32_t nbytes, 30 u16_t nbds, 31 u8_t fin) 32 { 33 volatile struct toe_tx_db_data *db_data = tx_con->db_data.tx; 34 35 db_data->bds_prod += nbds; /* nbds should be written before nbytes (FW assumption) */ 36 DbgBreakIf((db_data->bds_prod & 0xff) == 0); 37 db_data->bytes_prod_seq += nbytes; 38 39 if(fin) { 40 DbgBreakIf(db_data->flags & (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT)); 41 db_data->flags |= (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT); 42 } 43 44 if (!(tx_con->flags & TCP_TX_DB_BLOCKED)) { 45 DbgMessage(pdev, INFORMl4tx, 46 "ringing tx doorbell: cid=%d, (nbytes+=%d, nbds+=%d, fin=%d)\n", 47 cid, nbytes, nbds, fin); 48 TOE_TX_DOORBELL(pdev, cid); 49 } 50 } 51 52 static __inline void lm_tcp_tx_write_db( 53 lm_device_t *pdev, 54 lm_tcp_state_t *tcp, 55 u8_t post_end) 56 { 57 lm_tcp_con_t *tx_con = tcp->tx_con; 58 59 /* define a policy for ringing the doorbell */ 60 #define MAX_BYTES_PER_TX_DB 0xffff 61 #define MAX_BDS_PER_TX_DB 64 62 63 if (post_end || 64 tx_con->db_more_bytes >= MAX_BYTES_PER_TX_DB || 65 tx_con->db_more_bds >= MAX_BDS_PER_TX_DB) { 66 _lm_tcp_tx_write_db(pdev, tx_con, tcp->cid, tx_con->db_more_bytes, tx_con->db_more_bds, 0); 67 68 /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */ 69 DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt + tx_con->db_more_bytes, tx_con->bytes_comp_cnt) < 0); 70 tx_con->bytes_post_cnt += tx_con->db_more_bytes; 71 tx_con->buffer_post_cnt += tx_con->db_more_bufs; 72 tx_con->db_more_bytes = tx_con->db_more_bds = tx_con->db_more_bufs = 0; 73 tx_con->fp_db_cnt++; 74 } else { 75 DbgMessage(pdev, INFORMl4tx, 76 "skipped doorbell ringing for cid=%d\n", tcp->cid); 77 } 78 } 79 80 /** Description: 81 * Post a single tcp buffer to the Tx bd chain 82 * Assumptions: 83 * - caller initiated tcp_buf->flags field with BUFFER_START/BUFFER_END appropriately 84 * Returns: 85 * - SUCCESS - tcp buf was successfully attached to the bd chain 86 * - RESOURCE - not enough available BDs on bd chain for given tcp buf 87 * - CONNECTION_CLOSED - whenever connection's flag are marked as 'POST BLOCKED' */ 88 lm_status_t lm_tcp_tx_post_buf( 89 struct _lm_device_t *pdev, 90 lm_tcp_state_t *tcp, 91 lm_tcp_buffer_t *tcp_buf, 92 lm_frag_list_t *frag_list) 93 { 94 lm_tcp_con_t *tx_con; 95 lm_bd_chain_t *tx_chain; 96 struct toe_tx_bd *tx_bd = NULL ; 97 lm_frag_t *frag; 98 u32_t i, dbg_buf_size = 0; 99 u32_t dbg_bytes_prod_seq; 100 u16_t old_prod, new_prod; 101 102 DbgMessage(pdev, VERBOSEl4tx, "###lm_tcp_tx_post_buf\n"); 103 DbgBreakIf(!(pdev && tcp && tcp_buf && frag_list)); 104 DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid))); 105 DbgBreakIf(frag_list->cnt == 0); 106 tx_con = tcp->tx_con; 107 tx_chain = &tx_con->bd_chain; 108 frag = frag_list->frag_arr; 109 110 DbgBreakIf(tx_con->flags & TCP_FIN_REQ_POSTED); 111 112 /* check if tx con is already closed */ 113 if(tx_con->flags & TCP_TX_POST_BLOCKED) { 114 DbgMessage(pdev, WARNl4tx, "post tx buf failed, posting is blocked (cid=%d, con->flags=%x)\n", 115 tcp->cid, tx_con->flags); 116 return LM_STATUS_CONNECTION_CLOSED; 117 } 118 /* check bd chain availability (including additional bd that should 119 * be kept available for future fin request) */ 120 if(lm_bd_chain_avail_bds(tx_chain) < frag_list->cnt + 1) { 121 DbgMessage(pdev, INFORMl4tx, "post tx buf failed, tx chain is full (cid=%d, avail bds=%d, buf nfrags=%d)\n", 122 tcp->cid, lm_bd_chain_avail_bds(tx_chain), frag_list->cnt); 123 124 LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, tx_no_l4_bd); 125 126 if (tx_con->db_more_bds) { 127 /* if doorbell ringing was deferred (e.g. until an end of 128 * application buffer), it can no longer be deferred since 129 * the place in the bd chain is now required */ 130 lm_tcp_tx_write_db(pdev, tcp, 1); 131 } 132 return LM_STATUS_RESOURCE; 133 } 134 135 old_prod = lm_bd_chain_prod_idx(tx_chain); 136 137 dbg_bytes_prod_seq = tx_con->db_data.tx->bytes_prod_seq + tx_con->db_more_bytes; 138 /* "attach" the frags to the bd chain */ 139 for(i = 0; i < frag_list->cnt; i++, frag++) { 140 DbgBreakIf(frag->size > 0xffff || frag->size == 0); /* hw limit: each bd can point to a buffer with max size of 64KB */ 141 tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(tx_chain); 142 tx_bd->addr_hi = frag->addr.as_u32.high; 143 tx_bd->addr_lo = frag->addr.as_u32.low; 144 tx_bd->flags = 0; 145 tx_bd->size = (u16_t)frag->size; 146 dbg_bytes_prod_seq += frag->size; 147 tx_bd->nextBdStartSeq = dbg_bytes_prod_seq; 148 dbg_buf_size += frag->size; 149 150 /* Support for FW Nagle Algorithm: 151 * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss. 152 */ 153 if ((u32_t)frag_list->size >= tx_con->u.tx.mss) { 154 tx_bd->flags |= TOE_TX_BD_LARGE_IO; 155 } 156 157 DbgMessage(pdev, VERBOSEl4tx, "Setting Tx BD, addr_lo=0x%x, addr_hi=0x%x, size=%d\n", 158 tx_bd->addr_lo, tx_bd->addr_hi, tx_bd->size); 159 } 160 161 DbgBreakIf(frag_list->cnt > 0xffff); 162 tcp_buf->bd_used = frag_list->cnt & 0xffff; 163 tcp_buf->size = tcp_buf->more_to_comp = (u32_t)frag_list->size; 164 DbgBreakIf(tcp_buf->size != dbg_buf_size); 165 166 DbgBreakIf(!(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START ? 167 tx_con->app_buf_bytes_acc_post == 0 : 168 tx_con->app_buf_bytes_acc_post > 0)); 169 tx_con->app_buf_bytes_acc_post += tcp_buf->size; 170 tx_con->db_more_bytes += tcp_buf->size; 171 new_prod = lm_bd_chain_prod_idx(tx_chain); 172 DbgBreakIf(S16_SUB(new_prod, old_prod) < tcp_buf->bd_used); 173 tx_con->db_more_bds += S16_SUB(new_prod, old_prod); 174 tx_con->db_more_bufs++; 175 176 /* Support for FW Nagle Algorithm: 177 * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss. 178 */ 179 if (tcp_buf->size >= tx_con->u.tx.mss) { 180 tx_bd->flags |= TOE_TX_BD_LARGE_IO; 181 } 182 183 /* special care in case of last tcp buffer of an application buffer */ 184 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) { 185 tcp_buf->app_buf_xferred = 0; /* just for safety */ 186 tcp_buf->app_buf_size = tx_con->app_buf_bytes_acc_post; 187 tx_con->app_buf_bytes_acc_post = 0; 188 189 /* special care for the last bd: */ 190 tx_bd->flags |= TOE_TX_BD_NOTIFY; 191 tx_con->u.tx.bds_without_comp_flag = 0; 192 tx_bd->flags |= TOE_TX_BD_PUSH; 193 194 DbgMessage(pdev, VERBOSEl4tx, 195 "Setting Tx BD, last bd of app buf, flags=%d\n", tx_bd->flags); 196 } else { 197 /* make sure there aren't 'too many' bds without completion flag */ 198 tx_con->u.tx.bds_without_comp_flag += tcp_buf->bd_used; 199 if (tx_con->u.tx.bds_without_comp_flag > (tx_chain->capacity - MAX_FRAG_CNT_PER_TB)) { 200 tx_bd->flags |= TOE_TX_BD_NOTIFY; 201 tx_con->u.tx.bds_without_comp_flag = 0; 202 } 203 } 204 205 s_list_push_tail(&tx_con->active_tb_list, &tcp_buf->link); 206 tx_con->rq_nbytes += tcp_buf->size; 207 lm_tcp_tx_write_db(pdev, tcp, tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END); 208 209 /* network reachability (NOT IMPLEMENTED): 210 if(lm_neigh_is_cache_entry_staled(tcp->path->neigh)) 211 lm_neigh_indicate_staled_cache_entry(tcp->path->neigh); 212 */ 213 214 DbgMessage(pdev, VERBOSEl4tx, "posted tx buf for cid=%d, buf size=%d, bd used=%d, buf flags=%x, app_buf_size=%d\n", 215 tcp->cid, tcp_buf->size, tcp_buf->bd_used, tcp_buf->flags, tcp_buf->app_buf_size); 216 DbgMessage(pdev, VERBOSEl4tx, "after posting tx buf, tx_con->active_tb_list=%d\n", 217 s_list_entry_cnt(&tx_con->active_tb_list)); 218 219 return LM_STATUS_SUCCESS; 220 } /* lm_tcp_tx_post_buf */ 221 222 /** Description 223 * indicates graceful disconnect completion to client. 224 * Assumtpions: 225 * tx-lock is taken by caller 226 */ 227 static __inline void lm_tcp_tx_graceful_disconnect_complete(lm_device_t * pdev, lm_tcp_state_t * tcp) 228 { 229 u8_t ip_version; 230 DbgBreakIf(!s_list_is_empty(&tcp->tx_con->active_tb_list)); 231 DbgBreakIf(tcp->tx_con->flags & TCP_FIN_REQ_COMPLETED); 232 tcp->tx_con->flags |= TCP_FIN_REQ_COMPLETED; 233 DbgMessage(pdev, INFORMl4tx, "fin request completed (cid=%d)\n", tcp->cid); 234 tcp->tcp_state_calc.fin_completed_time = mm_get_current_time(pdev); 235 if (!(tcp->tx_con->u.tx.flags & TCP_CON_FIN_REQ_LM_INTERNAL)) { 236 ip_version = (tcp->path->path_const.ip_version == IP_VERSION_IPV4)? STATS_IP_4_IDX : STATS_IP_6_IDX; 237 LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, ipv[ip_version].out_fin); 238 mm_tcp_graceful_disconnect_done(pdev,tcp, LM_STATUS_SUCCESS); 239 } 240 } 241 242 void lm_tcp_tx_cmp_process( 243 struct _lm_device_t *pdev, 244 lm_tcp_state_t *tcp, 245 u32_t completed_bytes 246 ) 247 { 248 lm_tcp_con_t *tx_con = tcp->tx_con; 249 u32_t actual_completed; /* number of bytes actually completed (could be different than completed in case of fin) */ 250 MM_INIT_TCP_LOCK_HANDLE(); 251 252 DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_app_cmp_process, cid=%d, completed_bytes=%d\n", 253 tcp->cid, completed_bytes); 254 255 DbgBreakIf(tx_con->flags & TCP_TX_COMP_BLOCKED); 256 257 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) { 258 mm_acquire_tcp_lock(pdev, tx_con); 259 } 260 tx_con->bytes_comp_cnt += completed_bytes; 261 DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt, tx_con->bytes_comp_cnt) < 0); 262 263 DbgBreakIf(!completed_bytes); 264 265 actual_completed = lm_tcp_complete_nbytes(pdev, tcp, tcp->tx_con, completed_bytes, FALSE); 266 267 if (actual_completed != completed_bytes) { 268 DbgBreakIf(actual_completed > completed_bytes); 269 DbgBreakIf((completed_bytes - actual_completed) != 1); 270 DbgBreakIf(!(tx_con->flags & TCP_FIN_REQ_POSTED)); 271 DbgBreakIf(tx_con->bytes_post_cnt != tx_con->bytes_comp_cnt); 272 /* fin completed */ 273 tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_CMP; 274 tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_FIN_REQ_COMPLETED */ 275 } 276 277 /* network reachability (NOT IMPLEMENTED): 278 lm_neigh_update_nic_reachability_time(tcp->path->neigh) 279 */ 280 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) { 281 mm_release_tcp_lock(pdev, tx_con); 282 } 283 284 } /* lm_tcp_tx_app_cmp_process */ 285 286 u8_t lm_toe_is_tx_completion(lm_device_t *pdev, u8_t drv_toe_rss_id) 287 { 288 u8_t result = FALSE; 289 lm_tcp_scq_t *scq = NULL; 290 291 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id)); 292 293 scq = &pdev->toe_info.scqs[drv_toe_rss_id]; 294 295 if ( scq->hw_con_idx_ptr && 296 *scq->hw_con_idx_ptr != lm_bd_chain_cons_idx(&scq->bd_chain) ) 297 { 298 result = TRUE; 299 } 300 DbgMessage(pdev, INFORMl4int, "lm_toe_is_tx_completion(): result is:%s\n", result? "TRUE" : "FALSE"); 301 302 return result; 303 } 304 305 void lm_tcp_tx_inc_trm_aborted_bytes( 306 struct _lm_device_t *pdev, 307 lm_tcp_state_t *tcp, 308 u32_t aborted_bytes 309 ) 310 { 311 lm_tcp_con_t *tx_con = tcp->tx_con; 312 MM_INIT_TCP_LOCK_HANDLE(); 313 314 DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_inc_aborted_count, cid=%d, aborted_bytes=%d\n", 315 tcp->cid, aborted_bytes); 316 317 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) { 318 mm_acquire_tcp_lock(pdev, tx_con); 319 } 320 321 tx_con->bytes_trm_aborted_cnt += aborted_bytes; 322 323 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) { 324 mm_release_tcp_lock(pdev, tx_con); 325 } 326 327 } /* lm_tcp_tx_inc_aborted_count */ 328 329 /** Description 330 * completes the fast-path operations for a certain connection 331 * Assumption: 332 * fp-tx lock is taken 333 */ 334 void lm_tcp_tx_complete_tcp_fp(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con) 335 { 336 /**** Client completing : may result in lock-release *****/ 337 /* during lock-release, due to this function being called from service_deferred, more 338 * cqes can be processed. We don't want to mix. This function is mutually exclusive, so 339 * any processing makes it's way to being completed by calling this function. 340 * the following define a "fast-path completion" 341 * (i) RQ buffers to be completed 342 * defined by dpc_completed_tail and are collected during lm_tcp_complete_bufs BEFORE lock 343 * is released, so no more buffer processing can make it's way into this buffer completion. 344 * (ii) Fin to be completed 345 * determined by the flags, since dpc_flags CAN be modified during processing we copy 346 * them to a snapshot_flags parameter, which is initialized in this function only, so no fin 347 * can can make its way in while we release the lock. 348 * (iv) Remainders for sp 349 * all sp operations are logged in dpc_flags. for the same reason as (iii) no sp commands can 350 * make their way in during this fp-completion, all sp-processing after will relate to this point in time. 351 */ 352 353 con->dpc_info.snapshot_flags = con->dpc_info.dpc_flags; 354 con->dpc_info.dpc_flags = 0; 355 356 /* complete buffers to client */ 357 if (con->dpc_info.dpc_completed_tail != NULL) { 358 lm_tcp_complete_bufs(pdev, tcp, con); 359 } 360 361 /* Graceful Disconnect */ 362 if (con->dpc_info.snapshot_flags & LM_TCP_DPC_FIN_CMP) { 363 con->dpc_info.snapshot_flags &= ~LM_TCP_DPC_FIN_CMP; 364 lm_tcp_tx_graceful_disconnect_complete(pdev, con->tcp_state); 365 } 366 367 } 368 369 void lm_tcp_tx_process_cqe( 370 lm_device_t * pdev, 371 struct toe_tx_cqe * cqe, 372 lm_tcp_state_t * tcp 373 ) 374 { 375 enum toe_sq_opcode_type cmd; 376 377 /* get the cmd from cqe */ 378 cmd = ((cqe->params & TOE_TX_CQE_COMPLETION_OPCODE) >> TOE_TX_CQE_COMPLETION_OPCODE_SHIFT); 379 380 DbgMessage(pdev, INFORMl4tx, "###lm_tcp_tx_process_cqe cid=%d cmd=%d\n", tcp->cid, cmd); 381 DbgBreakIf( ! (pdev && tcp) ); 382 /* Check that the cqe len make sense, we could have got here by chance... */ 383 DbgBreakIfAll(cqe->len & 0xc0000000); /* two upper bits on show a completion larger than 1GB - a bit odd...*/ 384 385 /* Three types of completios: fast-path, reset-recv, ramrod-cmp. All completions may have a 386 * fast-path part (nbytes completed) which will be handled in any case that cqe->len > 0 */ 387 388 /* complete data if anything needs to be complete */ 389 if (cqe->len && 390 ((tcp->tx_con->dpc_info.dpc_flags & LM_TCP_DPC_RESET_RECV /* RST recv on this DPC on a previous CQE */ ) || 391 (tcp->tx_con->flags & TCP_REMOTE_RST_RECEIVED /* RST recv on previous DPC */ ))) 392 { 393 /* 10/28/08 - Since in exterme cases current FW may not complete all sent+acked bytes 394 on RST recv cqe and do so only later on one of the following ramrod completions, 395 we need to ignore this too late completed bytes thus we nullify cqe->len */ 396 DbgBreakIf((cmd != RAMROD_OPCODE_TOE_RESET_SEND) && 397 (cmd != RAMROD_OPCODE_TOE_INVALIDATE) && 398 (cmd != RAMROD_OPCODE_TOE_EMPTY_RAMROD) && 399 (cmd != RAMROD_OPCODE_TOE_TERMINATE)); 400 lm_tcp_tx_inc_trm_aborted_bytes(pdev, tcp, cqe->len); 401 cqe->len = 0; 402 } 403 if (cqe->len) { 404 DbgBreakIf(tcp->tx_con->dpc_info.dpc_comp_blocked); 405 lm_tcp_tx_cmp_process(pdev, tcp, cqe->len); 406 } 407 408 switch(cmd) { 409 case CMP_OPCODE_TOE_TX_CMP: 410 break; 411 case CMP_OPCODE_TOE_RST_RCV: 412 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RESET_RECV; 413 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_RST_RECEIVED */ 414 break; 415 case RAMROD_OPCODE_TOE_RESET_SEND: 416 DbgBreakIf(! tcp->sp_request); 417 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_ABORTIVE_DISCONNECT); 418 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 419 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_RST_REQ_COMPLETED */ 420 break; 421 case RAMROD_OPCODE_TOE_INVALIDATE: 422 DbgBreakIf(! tcp->sp_request); 423 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INVALIDATE); 424 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 425 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_INV_REQ_COMPLETED */ 426 break; 427 case RAMROD_OPCODE_TOE_TERMINATE: 428 DbgBreakIf(! tcp->sp_request); 429 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE1_OFFLOAD); 430 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 431 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_TRM_REQ_COMPLETED */ 432 break; 433 case RAMROD_OPCODE_TOE_EMPTY_RAMROD: 434 DbgBreakIf(cqe->len); 435 DbgBreakIf(! tcp->sp_request ); 436 DbgBreakIf((tcp->sp_request->type != SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT) && 437 (tcp->sp_request->type != SP_REQUEST_PENDING_REMOTE_DISCONNECT) && 438 (tcp->sp_request->type != SP_REQUEST_PENDING_TX_RST)); 439 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP; 440 break; 441 default: 442 DbgMessage(pdev, FATAL, "unexpected tx cqe opcode=%d\n", cmd); 443 DbgBreakIfAll(TRUE); 444 } 445 } 446 447 /** Description 448 * 449 * Assumptions 450 * connections is initialzed with a dummy head. 451 */ 452 void lm_tcp_tx_process_cqes(lm_device_t *pdev, u8_t drv_toe_rss_id, s_list_t * connections) 453 { 454 lm_tcp_scq_t *scq; 455 struct toe_tx_cqe *cqe, *hist_cqe; 456 lm_tcp_state_t *tcp; 457 u32_t cid; 458 u32_t avg_dpc_cnt; 459 u16_t cq_new_idx; 460 u16_t cq_old_idx; 461 u16_t num_to_reproduce = 0; 462 u8_t defer_cqe; 463 MM_INIT_TCP_LOCK_HANDLE(); 464 465 DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_tx_process_cqes\n"); 466 467 scq = &pdev->toe_info.scqs[drv_toe_rss_id]; 468 cq_new_idx = *(scq->hw_con_idx_ptr); 469 cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain); 470 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0); 471 472 /* save statistics */ 473 scq->num_cqes_last_dpc = S16_SUB(cq_new_idx, cq_old_idx); 474 if (scq->num_cqes_last_dpc) { /* Exclude zeroed value from statistics*/ 475 if(scq->max_cqes_per_dpc < scq->num_cqes_last_dpc) { 476 scq->max_cqes_per_dpc = scq->num_cqes_last_dpc; 477 } 478 /* we don't want to wrap around...*/ 479 if ((scq->sum_cqes_last_x_dpcs + scq->num_cqes_last_dpc) < scq->sum_cqes_last_x_dpcs) { 480 scq->avg_dpc_cnt = 0; 481 scq->sum_cqes_last_x_dpcs = 0; 482 } 483 scq->sum_cqes_last_x_dpcs += scq->num_cqes_last_dpc; 484 scq->avg_dpc_cnt++; 485 avg_dpc_cnt = scq->avg_dpc_cnt; 486 if (avg_dpc_cnt) { 487 scq->avg_cqes_per_dpc = scq->sum_cqes_last_x_dpcs / avg_dpc_cnt; 488 } else { 489 scq->sum_cqes_last_x_dpcs = 0; 490 } 491 } 492 493 while(cq_old_idx != cq_new_idx) { 494 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0); 495 496 /* get next consumed cqe */ 497 cqe = lm_toe_bd_chain_consume_bd(&scq->bd_chain); 498 DbgBreakIf(!cqe); 499 num_to_reproduce++; 500 501 /* get tcp state from cqe */ 502 cid = SW_CID(((cqe->params & TOE_TX_CQE_CID) >> TOE_TX_CQE_CID_SHIFT)); 503 tcp = lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, cid); 504 DbgBreakIf(!tcp); 505 /* save cqe in history_cqes */ 506 hist_cqe = (struct toe_tx_cqe *)lm_tcp_qe_buffer_next_cqe_override(&tcp->tx_con->history_cqes); 507 *hist_cqe = *cqe; 508 509 defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED); 510 if (defer_cqe) { 511 /* if we're deferring completions - just store the cqe and continue to the next one */ 512 /* Return if we are still deferred (may have changed since initial check was w/o a lock */ 513 mm_acquire_tcp_lock(pdev, tcp->tx_con); 514 /* check again under lock if we're deferred */ 515 defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED); 516 if (defer_cqe) { 517 tcp->tx_con->flags |= TCP_DEFERRED_PROCESSING; 518 lm_tcp_tx_process_cqe(pdev, cqe, tcp); 519 } 520 mm_release_tcp_lock(pdev, tcp->tx_con); 521 } 522 if (!defer_cqe) { 523 /* connections will always be initialized to a dummy, so once a tcp connection is added to the 524 * list, it's link will be initialized to point to another link other than NULL */ 525 if (s_list_next_entry(&tcp->tx_con->dpc_info.link) == NULL) { 526 s_list_push_head(connections, &tcp->tx_con->dpc_info.link); 527 } 528 lm_tcp_tx_process_cqe(pdev, cqe, tcp); 529 } 530 cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain); 531 /* GilR 5/12/2006 - TODO - decide with Alon if reading the hw_con again is required */ 532 //cq_new_idx = *(scq->hw_con_idx_ptr); 533 } 534 535 /* The fact that we post the producer here before we've handled any slow-path completions assures that 536 * the sp-ring will always be updated AFTER the producer was. */ 537 if (num_to_reproduce) { 538 lm_toe_bd_chain_bds_produced(&scq->bd_chain, num_to_reproduce); 539 540 /* GilR 5/13/2006 - TBA - save some stats? */ 541 542 /* notify the fw of the prod of the SCQ */ 543 LM_INTMEM_WRITE16(pdev, CSTORM_TOE_CQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)), 544 lm_bd_chain_prod_idx(&scq->bd_chain), BAR_CSTRORM_INTMEM); 545 } 546 } 547 548 void lm_toe_service_tx_intr(lm_device_t *pdev, u8_t drv_toe_rss_id) 549 { 550 s_list_t connections; 551 s_list_entry_t dummy; 552 lm_tcp_con_t * con; 553 lm_tcp_state_t * tcp; 554 555 MM_INIT_TCP_LOCK_HANDLE(); 556 557 DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_tx_intr\n"); 558 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id)); 559 560 s_list_clear(&connections); 561 s_list_push_head(&connections, &dummy); 562 /* process the cqes and initialize connections with all the connections that appeared 563 * in the DPC */ 564 lm_tcp_tx_process_cqes(pdev,drv_toe_rss_id,&connections); 565 566 /* complete the fp/sp parts of the connections remember to ignore the last one */ 567 con = (lm_tcp_con_t *)s_list_peek_head(&connections); 568 tcp = con->tcp_state; 569 while (s_list_next_entry(&con->dpc_info.link) != NULL) { 570 mm_acquire_tcp_lock(pdev, con); 571 lm_tcp_tx_complete_tcp_fp(pdev, con->tcp_state, con); 572 mm_release_tcp_lock(pdev, con); 573 con = (lm_tcp_con_t *)s_list_next_entry(&con->dpc_info.link); 574 tcp = con->tcp_state; 575 } 576 577 /* SP : traverse the connections. remember to ignore the last one */ 578 con = (lm_tcp_con_t *)s_list_pop_head(&connections); 579 s_list_next_entry(&con->dpc_info.link) = NULL; 580 tcp = con->tcp_state; 581 while (s_list_entry_cnt(&connections) > 0) { 582 /* we access snapshot and not dpc, since once the dpc_flags were copied 583 * to snapshot they were zeroized */ 584 if (con->dpc_info.snapshot_flags) { 585 lm_tcp_tx_complete_tcp_sp(pdev, tcp, con); 586 } 587 con = (lm_tcp_con_t *)s_list_pop_head(&connections); 588 s_list_next_entry(&con->dpc_info.link) = NULL; 589 tcp = con->tcp_state; 590 } 591 592 } 593 594 lm_status_t lm_tcp_graceful_disconnect( 595 IN lm_device_t * pdev, 596 IN lm_tcp_state_t * tcp_state 597 ) 598 { 599 struct toe_tx_bd *tx_bd; 600 lm_tcp_con_t *tcp_con = tcp_state->tx_con; 601 u16_t old_prod, new_prod; 602 u32_t dbg_bytes_prod_seq; 603 604 DbgMessage(pdev, INFORMl4tx, "###lm_tcp_graceful_disconnect\n"); 605 606 if ( tcp_con->flags & TCP_TX_POST_BLOCKED ) { 607 return LM_STATUS_CONNECTION_CLOSED; 608 } 609 610 DbgBreakIf( (tcp_con->app_buf_bytes_acc_post != 0) || 611 (tcp_con->db_more_bytes != 0) || 612 (tcp_con->db_more_bds != 0) || 613 (tcp_con->u.tx.bds_without_comp_flag != 0) 614 ); 615 616 old_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain)); 617 618 /* Post FIN BD on Tx chain */ 619 tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(&(tcp_con->bd_chain)); 620 tx_bd->flags = TOE_TX_BD_FIN; /* Vladz: Pay attention when u move this 621 line - there is an assignment to flags, NOT bitwise OR */ 622 tx_bd->flags |= TOE_TX_BD_NOTIFY; 623 tx_bd->size = 1; 624 /* For a safety */ 625 tx_bd->addr_hi = tx_bd->addr_lo = 0; 626 627 dbg_bytes_prod_seq = tcp_con->db_data.tx->bytes_prod_seq + tcp_con->db_more_bytes; 628 dbg_bytes_prod_seq += tx_bd->size; 629 tx_bd->nextBdStartSeq = dbg_bytes_prod_seq; 630 631 new_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain)); 632 DbgBreakIf(S16_SUB(new_prod, old_prod) >= 3); 633 DbgBreakIf(S16_SUB(new_prod, old_prod) <= 0); 634 635 DbgBreakIf(tcp_con->flags & TCP_FIN_REQ_POSTED); 636 tcp_con->flags |= TCP_FIN_REQ_POSTED; 637 638 /* Update fin request time, if not already set by the caller */ 639 if (!tcp_state->tcp_state_calc.fin_request_time) { 640 tcp_state->tcp_state_calc.fin_request_time = mm_get_current_time(pdev); 641 if (tcp_state->tcp_state_calc.fin_request_time == tcp_state->tcp_state_calc.fin_reception_time){ 642 tcp_state->tcp_state_calc.fin_reception_time -= 1; 643 } 644 } 645 646 /* Doorbell FIN */ 647 _lm_tcp_tx_write_db(pdev, tcp_con, tcp_state->cid, 0, (u16_t)S16_SUB(new_prod, old_prod), 1); 648 649 /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */ 650 DbgBreakIf(S64_SUB(tcp_con->bytes_post_cnt + 1, tcp_con->bytes_comp_cnt) < 0); 651 tcp_con->bytes_post_cnt++; 652 tcp_con->fp_db_cnt++; 653 654 return LM_STATUS_SUCCESS; 655 } 656 657 658