1 /******************************************************************************* 2 * lm_l4st.h - L4 lm data structures 3 ******************************************************************************/ 4 #ifndef _LM_L4ST_H 5 #define _LM_L4ST_H 6 7 #include "l4states.h" 8 #include "bd_chain_st.h" 9 #include "lm_sp_req_mgr.h" 10 #include "toe_constants.h" 11 12 #define MAX_L4_RX_CHAIN 16 13 #define MAX_L4_TX_CHAIN 16 14 15 #define PATTERN_COUNTER_IDX_COMPLETION 0 16 #define PATTERN_COUNTER_IDX_CQE 1 17 #define MAX_PATTERN_IDX 2 18 19 typedef struct _lm_tcp_integrity_info_t 20 { 21 u32_t current_offset_in_pattern_buf[MAX_PATTERN_IDX]; 22 u32_t skip_bytes_in_incoming_buf[MAX_PATTERN_IDX]; 23 u32_t is_offsets_initialized; 24 } lm_tcp_integrity_info_t; 25 26 27 typedef struct _lm_toe_integrity_info_t 28 { 29 u8_t * pattern_buf; 30 u32_t pattern_buf_size; 31 u32_t pattern_size; 32 } lm_toe_integrity_info_t; 33 34 35 /******************************************************************************* 36 * 'Posting' TCP buffer. 37 ******************************************************************************/ 38 typedef struct _lm_tcp_buffer_t 39 { 40 /* Must be the first entry in this structure. */ 41 s_list_entry_t link; 42 43 /* Corresponds to l4buffer_t buffer_size. 44 * The number of bytes in this buffer may not corresponds to the 45 * the number of bytes of the application buffer. An application buffer 46 * could span multiple tcp_bufs. The flags field is used to mark the 47 * starting and the end of an application buffer. */ 48 u32_t size; 49 50 /* Number of bytes that were not completed yet */ 51 u32_t more_to_comp; 52 53 u32_t flags; /* Flags for indicating the start and end of an io buffer. */ 54 #define TCP_BUF_FLAG_NONE 0x00 55 #define TCP_BUF_FLAG_L4_POST_START 0x01 56 #define TCP_BUF_FLAG_L4_POST_END 0x02 57 #define TCP_BUF_FLAG_L4_RESERVED1 0x04 /* used in Teton for dummy buffer. */ 58 #define TCP_BUF_FLAG_L4_SPLIT 0x04 /* Used in everest for split buffer Everest cleans before completing to miniport */ 59 #define TCP_BUF_FLAG_L4_RESERVED2 0x08 /* used only in Miniport as 'last post' */ 60 #define TCP_BUF_FLAG_L4_RX_NO_PUSH 0x10 61 #define TCP_BUF_FLAG_L4_PARTIAL_FILLED 0x20 62 /* NOTE: lm_tcp_buffer flags values must correspond to flags definition in l4buffer_t */ 63 64 u16_t bd_used; /* Number of BDs consumed in the bd chain for this tcp buffer */ 65 u16_t _pad; 66 67 /* These fields are valid when TCP_BUF_FLAG_L4_POST_END flag is set. */ 68 u32_t app_buf_size; /* Number of bytes of all buffers from BUFFER_START till BUFFER_END */ 69 u32_t app_buf_xferred; /* Number of bytes xferred on all buffers from BUFFER_START till BUFFER_END */ 70 } lm_tcp_buffer_t; 71 72 /******************************************************************************* 73 * state header. 74 * Each state must start with this entry, which is used for chaining 75 * states together and for identifying a particular state. 76 ******************************************************************************/ 77 typedef struct _lm_state_header_t 78 { 79 d_list_entry_t link; 80 struct _lm_state_block_t *state_blk; 81 82 u32_t state_id; 83 #define STATE_ID_UNKNOWN 0 84 #define STATE_ID_TCP 1 85 #define STATE_ID_PATH 2 86 #define STATE_ID_NEIGH 3 87 88 u32_t status; 89 #define STATE_STATUS_UNKNOWN 0 90 #define STATE_STATUS_INIT 1 91 #define STATE_STATUS_INIT_CONTEXT 2 92 #define STATE_STATUS_OFFLOAD_PENDING 3 93 #define STATE_STATUS_NORMAL 4 94 #define STATE_STATUS_ABORTED 5 95 #define STATE_STATUS_INVALIDATED 6 96 #define STATE_STATUS_UPLOAD_PENDING 7 97 #define STATE_STATUS_UPLOAD_DONE 8 98 #define STATE_STATUS_INIT_OFFLOAD_ERR 9 99 #define STATE_STATUS_ERR 10 100 } lm_state_header_t; 101 102 /******************************************************************************* 103 * neighbor state 104 ******************************************************************************/ 105 typedef struct _lm_neigh_state_t 106 { 107 lm_state_header_t hdr; 108 109 l4_neigh_const_state_t neigh_const; 110 l4_neigh_cached_state_t neigh_cached; 111 l4_neigh_delegated_state_t neigh_delegated; 112 113 /* network reachability */ 114 u32_t host_reachability_time; 115 u32_t nic_reachability_time; 116 u8_t stale; 117 u8_t _pad[3]; 118 119 /* debug */ 120 u32_t num_dependents; /* number of dependent path states */ 121 } lm_neigh_state_t; 122 123 /******************************************************************************* 124 * path state 125 ******************************************************************************/ 126 typedef struct _lm_path_state_t 127 { 128 lm_state_header_t hdr; 129 130 lm_neigh_state_t *neigh; /* parent neighbor state */ 131 132 l4_path_const_state_t path_const; 133 l4_path_cached_state_t path_cached; 134 l4_path_delegated_state_t path_delegated; 135 136 /* debug */ 137 u32_t num_dependents; /* number of dependent tcp states */ 138 } lm_path_state_t; 139 140 /******************************************************************************* 141 * queue element buffer - for buffering queue elements (of any type) 142 ******************************************************************************/ 143 typedef struct _lm_tcp_qe_buffer_t 144 { 145 char *first; 146 char *tail; 147 char *head; 148 char *last; 149 150 u32_t qe_size; /* queue element size */ 151 u32_t left; 152 } lm_tcp_qe_buffer_t; 153 154 155 /******************************************************************************* 156 * Memory Blocks 157 ******************************************************************************/ 158 typedef struct _lm_tcp_mem_block_t 159 { 160 s_list_entry_t link; /* Must be the first entry... */ 161 162 u8_t * start; /* Start of the memory block */ 163 u8_t * free; /* Pointer to the start of the remaining free space of the block */ 164 u32_t total; /* Size of the entire block */ 165 u32_t left; /* free bytes left in the block */ 166 u8_t flags; /* virt-memblock-pool member or not */ 167 #define MBLK_RETURN_TO_POOL 0x1 168 } lm_tcp_mem_block_t; 169 170 typedef struct _lm_tcp_phy_mem_block_t 171 { 172 s_list_entry_t link; 173 174 u8_t * start; /* Start of the memory block */ 175 u8_t * free; /* Pointer to the start of the remaining free space of the block */ 176 u32_t total; /* Size of the entire block */ 177 u32_t left; /* free bytes left in the block */ 178 179 lm_address_t start_phy; 180 lm_address_t free_phy; 181 } lm_tcp_phy_mem_block_t; 182 183 #define DEBUG_OOO_CQE 184 typedef struct _lm_isle_t 185 { 186 d_list_entry_t isle_link; 187 d_list_t isle_gen_bufs_list_head; 188 u32_t isle_nbytes; 189 #ifdef DEBUG_OOO_CQE 190 u32_t dedicated_cid; 191 u32_t recent_ooo_combined_cqe; 192 #endif 193 } lm_isle_t; 194 195 #ifdef DEBUG_OOO_CQE 196 #define SET_DEBUG_OOO_INFO(_isle, _cmd, _data) \ 197 (_isle)->recent_ooo_combined_cqe = ((((_cmd) << TOE_RX_CQE_COMPLETION_OPCODE_SHIFT) & TOE_RX_CQE_COMPLETION_OPCODE) \ 198 | (((_data) << TOE_RX_CQE_OOO_PARAMS_NBYTES_SHIFT) & TOE_RX_CQE_OOO_PARAMS_NBYTES)) 199 #define GET_RECENT_OOO_CMD(_isle) \ 200 (((_isle)->recent_ooo_combined_cqe & TOE_RX_CQE_COMPLETION_OPCODE) >> TOE_RX_CQE_COMPLETION_OPCODE_SHIFT) 201 #define GET_RECENT_OOO_DATA(_isle) \ 202 (((_isle)->recent_ooo_combined_cqe & TOE_RX_CQE_OOO_PARAMS_NBYTES) >> TOE_RX_CQE_OOO_PARAMS_NBYTES_SHIFT) 203 #endif 204 205 /******************************************************************************* 206 * Rx connection's generic buffers info. 207 ******************************************************************************/ 208 typedef struct _lm_tcp_con_rx_gen_info_t 209 { 210 d_list_t peninsula_list; /* accessed only via lock */ 211 d_list_t dpc_peninsula_list; /* accessed lock-free only in dpc */ 212 213 d_list_t isles_list; 214 lm_isle_t first_isle; 215 lm_isle_t * current_isle; 216 u8_t current_isle_number; 217 u8_t max_number_of_isles; 218 u8_t _isle_pad[2]; 219 220 lm_frag_list_t * frag_list; /* allocated in initialization of connection */ 221 u32_t max_frag_count; /* the number of frags statically allocated */ 222 223 u32_t peninsula_nbytes; 224 u32_t dpc_peninsula_nbytes; 225 u32_t isle_nbytes; 226 u16_t first_buf_offset; 227 228 /* How many buffers (head of indications) were indicated for this connection and haven't 229 * returned yet from NDIS. We need to know that to make sure we don't delete the connection 230 * before all buffers pointing to it have returned. 231 */ 232 u16_t pending_return_indications; 233 /* bytes indicated that their buffers have not yet been returned, this is a value that will increase 234 * the window. If we're uploaded and we still have pending_indicated_bytes we need to increase them immediataly 235 * and not wait... */ 236 u32_t pending_indicated_bytes; 237 238 /* Each indication may result in us updating the window - this depends on the #of bytes accepted AND the update_window_mode 239 * we're in. We aggregate this over all indications (mm_tcp_rx_indicate_gen may be called several times if more generic data 240 * was received during indicate). This field is updated ONLY by the function lm_tcp_rx_buffered_data_indicated, and is accessed 241 * once the mm_tcp_rx_indicate_gen function completes. The main reason for this aggregation, unfortunatelly, is for passing 242 * SpartaTest - receive_indications, which expects a specific number of indications. */ 243 u32_t add_sws_bytes; 244 245 u8_t wait_for_isle_left; 246 u8_t _padding; 247 248 /* The update window mode is taken from the toe information before an indication 249 * We can't use the main copy because it may change between the time we indicate 250 * (after we've marked the buffer) and the time we get an answer (and need to determine 251 * whether to update the window or not) */ 252 u8_t update_window_mode; 253 254 /* debug/statistics */ 255 /* DEFINITION: A generic buffer can be 'done' with as a result of a succesfull indicate or as a result of a copy 256 * operation to an application buffer. (regardless of its state before: partially indicated/partially copied). 257 * We count the number of times generic buffers were 'done' with */ 258 u8_t peninsula_blocked; /* peninsula is blocked as a result of terminate (get_buffered_data) */ 259 u32_t num_buffers_indicated; /* 'done' with as a result of an indicate */ 260 u32_t num_buffers_copied_grq;/* # grq buffers copied */ 261 u32_t num_buffers_copied_rq; /* # rq buffers copied TBD how to count*/ 262 u32_t num_bytes_indicated; /* all bytes indicated in either full/partial indications */ 263 u32_t copy_gen_buf_fail_cnt; /* counts the number of times a client.copy operation failed */ 264 u32_t copy_gen_buf_dmae_cnt; /* counts the number of times dmae copy operation was used */ 265 u32_t num_success_indicates; /* number of times indicate succeeded */ 266 u32_t num_failed_indicates; /* number of times indicate failed */ 267 u32_t bufs_indicated_rejected; /* number of rejected bufs */ 268 u64_t bytes_copied_cnt_in_process; 269 u64_t bytes_copied_cnt_in_post; 270 u64_t bytes_copied_cnt_in_comp; 271 u64_t bytes_indicated_accepted; 272 u64_t bytes_indicated_rejected; 273 u32_t dont_send_to_system_more_then_rwin; 274 u32_t num_non_full_indications; 275 276 } lm_tcp_con_rx_gen_info_t; 277 278 /******************************************************************************* 279 * Rx connection's receive window information for silly window syndrome avoidance 280 ******************************************************************************/ 281 #define MAX_INITIAL_RCV_WND 0x80000000 /* 2GB (due to cyclic counters and window-update algorithm */ 282 283 /* DWA: Delayed Window Update Algorithm : the twin of DCA, delay the window updates according to the delayed completions. */ 284 285 #define MAX_DW_THRESH_ENTRY_CNT 16 /* a new entry is created each time we see a NDC completion (non-delayed-complete). We 286 * can limit these to 16 'active completions' i.e. completions that haven't received a 287 * window-update yet. FW-DCA works with quad-buffer, therefore 16 is more than enough. */ 288 289 typedef struct _lm_tcp_rx_dwa_info { 290 u32_t dw_thresh[MAX_DW_THRESH_ENTRY_CNT]; /* delayed window update thresholds. */ 291 u8_t head; /* head of the the cyclic buffer dw_thresh (next empty entry) */ 292 u8_t tail; /* tail of the the cyclic buffer dw_thresh */ 293 u16_t _pad; 294 } lm_tcp_rx_dwa_info; 295 296 typedef struct _lm_tcp_con_rx_sws_info_t 297 { 298 u32_t drv_rcv_win_right_edge; /* The drivers window right edge (shadow of fw, and may be 299 * larger if the difference is smaller than mss) */ 300 u32_t mss; /* min(tcp_const.remote_mss, 301 parent_path->path_cached.path_mtu - HEADERS size) */ 302 u32_t extra_bytes; 303 304 u8_t timer_on; 305 } lm_tcp_con_rx_sws_info_t; 306 307 /******************************************************************************* 308 * Rx connection's special information 309 ******************************************************************************/ 310 typedef struct _lm_tcp_con_rx_t 311 { 312 lm_tcp_con_rx_gen_info_t gen_info; 313 lm_tcp_con_rx_sws_info_t sws_info; 314 315 /* Last bd written to: required in spcecial case of very large application buffers 316 * not fitting into the bd-chain . */ 317 struct toe_rx_bd * last_rx_bd; 318 319 /* Remember a remote disconnect event until all received data is 320 * completed/indicated successfully to the client */ 321 u8_t flags; 322 #define TCP_CON_RST_IND_PENDING 0x1 323 #define TCP_CON_FIN_IND_PENDING 0x2 324 u8_t zero_byte_posted_during_ind; 325 u8_t check_data_integrity_on_complete; 326 u8_t check_data_integrity_on_receive; 327 u32_t compared_bytes; 328 329 u32_t skp_bytes_copied; /* counter of bytes that were already copied to the buffer at post that we 330 * will receive a skip for which we'll need to ignore...This counter must be protected 331 * by a lock */ 332 /* GilR 4/3/2006 - TBA - add lm tcp con rx debug/stats fields? */ 333 u32_t rx_zero_byte_recv_reqs; /* #Zero byte receeive requests */ 334 } lm_tcp_con_rx_t; 335 336 /******************************************************************************* 337 * Tx connection's special information 338 ******************************************************************************/ 339 typedef struct _lm_tcp_con_tx_t 340 { 341 u16_t bds_without_comp_flag; /* counter of consecutive BDs without CompFlag */ 342 u8_t flags; 343 #define TCP_CON_FIN_REQ_LM_INTERNAL 0x1 /* FIN request completion should 344 * not be indicated to mm */ 345 #define TCP_CON_RST_IND_NOT_SAFE 0x2 346 347 348 u8_t _pad; 349 u32_t mss; 350 } lm_tcp_con_tx_t; 351 352 353 /******************************************************************************* 354 * TCP connection - rx OR tx 355 ******************************************************************************/ 356 /* This structure is used to collect information during a DPC without taking the 357 * fp-lock. All fields in this structure must be accessed ONLY from within the 358 * the DPC 359 */ 360 typedef struct _lm_tcp_dpc_info_t 361 { 362 s_list_entry_t link; /* must be the first entry here */ 363 s_list_entry_t * dpc_completed_tail; /* points to the tail of the sub-list of active_tb_list that needs to 364 * be completed. */ 365 u32_t dpc_bufs_completed; /* number of buffers completed in the dpc (aggregated during process 366 * stage for fast splitting of the active_tb_list at completion stage)*/ 367 u32_t dpc_rq_placed_bytes; /* how many bytes were placed on rq as a result of rq-cmp / copying from grq->rq */ 368 u32_t dpc_actual_bytes_completed; /* number of bytes completed to client - aggregated during process stage */ 369 u16_t dpc_bd_used; /* number of bds used - aggregated during process stage */ 370 u16_t dpc_flags; /* flags marked during cqe processing - only accessed during processing and 371 * snapshot-ed under a lock */ 372 #define LM_TCP_DPC_RESET_RECV 0x1 373 #define LM_TCP_DPC_FIN_RECV 0x2 374 #define LM_TCP_DPC_FIN_CMP 0x4 375 #define LM_TCP_DPC_KA_TO 0x8 376 #define LM_TCP_DPC_RT_TO 0x10 377 #define LM_TCP_DPC_URG 0x20 378 #define LM_TCP_DPC_RAMROD_CMP 0x40 379 // #define LM_TCP_DPC_NDC 0x80 380 #define LM_TCP_DPC_DBT_RE 0x100 381 #define LM_TCP_DPC_OPT_ERR 0x200 382 #define LM_TCP_DPC_UPLD_CLOSE 0x400 383 #define LM_TCP_DPC_FIN_RECV_UPL 0x800 384 #define LM_TCP_DPC_TOO_BIG_ISLE 0x1000 385 #define LM_TCP_DPC_TOO_MANY_ISLES 0x2000 386 387 /* 388 #define LM_TCP_COMPLETE_FP (LM_TCP_DPC_RESET_RECV | LM_TCP_DPC_FIN_RECV | LM_TCP_DPC_FIN_RECV_UPL | LM_TCP_DPC_FIN_CMP | \ 389 LM_TCP_DPC_KA_TO | LM_TCP_DPC_RT_TO | LM_TCP_DPC_URG | LM_TCP_DPC_RAMROD_CMP | LM_TCP_DPC_NDC | \ 390 LM_TCP_DPC_DBT_RE | LM_TCP_DPC_OPT_ERR | LM_TCP_DPC_UPLD_CLOSE | \ 391 LM_TCP_DPC_TOO_BIG_ISLE | LM_TCP_DPC_TOO_MANY_ISLES) 392 */ 393 /* dpc snapshot parameters: taken before an operation that can release a lock is done 394 * in lm_tcp_xx_complete_fp */ 395 u16_t snapshot_flags; /* only accessed under lock */ 396 397 /* we have special cases where lm blocks um from posting until a specific buffer gets completed, we have a flag for this 398 * this flag is accessed with the post flow, so it should be protected by a lock, therefore we remember we have to unset it 399 * in the completion stage (under a lock) */ 400 u8_t dpc_unblock_post; 401 /* debug / stats */ 402 u8_t dpc_comp_blocked; 403 404 /* the window size returned from the fw after window size decreasment request returned, written back to the fw */ 405 u32_t dpc_fw_wnd_after_dec; 406 407 } lm_tcp_dpc_info_t; 408 409 typedef struct _lm_tcp_con_t 410 { 411 lm_tcp_dpc_info_t dpc_info; /* must be the first field */ 412 413 struct _lm_tcp_state_t * tcp_state; /* The tcp state associated with this connection */ 414 415 union { 416 volatile struct toe_rx_db_data *rx; 417 volatile struct toe_tx_db_data *tx; 418 } db_data; 419 lm_address_t phys_db_data; 420 421 /* rx/tx tcp connection info. */ 422 union 423 { 424 lm_tcp_con_rx_t rx; 425 lm_tcp_con_tx_t tx; 426 } u; 427 428 lm_bd_chain_t bd_chain; 429 430 /* List of posted buffers (i.e. attached to the bd chain) */ 431 s_list_t active_tb_list; 432 u32_t rq_nbytes; /* how many bytes are in the active-tb-list */ 433 434 /* buffer of cqes that represent the last X cqes received */ 435 lm_tcp_qe_buffer_t history_cqes; 436 437 u32_t type; 438 #define TCP_CON_TYPE_RX 1 439 #define TCP_CON_TYPE_TX 2 440 441 /* accumulator of currently posted application buffer bytes. 442 * accumulated in order to set lm_tcp_buffer.app_buf_size of 443 * the last tcp buffer of the application buffer */ 444 u32_t app_buf_bytes_acc_post; 445 446 /* accumulator of currently completed application buffer bytes. 447 * accumulated in order to set lm_tcp_buffer.app_buf_xferred of 448 * the last tcp buffer of the application buffer */ 449 u32_t app_buf_bytes_acc_comp; 450 451 u32_t db_more_bytes; /* number of bytes to be produced in next doorbell */ 452 u16_t db_more_bufs; /* number of tcp buffers to be produced in next doorbell */ 453 u16_t db_more_bds; /* number of bds to be produced in next doorbell */ 454 455 /* flags are used for managing the connection's posting/completing/indicating state machines */ 456 u32_t flags; 457 #define TCP_FIN_REQ_POSTED 0x0001 458 #define TCP_RST_REQ_POSTED 0x0002 459 #define TCP_INV_REQ_POSTED 0x0004 460 #define TCP_TRM_REQ_POSTED 0x0008 461 #define TCP_FIN_REQ_COMPLETED 0x0010 462 #define TCP_RST_REQ_COMPLETED 0x0020 463 #define TCP_INV_REQ_COMPLETED 0x0040 464 #define TCP_TRM_REQ_COMPLETED 0x0080 465 #define TCP_REMOTE_FIN_RECEIVED 0x0100 466 #define TCP_REMOTE_RST_RECEIVED 0x0200 467 #define TCP_REMOTE_FIN_RECEIVED_ALL_RX_INDICATED 0x0400 468 #define TCP_REMOTE_RST_RECEIVED_ALL_RX_INDICATED 0x0800 469 #define TCP_INDICATE_REJECTED 0x1000 470 #define TCP_POST_BLOCKED 0x2000 471 #define TCP_COMP_BLOCKED 0x4000 472 #define TCP_COMP_DEFERRED 0x8000 473 #define TCP_BUFFERS_ABORTED 0x10000 474 #define TCP_DEFERRED_PROCESSING 0x20000 475 #define TCP_POST_DELAYED 0x40000 /* lm sets this when posting buffers is delay for some reason */ 476 #define TCP_POST_COMPLETE_SPLIT 0x80000 /* lm sets this when every split buffer that'll be posted will be completed immediately */ 477 #define TCP_POST_NO_SKP 0x100000 /* lm sets this when there will be no more skp completions from fw (comp blocked...) */ 478 #define TCP_UPLOAD_REQUESTED 0x200000 /* lm sets this when FW requests an upload for any reason - after this is set, no more uploads will be requested*/ 479 #define TCP_DB_BLOCKED 0x400000 480 #define TCP_RX_DB_BLOCKED (TCP_REMOTE_FIN_RECEIVED | TCP_REMOTE_RST_RECEIVED | TCP_DB_BLOCKED) 481 #define TCP_TX_DB_BLOCKED (TCP_REMOTE_RST_RECEIVED | TCP_DB_BLOCKED) 482 #define TCP_TX_POST_BLOCKED (TCP_FIN_REQ_POSTED | TCP_RST_REQ_POSTED | TCP_INV_REQ_POSTED | TCP_TRM_REQ_POSTED | \ 483 TCP_REMOTE_RST_RECEIVED_ALL_RX_INDICATED | TCP_POST_BLOCKED \ 484 ) 485 /* GilR 4/4/2006 - TBD - open issue with Hav, for Tx POST BLOCKED we might not wait for 'rx indicated' after RST received */ 486 #define TCP_RX_POST_BLOCKED (TCP_RST_REQ_POSTED | TCP_INV_REQ_POSTED | TCP_TRM_REQ_POSTED | \ 487 TCP_REMOTE_RST_RECEIVED_ALL_RX_INDICATED | TCP_REMOTE_FIN_RECEIVED_ALL_RX_INDICATED | \ 488 TCP_POST_BLOCKED) 489 #define TCP_TX_COMP_BLOCKED (TCP_RST_REQ_COMPLETED | TCP_FIN_REQ_COMPLETED | TCP_REMOTE_RST_RECEIVED | \ 490 TCP_INV_REQ_COMPLETED | TCP_TRM_REQ_COMPLETED | TCP_COMP_BLOCKED \ 491 ) 492 #define TCP_RX_COMP_BLOCKED (TCP_RST_REQ_COMPLETED | TCP_REMOTE_FIN_RECEIVED | TCP_REMOTE_RST_RECEIVED | \ 493 TCP_INV_REQ_COMPLETED | TCP_TRM_REQ_COMPLETED | TCP_COMP_BLOCKED \ 494 ) 495 #define TCP_TX_COMP_DEFERRED TCP_COMP_DEFERRED 496 #define TCP_RX_COMP_DEFERRED TCP_COMP_DEFERRED 497 #define TCP_RX_IND_BLOCKED (TCP_RST_REQ_POSTED | TCP_INV_REQ_POSTED | TCP_TRM_REQ_POSTED | TCP_INDICATE_REJECTED) 498 499 /* GilR 4/3/2006 - TBA - add lm con debug/statistics */ 500 u64_t bytes_post_cnt; /* cyclic counter of posted application buffer bytes */ 501 u64_t bytes_comp_cnt; /* cyclic counter of completed application buffer bytes (including skipped bytes due to push) */ 502 u64_t bytes_push_skip_cnt; 503 u64_t bytes_skip_post_cnt; /* skipped post because of generic data */ 504 u32_t buffer_skip_post_cnt; /* skipped post because of generic data */ 505 u32_t buffer_post_cnt; 506 u32_t buffer_completed_cnt; 507 u32_t rq_completion_calls; 508 u32_t partially_completed_buf_cnt; /* included in 'buffer_completed_cnt' above */ 509 u32_t buffer_aborted_cnt; 510 u64_t bytes_aborted_cnt; /* cyclic counter of aborted application buffer bytes */ 511 u32_t bytes_trm_aborted_cnt; /* cyclic counter of bytes received with rst ramrod completion */ 512 u32_t fp_db_cnt; /* Fast path doorbell counter - doens't count Adv. Wnd. doorbells*/ 513 u32_t indicate_once_more_cnt; 514 u32_t droped_non_empty_isles; 515 u32_t droped_empty_isles; 516 u32_t rx_post_blocked; 517 u32_t zb_rx_post_blocked; 518 u32_t partially_filled_buf_sent; 519 u32_t abortion_under_flr; 520 } lm_tcp_con_t; 521 522 523 /******************************************************************************* 524 * Slow path request information 525 ******************************************************************************/ 526 /* structure used for storing the data returned by a completion of a slow-path request */ 527 typedef union _lm_tcp_slow_path_ret_data_t 528 { 529 struct { 530 lm_frag_list_t * frag_list; 531 struct _lm_tcp_gen_buf_t * ret_buf_ctx; 532 } tcp_upload_data; 533 } lm_tcp_slow_path_ret_data_t; 534 535 /* structure used for storing the data required for a slow-path request */ 536 typedef struct _lm_tcp_path_relink_cached_t 537 { 538 l4_path_cached_state_t path_cached; 539 l4_neigh_cached_state_t neigh_cached; 540 } lm_tcp_path_relink_cached_t; 541 542 typedef union _lm_tcp_slow_path_sent_data_t { 543 struct { 544 void * data; 545 } tcp_update_data; 546 } lm_tcp_slow_path_sent_data_t ; 547 548 typedef union _lm_tcp_slow_path_phys_data_t 549 { 550 struct toe_context toe_ctx; /* used by query slow path request */ 551 struct toe_update_ramrod_cached_params update_ctx; /* used by update slow path request */ 552 553 } lm_tcp_slow_path_phys_data_t; 554 555 typedef struct _lm_tcp_slow_path_data_t { 556 lm_tcp_slow_path_phys_data_t * virt_addr; 557 lm_address_t phys_addr; 558 }lm_tcp_slow_path_data_t ; 559 560 typedef struct _lm_tcp_slow_path_request_t 561 { 562 lm_sp_req_common_t sp_req_common; 563 lm_tcp_slow_path_ret_data_t ret_data; /* SP req. output data */ 564 lm_tcp_slow_path_sent_data_t sent_data; /* SP req. input data */ 565 566 u32_t type; 567 #define SP_REQUEST_NONE 0 568 #define SP_REQUEST_INITIATE_OFFLOAD 1 569 #define SP_REQUEST_TERMINATE_OFFLOAD 2 570 #define SP_REQUEST_QUERY 3 571 #define SP_REQUEST_UPDATE_TCP 4 572 #define SP_REQUEST_UPDATE_PATH 5 573 #define SP_REQUEST_UPDATE_NEIGH 6 574 #define SP_REQUEST_INVALIDATE 7 575 #define SP_REQUEST_ABORTIVE_DISCONNECT 8 576 #define SP_REQUEST_TERMINATE1_OFFLOAD 9 577 #define SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT 10 /* used only for LOCAL graceful disconnect */ 578 #define SP_REQUEST_PENDING_REMOTE_DISCONNECT 11 /* used for both abortive and graceful disconnect */ 579 #define SP_REQUEST_PENDING_TX_RST 12 /* used for TX Reset received while buffers in the active-tb-list */ 580 #define SP_REQUEST_BLOCKED 13 /* when there is no pending connection, we just want to block sp-command 581 * for example, delay offload */ 582 #define SP_REQUEST_UPDATE_PATH_RELINK 14 583 lm_status_t status; /* request completion status */ 584 } lm_tcp_slow_path_request_t; 585 586 /******************************************************************************* 587 * information required for calculating the TCP state on 'query' 588 * and 'terminate' completions 589 ******************************************************************************/ 590 typedef struct _lm_tcp_state_calculation_t 591 { 592 u64_t fin_request_time; /* written by Tx path, when a fin request is posted to the chip */ 593 u64_t fin_completed_time; /* written by Tx path, when a fin request is completed by the chip */ 594 u64_t fin_reception_time; /* written by Rx path, when a remote fin is received */ 595 u8_t con_rst_flag; /* set whenever chip reports RST reception or RST sent completion */ 596 u8_t con_upld_close_flag; /* set whenever chip reports request to upload a connection after SYN was received or FIN_WAIT2 timer expired */ 597 u8_t _pad[2]; 598 } lm_tcp_state_calculation_t; 599 600 /******************************************************************************* 601 * tcp state 602 ******************************************************************************/ 603 typedef struct _lm_tcp_state_t 604 { 605 lm_state_header_t hdr; 606 lm_path_state_t *path; 607 lm_tcp_con_t *rx_con; 608 lm_tcp_con_t *tx_con; 609 lm_tcp_slow_path_request_t *sp_request; 610 lm_tcp_slow_path_data_t sp_req_data; 611 lm_tcp_state_calculation_t tcp_state_calc; 612 void *ctx_virt; /* Can point to different structures depending on the ulp_type */ 613 lm_address_t ctx_phys; 614 l4_tcp_delegated_state_t tcp_delegated; 615 l4_tcp_const_state_t tcp_const; 616 l4_tcp_cached_state_t tcp_cached; 617 618 u32_t cid; 619 #define TCP_CID_MASK 0xffffff 620 621 /* synchronization between Tx and Rx completions of slow path events */ 622 u16_t sp_flags; 623 #define SP_REQUEST_COMPLETED_RX 0x001 624 #define SP_REQUEST_COMPLETED_TX 0x002 625 #define REMOTE_RST_INDICATED_RX 0x004 626 #define REMOTE_RST_INDICATED_TX 0x008 627 /* mainly for debugging purposes... slow-path indications when there is no fp... */ 628 #define SP_TCP_OFLD_REQ_POSTED 0x010 629 #define SP_TCP_SRC_REQ_POSTED 0x020 630 #define SP_TCP_TRM_REQ_POSTED 0x040 631 #define SP_TCP_QRY_REQ_POSTED 0x080 632 #define SP_TCP_OFLD_REQ_COMP 0x100 633 #define SP_TCP_SRC_REQ_COMP 0x200 634 #define SP_TCP_TRM_REQ_COMP 0x400 635 #define SP_TCP_QRY_REQ_COMP 0x800 636 637 u8_t in_searcher; /* was the tcp state added to searcher hash */ 638 u8_t ulp_type; 639 void * aux_memory; 640 u32_t aux_mem_size; 641 u8_t type_of_aux_memory; 642 #define TCP_CON_AUX_RT_MEM 0x1 643 u8_t aux_mem_flag; 644 #define TCP_CON_AUX_RT_MEM_SUCCSESS_ALLOCATION 0x1 645 #define TCP_CON_AUX_RT_MEM_FAILED_ALLOCATION 0x2 646 647 u8_t sp_request_pending_completion; 648 u8_t pending_abortive_disconnect; 649 650 lm_tcp_integrity_info_t integrity_info; 651 /* GilR 4/3/2006 - TBA - add lm tcp state debug/statistics */ 652 } lm_tcp_state_t; 653 654 /******************************************************************************* 655 * Generic TCP buffer. 656 ******************************************************************************/ 657 typedef struct _lm_tcp_gen_buf_t 658 { 659 d_list_entry_t link; /* MUST be the first field in this structure */ 660 /* generic buffers create a list of generic buffers. The next element is infact a d_list_entry, 661 * however, the generic buffer list is not always accessed as a d_list, it is sometime traversed as 662 * a list ending with NULL */ 663 #define NEXT_GEN_BUF(_gen_buf) (struct _lm_tcp_gen_buf_t *)d_list_next_entry(&((_gen_buf)->link)) 664 #define PREV_GEN_BUF(_gen_buf) (struct _lm_tcp_gen_buf_t *)d_list_prev_entry(&((_gen_buf)->link)) 665 666 lm_address_t buf_phys; 667 lm_tcp_state_t * tcp; /* mainly for updating pending_return_indications */ 668 u8_t * buf_virt; 669 670 /* Following 4 fields are used for supporting SWS accessed when buffer is returned */ 671 u32_t ind_bytes; /* set only in buffer that is head of indication - how many bytes were indicated */ 672 u32_t ind_nbufs; /** how many buffers were included in the indication. Needed for: 673 * - returning buffers to generic pool 674 * - efficiently restore the peninsula list */ 675 /** refcnt required only if we support RcvIndicationSize > 0 */ 676 u16_t refcnt; /* reference count for number of times the buffer was succesfully indicated to um */ 677 u16_t placed_bytes; 678 679 /* The FREE_WHEN_DONE flag indicates that this generic buffer 680 * contains the buffered data received when doing tcp_offload and when it is completed, this 681 * generic buffer is freed back into system memory instead of the generic buffer pool. */ 682 u8_t flags; 683 #define GEN_FLAG_FREE_WHEN_DONE 0x01 684 #define GEN_FLAG_SWS_UPDATE 0x02 /* In certain cases succesfull indication updates the window immediately, however 685 * when we enter a 'safe-mode' we wait for the generic buffers to return before we 686 * update the window. This flag indicates whether or not we have to update. */ 687 688 u16_t phys_offset; /* When allocating gen bufs for buffered data, save the offset 689 from the original phys addr, and use it when when we free the gen buf */ 690 691 } lm_tcp_gen_buf_t; 692 693 694 /******************************************************************************* 695 * generic buffer queue 696 ******************************************************************************/ 697 typedef struct _lm_tcp_grq_t 698 { 699 lm_bd_chain_t bd_chain; 700 701 /* List of posted generic buffers (i.e. attached to the bd chain) */ 702 d_list_t active_gen_list; 703 704 /* List of returned generic buffers, may be used to immediate compensation this grq */ 705 d_list_t aux_gen_list; 706 707 lm_isle_t* isles_pool; 708 /* Flag indicating that the grq needs to be compensated after generic buffers are allocated... */ 709 u8_t grq_compensate_on_alloc; 710 u8_t grq_invloved_in_rss; 711 712 u16_t low_bds_threshold; 713 u16_t high_bds_threshold; 714 715 s16_t number_of_isles_delta; 716 s32_t gen_bufs_in_isles_delta; 717 718 /* statistics */ 719 u16_t max_grqs_per_dpc; /* maximum grqs compensated in dpc */ 720 u16_t num_grqs_last_dpc; 721 u16_t num_deficient; /* number of times compensation wasn't complete */ 722 u16_t avg_grqs_per_dpc; 723 u32_t avg_dpc_cnt; 724 u32_t sum_grqs_last_x_dpcs; 725 u32_t gen_bufs_compensated_from_bypass_only; 726 u32_t gen_bufs_compensated_till_low_threshold; 727 u32_t gen_bufs_collected_to_later_compensation; 728 } lm_tcp_grq_t; 729 730 /******************************************************************************* 731 * L4 receive completion queue 732 ******************************************************************************/ 733 typedef struct _lm_tcp_rcq_t 734 { 735 lm_bd_chain_t bd_chain; 736 737 /* points directly to the TOE Rx index in the USTORM part 738 * of the non-default status block */ 739 u16_t volatile *hw_con_idx_ptr; 740 741 /* for RSS indirection table update synchronization */ 742 u8_t rss_update_pending; /* unused */ 743 u8_t suspend_processing; 744 u32_t update_cid; 745 u32_t rss_update_stats_quiet; 746 u32_t rss_update_stats_sleeping; 747 u32_t rss_update_stats_delayed; 748 u32_t rss_update_processing_delayed; 749 u32_t rss_update_processing_continued; 750 u32_t rss_update_processing_max_continued; 751 752 /* statistics */ 753 u16_t max_cqes_per_dpc; 754 u16_t num_cqes_last_dpc; 755 u16_t avg_cqes_per_dpc; 756 u16_t _pad16; 757 u32_t avg_dpc_cnt; 758 u32_t sum_cqes_last_x_dpcs; 759 760 lm_hc_sb_info_t hc_sb_info; 761 762 } lm_tcp_rcq_t; 763 764 /******************************************************************************* 765 * L4 send completion queue 766 ******************************************************************************/ 767 typedef struct _lm_tcp_scq_t 768 { 769 lm_bd_chain_t bd_chain; 770 771 /* points directly to the TOE Tx index in the CSTORM part 772 * of the non-default status block */ 773 u16_t volatile *hw_con_idx_ptr; 774 775 /* statistics */ 776 u16_t max_cqes_per_dpc; 777 u16_t num_cqes_last_dpc; 778 u16_t avg_cqes_per_dpc; 779 u16_t _pad16; 780 u32_t avg_dpc_cnt; 781 u32_t sum_cqes_last_x_dpcs; 782 783 lm_hc_sb_info_t hc_sb_info; 784 785 } lm_tcp_scq_t; 786 787 /******************************************************************************* 788 * states block - includes all offloaded states and possibly other offload 789 * information of a specific client. 790 ******************************************************************************/ 791 typedef struct _lm_state_block_t 792 { 793 d_list_t tcp_list; 794 d_list_t path_list; 795 d_list_t neigh_list; 796 } lm_state_block_t; 797 798 799 typedef struct _lm_toe_statistics_t 800 { 801 u32_t total_ofld; /* cyclic counter of number of offloaded tcp states */ 802 u32_t total_upld; /* cyclic counter of number of uploaded tcp states */ 803 s32_t total_indicated; /* cyclic counter of number of generic indications (sum of connections pending...) */ 804 s32_t total_indicated_returned; /* cyclic counter of number of generic indications that have returned */ 805 806 /* aggregative per-connections statistics */ 807 u32_t rx_rq_complete_calls; /* #RQ completion calls (total, copy + zero copy) */ 808 u32_t rx_rq_bufs_completed; /* #RQ completion buffers */ 809 u64_t rx_bytes_completed_total; /* #RQ completion bytes */ 810 811 u32_t rx_accepted_indications; /* #GRQ completion calls (indicate) */ 812 u32_t rx_bufs_indicated_accepted; /* #GRQ completion buffers */ 813 u64_t rx_bytes_indicated_accepted; /* #GRQ completion bytes */ 814 815 u32_t rx_rejected_indications; /* #failed or partially consumed indicate calls */ 816 u32_t rx_bufs_indicated_rejected; /* #GRQ completion bytes */ 817 u64_t rx_bytes_indicated_rejected; /* #GRQ completion bytes */ 818 819 u32_t rx_zero_byte_recv_reqs; /* #Zero byte receeive requests */ 820 u32_t rx_bufs_copied_grq; /* #VBD copy bufs total */ 821 u32_t rx_bufs_copied_rq; /* #VBD copy bufs total */ 822 u32_t _pad32_1; 823 u64_t rx_bytes_copied_in_post; /* #VBD copy bytes in post phase*/ 824 u64_t rx_bytes_copied_in_comp; /* #VBD copy bytes in completion phase */ 825 u64_t rx_bytes_copied_in_process; /* #VBD copy bytes in process phase */ 826 827 /* post */ 828 u32_t rx_bufs_posted_total; 829 u32_t rx_bufs_skipped_post; 830 u64_t rx_bytes_skipped_post; 831 u64_t rx_bytes_posted_total; 832 833 /* push related */ 834 u64_t rx_bytes_skipped_push; 835 u32_t rx_partially_completed_buf_cnt; 836 837 /* abort */ 838 u32_t rx_buffer_aborted_cnt; 839 840 u32_t tx_rq_complete_calls; 841 u32_t tx_rq_bufs_completed; 842 u64_t tx_bytes_posted_total; 843 u64_t tx_bytes_completed_total; 844 845 u32_t total_dbg_upld_requested; 846 u32_t total_fin_upld_requested; 847 u32_t total_rst_upld_requested; 848 u32_t total_close_upld_requested; 849 u32_t total_dbt_upld_requested; 850 u32_t total_opt_upld_requested; 851 u32_t total_big_isle_upld_requesed; 852 u32_t total_many_isles_upld_requesed; 853 u32_t total_upld_requested[L4_UPLOAD_REASON_MAX]; 854 u32_t con_state_on_upload[L4_TCP_CON_STATE_MAX]; 855 u32_t total_bytes_lost_on_upload; 856 u32_t total_droped_non_empty_isles; 857 u32_t total_droped_empty_isles; 858 u32_t total_rx_post_blocked; 859 u32_t total_zb_rx_post_blocked; 860 u32_t total_cfc_delete_error; 861 u32_t total_num_non_full_indications; 862 u32_t total_aux_mem_success_allocations; 863 u32_t total_aux_mem_failed_allocations; 864 u32_t total_rx_abortion_under_flr; 865 u32_t total_tx_abortion_under_flr; 866 u32_t max_number_of_isles_in_single_con; 867 u32_t total_aborive_disconnect_during_completion; 868 u32_t total_pending_aborive_disconnect_completed; 869 u32_t total_aborive_disconnect_completed; 870 871 u64_t total_buffered_data; 872 } lm_toe_statistics_t; 873 874 typedef struct _lm_toe_isles_t 875 { 876 s32_t gen_bufs_in_isles; 877 s32_t max_gen_bufs_in_isles; 878 s16_t number_of_isles; 879 s16_t max_number_of_isles; 880 u8_t l4_decrease_archipelago; 881 u8_t __pad[3]; 882 } lm_toe_isles_t; 883 884 /******************************************************************************* 885 * toe info - all TOE (L4) information/data structures of the lm_device 886 ******************************************************************************/ 887 typedef struct _lm_toe_info_t 888 { 889 struct _lm_device_t *pdev; 890 lm_state_block_t state_blk; 891 892 lm_toe_statistics_t stats; 893 lm_toe_isles_t archipelago; 894 895 lm_tcp_scq_t scqs[MAX_L4_TX_CHAIN]; 896 lm_tcp_rcq_t rcqs[MAX_L4_RX_CHAIN]; 897 lm_tcp_grq_t grqs[MAX_L4_RX_CHAIN]; 898 899 u8_t indirection_table[TOE_INDIRECTION_TABLE_SIZE]; 900 u32_t rss_update_cnt; /* GilR 4/4/2006 - TBD on RSS indirection table update implementation */ 901 u32_t gen_buf_size; /* The size of a generic buffer based on gen_buf_min_size and mtu */ 902 903 u8_t state; 904 #define LM_TOE_STATE_NONE 0 905 #define LM_TOE_STATE_INIT 1 906 #define LM_TOE_STATE_NORMAL 2 907 908 /* Once a generic indication succeeded and the buffers are given to the client we have to choose whether we want 909 * to give a window-update immediately (short-loop) or wait for the buffer to return (long-loop). The mode is determined 910 * by a set of rules in the UM related to the generic buffer pool and its state. The UM sets this parameter for the lm, 911 * and at each indication the lm checks which mode it is in, marks the generic buffer and gives a window-update accordingly */ 912 u8_t update_window_mode; 913 #define LM_TOE_UPDATE_MODE_LONG_LOOP 0 914 #define LM_TOE_UPDATE_MODE_SHORT_LOOP 1 915 916 /* This field is used to indicate that certain events have occured in TOE. Should be updated under TOE-LOCK */ 917 u8_t toe_events; 918 #define LM_TOE_EVENT_WINDOW_DECREASE 0x1 919 u8_t __pad[1]; 920 921 lm_toe_integrity_info_t integrity_info; 922 923 /* Slow-path data for toe-rss (common and not per connection, therefore located here! ) */ 924 struct toe_rss_update_ramrod_data * rss_update_data; 925 lm_address_t rss_update_data_phys; 926 } lm_toe_info_t; 927 928 929 #endif /* _LM_L4ST_H */ 930