1 #include "lm5710.h" 2 #include "command.h" 3 #include "bd_chain.h" 4 #include "ecore_common.h" 5 #include "mm.h" 6 7 #define OOO_CID_USTRORM_PROD_DIFF (0x4000) 8 9 u8_t lm_is_rx_completion(lm_device_t *pdev, u8_t chain_idx) 10 { 11 u8_t result = FALSE; 12 lm_rcq_chain_t *rcq_chain = &LM_RCQ(pdev, chain_idx); 13 14 DbgBreakIf(!(pdev && rcq_chain)); 15 16 //the hw_con_idx_ptr of the rcq_chain points directly to the Rx index in the USTORM part of the non-default status block 17 if (rcq_chain->hw_con_idx_ptr && 18 (mm_le16_to_cpu(*rcq_chain->hw_con_idx_ptr) != 19 lm_bd_chain_cons_idx(&rcq_chain->bd_chain))) 20 { 21 result = TRUE; 22 } 23 DbgMessage(pdev, INFORMi, "lm_is_rx_completion: result is:%s\n", result? "TRUE" : "FALSE"); 24 25 return result; 26 } 27 28 /******************************************************************************* 29 * Description: 30 * set both rcq, rx bd and rx sge (if valid) prods 31 * Return: 32 ******************************************************************************/ 33 static void FORCEINLINE lm_rx_set_prods( lm_device_t *pdev, 34 u16_t const iro_prod_offset, 35 lm_bd_chain_t *rcq_chain_bd, 36 lm_bd_chain_t *rx_chain_bd, 37 lm_bd_chain_t *rx_chain_sge, 38 const u32_t chain_idx ) 39 { 40 lm_rx_chain_t* rxq_chain = &LM_RXQ(pdev, chain_idx); 41 u32_t val32 = 0; 42 u64_t val64 = 0; 43 u16_t val16_lo = lm_bd_chain_prod_idx(rcq_chain_bd); 44 u16_t val16_hi = lm_bd_chain_prod_idx(rx_chain_bd); 45 u32_t const ustorm_bar_offset = (IS_CHANNEL_VFDEV(pdev)) ? VF_BAR0_USDM_QUEUES_OFFSET: BAR_USTRORM_INTMEM ; 46 47 if(OOO_CID(pdev) == chain_idx) 48 { 49 DbgBreakIfFastPath( NULL != rx_chain_sge ); 50 DbgBreakIfFastPath(IS_CHANNEL_VFDEV(pdev)); 51 52 LM_INTMEM_WRITE16(PFDEV(pdev), 53 TSTORM_ISCSI_L2_ISCSI_OOO_PROD_OFFSET(FUNC_ID(pdev)), 54 rxq_chain->common.bd_prod_without_next, 55 BAR_TSTRORM_INTMEM); 56 57 // Ugly FW solution OOO FW wants the 58 val16_lo += OOO_CID_USTRORM_PROD_DIFF; 59 val16_hi += OOO_CID_USTRORM_PROD_DIFF; 60 } 61 62 val32 = ((u32_t)(val16_hi << 16) | val16_lo); 63 64 //notify the fw of the prod of the RCQ. No need to do that for the Rx bd chain. 65 if( rx_chain_sge ) 66 { 67 val64 = (((u64_t)lm_bd_chain_prod_idx(rx_chain_sge))<<32) | val32 ; 68 69 LM_INTMEM_WRITE64(PFDEV(pdev), 70 iro_prod_offset, 71 val64, 72 ustorm_bar_offset); 73 } 74 else 75 { 76 LM_INTMEM_WRITE32(PFDEV(pdev), 77 iro_prod_offset, 78 val32, 79 ustorm_bar_offset); 80 } 81 } 82 /******************************************************************************* 83 * Description: 84 * rx_chain_bd always valid, rx_chain_sge valid only in case we are LAH enabled in this queue 85 * all if() checking will be always done on rx_chain_bd since he is always valid and sge should be consistent 86 * We verify it in case sge is valid 87 * all bd_xxx operations will be done on both 88 * Return: 89 ******************************************************************************/ 90 u32_t 91 lm_post_buffers( 92 lm_device_t *pdev, 93 u32_t chain_idx, 94 lm_packet_t *packet,/* optional. */ 95 u8_t const is_tpa) 96 { 97 lm_rx_chain_common_t* rxq_chain_common = NULL; 98 lm_bd_chain_t* rx_chain_bd = NULL; 99 lm_rx_chain_t* rxq_chain = NULL; 100 lm_tpa_chain_t * tpa_chain = NULL; 101 lm_bd_chain_t* bd_chain_to_check = NULL; 102 lm_rcq_chain_t* rcq_chain = &LM_RCQ(pdev, chain_idx); 103 lm_bd_chain_t* rx_chain_sge = NULL; 104 u32_t pkt_queued = 0; 105 struct eth_rx_bd* cur_bd = NULL; 106 struct eth_rx_sge* cur_sge = NULL; 107 u32_t prod_bseq = 0; 108 u32_t rcq_prod_bseq = 0; 109 u16_t current_prod = 0; 110 u16_t active_entry = 0; 111 112 DbgMessage(pdev, INFORMl2 , "### lm_post_buffers\n"); 113 114 // Verify BD's consistent 115 DbgBreakIfFastPath( rx_chain_sge && !lm_bd_chains_are_consistent( rx_chain_sge, rx_chain_bd ) ); 116 117 if(FALSE == is_tpa) 118 { 119 rxq_chain_common = &LM_RXQ_COMMON(pdev, chain_idx); 120 rx_chain_bd = &LM_RXQ_CHAIN_BD(pdev, chain_idx); 121 rx_chain_sge = LM_RXQ_SGE_PTR_IF_VALID(pdev, chain_idx); 122 rxq_chain = &LM_RXQ(pdev, chain_idx); 123 tpa_chain = NULL; 124 /* the assumption is that the number of cqes is less or equal to the corresponding rx bds, 125 therefore if there no cqes left, break */ 126 bd_chain_to_check = &rcq_chain->bd_chain; 127 } 128 else 129 { 130 rxq_chain_common = &LM_TPA_COMMON(pdev, chain_idx); 131 rx_chain_bd = &LM_TPA_CHAIN_BD(pdev, chain_idx); 132 rx_chain_sge = NULL; 133 rxq_chain = NULL; 134 tpa_chain = &LM_TPA(pdev, chain_idx); 135 // In TPA we don't add to the RCQ when posting buffers 136 bd_chain_to_check = rx_chain_bd; 137 } 138 /* Make sure we have a bd left for posting a receive buffer. */ 139 if(packet) 140 { 141 // Insert given packet. 142 DbgBreakIfFastPath(SIG(packet) != L2PACKET_RX_SIG); 143 144 if(lm_bd_chain_is_empty(bd_chain_to_check)) 145 { 146 s_list_push_tail(&rxq_chain_common->free_descq, &packet->link); 147 packet = NULL; 148 } 149 } 150 else if(!lm_bd_chain_is_empty(bd_chain_to_check)) 151 { 152 packet = (lm_packet_t *) s_list_pop_head(&rxq_chain_common->free_descq); 153 } 154 prod_bseq = rxq_chain_common->prod_bseq; 155 156 // In TPA we won't increment rcq_prod_bseq 157 rcq_prod_bseq = rcq_chain->prod_bseq; 158 159 while(packet) 160 { 161 162 current_prod = lm_bd_chain_prod_idx(rx_chain_bd); 163 cur_bd = lm_bd_chain_produce_bd(rx_chain_bd); 164 rxq_chain_common->bd_prod_without_next++; 165 cur_sge = rx_chain_sge ? lm_bd_chain_produce_bd(rx_chain_sge) : NULL; 166 167 prod_bseq += packet->l2pkt_rx_info->mem_size; 168 169 if(FALSE == is_tpa) 170 { 171 //take care of the RCQ related prod stuff. 172 173 //update the prod of the RCQ only AFTER the Rx bd! 174 rcq_prod_bseq += packet->l2pkt_rx_info->mem_size; 175 176 /* These were actually produced before by fw, but we only produce them now to make sure they're synced with the rx-chain */ 177 lm_bd_chain_bd_produced(&rcq_chain->bd_chain); 178 } 179 180 packet->u1.rx.next_bd_idx = lm_bd_chain_prod_idx(rx_chain_bd); 181 #if L2_RX_BUF_SIG 182 /* make sure signitures exist before and after the buffer */ 183 DbgBreakIfFastPath(SIG(packet->u1.rx.mem_virt - pdev->params.rcv_buffer_offset) != L2PACKET_RX_SIG); 184 DbgBreakIfFastPath(END_SIG(packet->u1.rx.mem_virt, MAX_L2_CLI_BUFFER_SIZE(pdev, chain_idx)) != L2PACKET_RX_SIG); 185 #endif /* L2_RX_BUF_SIG */ 186 187 cur_bd->addr_lo = mm_cpu_to_le32(packet->u1.rx.mem_phys[0].as_u32.low); 188 cur_bd->addr_hi = mm_cpu_to_le32(packet->u1.rx.mem_phys[0].as_u32.high); 189 190 if( cur_sge ) 191 { 192 cur_sge->addr_lo = mm_cpu_to_le32(packet->u1.rx.mem_phys[1].as_u32.low); 193 cur_sge->addr_hi = mm_cpu_to_le32(packet->u1.rx.mem_phys[1].as_u32.high); 194 } 195 196 pkt_queued++; 197 198 if(FALSE == is_tpa) 199 { 200 s_list_push_tail(&rxq_chain->active_descq, &packet->link); 201 } 202 else 203 { 204 // Active descriptor must sit in the same entry 205 active_entry = LM_TPA_BD_ENTRY_TO_ACTIVE_ENTRY(pdev, chain_idx, current_prod); 206 207 LM_TPA_ACTIVE_ENTRY_BOUNDARIES_VERIFY(pdev, chain_idx,active_entry); 208 tpa_chain->sge_chain.active_descq_array[active_entry] = packet; 209 } 210 211 if(lm_bd_chain_is_empty(bd_chain_to_check)) 212 { 213 break; 214 } 215 216 /* Make sure we have a bd left for posting a receive buffer. */ 217 packet = (lm_packet_t *) s_list_pop_head(&rxq_chain_common->free_descq); 218 } 219 220 rxq_chain_common->prod_bseq = prod_bseq; 221 222 223 //update the prod of the RCQ only AFTER the Rx bd! 224 // This code seems unnecessary maybe should be deleted. 225 // Im TPA we won't increment rcq_prod_bseq 226 rcq_chain->prod_bseq = rcq_prod_bseq; 227 228 if(pkt_queued) 229 { 230 //notify the fw of the prod 231 if(FALSE == is_tpa) 232 { 233 lm_rx_set_prods(pdev, rcq_chain->iro_prod_offset, &rcq_chain->bd_chain, rx_chain_bd, rx_chain_sge ,chain_idx); 234 } 235 else 236 { 237 lm_rx_set_prods(pdev, rcq_chain->iro_prod_offset, &rcq_chain->bd_chain, &LM_RXQ_CHAIN_BD(pdev, chain_idx), &LM_TPA_CHAIN_BD(pdev, chain_idx) ,chain_idx); 238 } 239 } 240 241 DbgMessage(pdev, INFORMl2 , "lm_post_buffers - bd con: %d bd prod: %d \n", 242 lm_bd_chain_cons_idx(rx_chain_bd),lm_bd_chain_prod_idx(rx_chain_bd)); 243 DbgMessage(pdev, INFORMl2 , "lm_post_buffers - cq con: %d cq prod: %d \n", 244 lm_bd_chain_cons_idx(&rcq_chain->bd_chain) ,lm_bd_chain_prod_idx(&rcq_chain->bd_chain)); 245 246 return pkt_queued; 247 } /* lm_post_buffers */ 248 249 /** 250 * @description 251 * Updates tpa_chain->last_max_cons_sge if there is a new max. 252 * Basic assumption is that is BD prod is always higher that BD 253 * cons. 254 * The minus will tell us who is closer to BD prod. 255 * @param pdev 256 * @param chain_idx 257 * @param new_index 258 * 259 * @return STATIC void 260 */ 261 __inline STATIC void 262 lm_tpa_sge_update_last_max(IN lm_device_t* pdev, 263 IN const u32_t chain_idx, 264 IN const u16_t new_index) 265 { 266 lm_tpa_sge_chain_t* sge_tpa_chain = &LM_SGE_TPA_CHAIN(pdev, chain_idx); 267 u16_t const prod_idx = lm_bd_chain_prod_idx(&LM_TPA_CHAIN_BD(pdev, chain_idx)); 268 u16_t const prod_minus_new_sge = prod_idx - new_index; 269 u16_t const prod_minus_saved = prod_idx - sge_tpa_chain->last_max_con; 270 271 if(prod_minus_new_sge < prod_minus_saved) 272 { 273 sge_tpa_chain->last_max_con = new_index; 274 } 275 276 /* 277 Cyclic would have been a nicer sulotion, but adds a limitation on bd ring size that would be (2^15) instead of 2^16 278 This limitation should be closed done when allocating the TPA BD chain 279 DbgBreakIf(LM_TPA_CHAIN_BD_NUM_ELEM(_pdev, chain_idx) < (2^15) ); 280 if (CYCLIC_GT_16(sge_index, sge_tpa_chain->last_max_con)) 281 sge_tpa_chain->last_max_con = sge_index; 282 */ 283 } 284 285 /** 286 * @description 287 * The TPA sge consumer will be increments in 64 bit 288 * resolutions. 289 * @param pdev 290 * @param chain_idx 291 * 292 * @return STATIC u32_t 293 */ 294 __inline STATIC void 295 lm_tpa_incr_sge_cons( IN lm_device_t* pdev, 296 IN const u32_t chain_idx, 297 IN const u16_t mask_entry_idx) 298 { 299 lm_tpa_sge_chain_t* sge_tpa_chain = &LM_SGE_TPA_CHAIN(pdev, chain_idx); 300 lm_bd_chain_t* bd_chain = &LM_TPA_CHAIN_BD(pdev, chain_idx); 301 u16_t bd_entry = 0; 302 u16_t active_entry = 0; 303 u16_t i = 0; 304 305 bd_chain->cons_idx += BIT_VEC64_ELEM_SZ; 306 307 DbgBreakIf(LM_TPA_MASK_LEN(pdev, chain_idx) <= mask_entry_idx); 308 sge_tpa_chain->mask_array[mask_entry_idx] = BIT_VEC64_ELEM_ONE_MASK; 309 310 // Make sure bds_per_page_mask is a power of 2 that is higher than 64 311 DbgBreakIf(0 != (lm_bd_chain_bds_per_page(bd_chain) & BIT_VEC64_ELEM_MASK)); 312 DbgBreakIf(BIT_VEC64_ELEM_SZ >= lm_bd_chain_bds_per_page(bd_chain)); 313 314 if((lm_bd_chain_cons_idx(bd_chain) & lm_bd_chain_bds_per_page_mask(bd_chain)) == 0) 315 { 316 // Just closed a page must refer to page end entries 317 lm_bd_chain_bds_consumed(bd_chain, (BIT_VEC64_ELEM_SZ - lm_bd_chain_bds_skip_eop(bd_chain))); 318 319 /* clear page-end entries */ 320 for(i = 1; i <= lm_bd_chain_bds_skip_eop(bd_chain); i++ ) 321 { 322 bd_entry = lm_bd_chain_cons_idx(bd_chain) - i; 323 active_entry = LM_TPA_BD_ENTRY_TO_ACTIVE_ENTRY(pdev, chain_idx, bd_entry); 324 LM_TPA_MASK_CLEAR_ACTIVE_BIT(pdev, chain_idx, active_entry); 325 } 326 } 327 else 328 { 329 // Same page 330 lm_bd_chain_bds_consumed(bd_chain, BIT_VEC64_ELEM_SZ); 331 } 332 } 333 /** 334 * @description 335 * Handle TPA stop code. 336 * @param pdev 337 * @param rcvd_list -Global receive list 338 * @param cqe 339 * @param chain_idx 340 * @param pkt_cnt 341 * @param queue_index 342 * 343 * @return STATIC u32_t pkt_cnt number of packets. The number is 344 * an input parameter and packets add to the global list 345 * are add. 346 */ 347 STATIC u32_t 348 lm_tpa_stop( IN lm_device_t* pdev, 349 INOUT s_list_t* rcvd_list, 350 IN const struct eth_end_agg_rx_cqe* cqe, 351 IN const u32_t chain_idx, 352 IN u32_t pkt_cnt, 353 IN const u8_t queue_index) 354 { 355 lm_tpa_chain_t* tpa_chain = &LM_TPA(pdev, chain_idx); 356 lm_tpa_sge_chain_t* sge_tpa_chain = &LM_SGE_TPA_CHAIN(pdev, chain_idx); 357 lm_bd_chain_t* bd_chain = &LM_TPA_CHAIN_BD(pdev, chain_idx); 358 lm_packet_t* pkt = tpa_chain->start_coales_bd[queue_index].packet;//Reads the TPA start coalesce array(PD_R) 359 u32_t sge_size = mm_le16_to_cpu(cqe->pkt_len) - pkt->l2pkt_rx_info->size; 360 u32_t const sge_num_elem = DIV_ROUND_UP_BITS(sge_size, LM_TPA_PAGE_BITS); 361 u32_t fw_sge_index = 0; 362 u16_t active_entry = 0; 363 u16_t first_max_set = 0; 364 u16_t last_max_set = 0; 365 u16_t i = 0; 366 u8_t b_force_first_enter = FALSE; 367 u16_t loop_cnt_dbg = 0; 368 const u32_t lm_tpa_page_size = LM_TPA_PAGE_SIZE; 369 370 // Total packet size given in end aggregation must be larger than the size given in start aggregation. 371 // The only case that the both size are equal is if stop aggregation doesn't contain data. 372 DbgBreakIf( mm_le16_to_cpu(cqe->pkt_len) < pkt->l2pkt_rx_info->size); 373 374 DbgBreakIf( TRUE != tpa_chain->start_coales_bd[queue_index].is_entry_used); 375 tpa_chain->start_coales_bd[queue_index].is_entry_used = FALSE; 376 377 // Indicate to upper layer this is a TPA packet 378 SET_FLAGS(pkt->l2pkt_rx_info->flags ,LM_RX_FLAG_START_RSC_TPA); 379 // Updates the TPA only fields from the CQE 380 pkt->l2pkt_rx_info->total_packet_size = mm_le16_to_cpu(cqe->pkt_len); 381 pkt->l2pkt_rx_info->coal_seg_cnt = mm_le16_to_cpu(cqe->num_of_coalesced_segs); 382 pkt->l2pkt_rx_info->dup_ack_cnt = cqe->pure_ack_count; 383 pkt->l2pkt_rx_info->ts_delta = mm_le32_to_cpu(cqe->timestamp_delta); 384 385 /* make sure packet size is larger than header size */ 386 DbgBreakIfFastPath(pkt->l2pkt_rx_info->total_packet_size < MIN_ETHERNET_PACKET_SIZE); 387 388 // Adds this packet descriptor to the global receive list (rcvd_list that is later indicated to miniport). 389 s_list_push_tail(rcvd_list, &pkt->link); 390 pkt_cnt++; 391 392 ASSERT_STATIC(LM_TPA_MAX_AGG_SIZE == ARRSIZE(cqe->sgl_or_raw_data.sgl)); 393 DbgBreakIf(ARRSIZE(cqe->sgl_or_raw_data.sgl) < sge_num_elem); 394 395 // If the TPA stop doesn't contain any new BDs. 396 if(0 == sge_num_elem ) 397 { 398 // Total packet size given in end aggregation must be equal to the size given in start aggregation. 399 // if stop aggregation doesn't contain data. 400 DbgBreakIf( mm_le16_to_cpu(cqe->pkt_len) != pkt->l2pkt_rx_info->size); 401 402 return pkt_cnt; 403 } 404 405 for(fw_sge_index = 0; fw_sge_index < sge_num_elem; fw_sge_index++) 406 { 407 DbgBreakIf(ARRSIZE(cqe->sgl_or_raw_data.sgl) <= fw_sge_index); 408 active_entry = LM_TPA_BD_ENTRY_TO_ACTIVE_ENTRY(pdev, chain_idx, mm_le16_to_cpu(cqe->sgl_or_raw_data.sgl[fw_sge_index])); 409 410 LM_TPA_ACTIVE_ENTRY_BOUNDARIES_VERIFY(pdev, chain_idx, active_entry); 411 pkt = tpa_chain->sge_chain.active_descq_array[active_entry]; 412 LM_TPA_MASK_CLEAR_ACTIVE_BIT(pdev, chain_idx, active_entry); 413 414 #if (DBG) 415 /************start TPA debbug code******************************/ 416 tpa_chain->dbg_params.pck_ret_from_chip++; 417 /************end TPA debbug code******************************/ 418 #endif //(DBG) 419 // For last SGE 420 DbgBreakIf((fw_sge_index != (sge_num_elem - 1)) && (sge_size < LM_TPA_PAGE_SIZE )); 421 pkt->l2pkt_rx_info->size = min(sge_size ,lm_tpa_page_size); 422 s_list_push_tail(rcvd_list, &(pkt->link)); 423 pkt_cnt++; 424 sge_size -= LM_TPA_PAGE_SIZE; 425 } 426 427 #if defined(_NTDDK_) 428 //PreFast 28182 :Prefast reviewed and suppress this situation shouldn't occur. 429 #pragma warning (push) 430 #pragma warning( disable:6385 ) 431 #endif // !_NTDDK_ 432 /* Here we assume that the last SGE index is the biggest */ 433 lm_tpa_sge_update_last_max(pdev, 434 chain_idx, 435 mm_le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_num_elem -1])); 436 437 #if defined(_NTDDK_) 438 #pragma warning (pop) 439 #endif // !_NTDDK_ 440 // Find the first cosumer that is a candidate to free and the last. 441 first_max_set = LM_TPA_BD_ENTRY_TO_MASK_ENTRY(pdev, chain_idx, lm_bd_chain_cons_idx(bd_chain)); 442 last_max_set = LM_TPA_BD_ENTRY_TO_MASK_ENTRY(pdev, chain_idx, sge_tpa_chain->last_max_con); 443 444 DbgBreakIf(0 != (lm_bd_chain_cons_idx(bd_chain) & BIT_VEC64_ELEM_MASK)); 445 /* If ring is full enter anyway*/ 446 if((last_max_set == first_max_set) && (lm_bd_chain_is_full(bd_chain))) 447 { 448 b_force_first_enter = TRUE; 449 } 450 /* Now update the cons */ 451 for (i = first_max_set;((i != last_max_set) || (TRUE == b_force_first_enter)); i = LM_TPA_MASK_NEXT_ELEM(pdev, chain_idx, i)) 452 { 453 DbgBreakIf(LM_TPA_MASK_LEN(pdev, chain_idx) <= i); 454 if (sge_tpa_chain->mask_array[i]) 455 { 456 break; 457 } 458 b_force_first_enter = FALSE; 459 460 lm_tpa_incr_sge_cons(pdev, 461 chain_idx, 462 i); 463 loop_cnt_dbg++; 464 DbgBreakIf(LM_TPA_MASK_LEN(pdev,chain_idx) < loop_cnt_dbg); 465 } 466 467 return pkt_cnt; 468 } 469 /** 470 * @description 471 * Handle TPA start code. 472 * @param pdev 473 * @param pkt 474 * @param chain_idx 475 * @param queue_index 476 * 477 * @return STATIC void 478 */ 479 __inline STATIC void 480 lm_tpa_start( IN lm_device_t* pdev, 481 IN lm_packet_t* pkt, 482 IN const u32_t chain_idx, 483 IN const u8_t queue_index) 484 { 485 lm_tpa_chain_t* tpa_chain = &LM_TPA(pdev, chain_idx); 486 487 DbgBreakIf( FALSE != tpa_chain->start_coales_bd[queue_index].is_entry_used); 488 489 tpa_chain->start_coales_bd[queue_index].is_entry_used = TRUE; 490 tpa_chain->start_coales_bd[queue_index].packet = pkt; 491 } 492 /** 493 * @description 494 * Set TPA start known flags. 495 * This is only an optimization to avoid known if's 496 * @param pdev 497 * 498 * @return STATIC void 499 */ 500 __inline STATIC void 501 lm_tpa_start_flags_handle( IN lm_device_t* pdev, 502 IN const struct eth_fast_path_rx_cqe* cqe, 503 INOUT lm_packet_t* pkt, 504 IN const u16_t parse_flags) 505 { 506 // TPA is always(only) above IPV4 or IPV6. 507 DbgBreakIf(FALSE == 508 ((GET_FLAGS_WITH_OFFSET(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL, 509 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT) == PRS_FLAG_OVERETH_IPV4) || 510 (GET_FLAGS_WITH_OFFSET(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL, 511 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT) == PRS_FLAG_OVERETH_IPV6))); 512 513 if(PRS_FLAG_OVERETH_IPV4 == GET_FLAGS_WITH_OFFSET(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL, 514 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT)) 515 { 516 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IS_IPV4_DATAGRAM); 517 518 DbgBreakIf(GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_IP_XSUM_NO_VALIDATION_FLG)); 519 // In IPV4 there is always a checksum 520 // TPA ip cksum is always valid 521 DbgBreakIf(GET_FLAGS(cqe->type_error_flags, ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG)); 522 523 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IP_CKSUM_IS_GOOD); 524 } 525 else 526 { 527 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IS_IPV6_DATAGRAM); 528 // In IPV6 there is no checksum 529 DbgBreakIf(0 == GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_IP_XSUM_NO_VALIDATION_FLG)); 530 } 531 532 533 // If there was a fagmentation it will be delivered by a regular BD (the TPA aggregation is stoped). 534 DbgBreakIf( GET_FLAGS(parse_flags,PARSING_FLAGS_FRAGMENTATION_STATUS)); 535 /* check if TCP segment */ 536 // TPA is always above TCP. 537 DbgBreakIf(PRS_FLAG_OVERIP_TCP != GET_FLAGS_WITH_OFFSET(parse_flags,PARSING_FLAGS_OVER_IP_PROTOCOL, 538 PARSING_FLAGS_OVER_IP_PROTOCOL_SHIFT)); 539 540 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IS_TCP_SEGMENT); 541 542 543 // TCP was checked before. TCP checksum must be done by FW in TPA. 544 DbgBreakIf(GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)); 545 // TCP checksum must be valid in a successful TPA aggregation. 546 DbgBreakIf(GET_FLAGS(cqe->type_error_flags, ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG)); 547 548 /* IN TPA tcp cksum is always validated */ 549 /* valid tcp/udp cksum */ 550 #define SHIFT_IS_GOOD 1 551 #define SHIFT_IS_BAD 2 552 ASSERT_STATIC(LM_RX_FLAG_UDP_CKSUM_IS_GOOD == LM_RX_FLAG_IS_UDP_DATAGRAM << SHIFT_IS_GOOD); 553 ASSERT_STATIC(LM_RX_FLAG_UDP_CKSUM_IS_BAD == LM_RX_FLAG_IS_UDP_DATAGRAM << SHIFT_IS_BAD); 554 ASSERT_STATIC(LM_RX_FLAG_TCP_CKSUM_IS_GOOD == LM_RX_FLAG_IS_TCP_SEGMENT << SHIFT_IS_GOOD); 555 ASSERT_STATIC(LM_RX_FLAG_TCP_CKSUM_IS_BAD == LM_RX_FLAG_IS_TCP_SEGMENT << SHIFT_IS_BAD); 556 557 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT)) << SHIFT_IS_GOOD ) ); 558 } 559 560 /** 561 * @description 562 * Set regular flags. 563 * This is only an optimization 564 * @param pdev 565 * 566 * @return STATIC void 567 */ 568 STATIC void 569 lm_regular_flags_handle( IN lm_device_t* pdev, 570 IN const struct eth_fast_path_rx_cqe* cqe, 571 INOUT lm_packet_t* pkt, 572 IN const u16_t parse_flags) 573 { 574 /* check if IP datagram (either IPv4 or IPv6) */ 575 if(((GET_FLAGS(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) >> 576 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT) == PRS_FLAG_OVERETH_IPV4) || 577 ((GET_FLAGS(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) >> 578 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT) == PRS_FLAG_OVERETH_IPV6)) 579 { 580 pkt->l2pkt_rx_info->flags |= 581 (GET_FLAGS(parse_flags,PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) >> 582 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL_SHIFT) == PRS_FLAG_OVERETH_IPV4 ? 583 LM_RX_FLAG_IS_IPV4_DATAGRAM : 584 LM_RX_FLAG_IS_IPV6_DATAGRAM; 585 if(!GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_IP_XSUM_NO_VALIDATION_FLG)) 586 { 587 /* ip cksum validated */ 588 if GET_FLAGS(cqe->type_error_flags, ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG) 589 { 590 /* invalid ip cksum */ 591 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IP_CKSUM_IS_BAD); 592 593 LM_COMMON_DRV_STATS_ATOMIC_INC_ETH(pdev, rx_ip_cs_error_count); 594 } 595 else 596 { 597 /* valid ip cksum */ 598 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IP_CKSUM_IS_GOOD); 599 } 600 } 601 } 602 603 // TCP or UDP segment. 604 if(!GET_FLAGS(parse_flags,PARSING_FLAGS_FRAGMENTATION_STATUS)) 605 { 606 /* check if TCP segment */ 607 if((GET_FLAGS(parse_flags,PARSING_FLAGS_OVER_IP_PROTOCOL) >> 608 PARSING_FLAGS_OVER_IP_PROTOCOL_SHIFT) == PRS_FLAG_OVERIP_TCP) 609 { 610 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IS_TCP_SEGMENT); 611 DbgMessage(pdev, INFORM, "--- TCP Packet --- \n"); 612 } 613 /* check if UDP segment */ 614 else if((GET_FLAGS(parse_flags,PARSING_FLAGS_OVER_IP_PROTOCOL) >> 615 PARSING_FLAGS_OVER_IP_PROTOCOL_SHIFT) == PRS_FLAG_OVERIP_UDP) 616 { 617 SET_FLAGS(pkt->l2pkt_rx_info->flags , LM_RX_FLAG_IS_UDP_DATAGRAM); 618 DbgMessage(pdev, INFORM, "--- UDP Packet --- \n"); 619 } 620 } 621 622 623 if( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) && 624 !GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)) 625 { 626 ASSERT_STATIC(LM_RX_FLAG_UDP_CKSUM_IS_GOOD == LM_RX_FLAG_IS_UDP_DATAGRAM << SHIFT_IS_GOOD); 627 ASSERT_STATIC(LM_RX_FLAG_UDP_CKSUM_IS_BAD == LM_RX_FLAG_IS_UDP_DATAGRAM << SHIFT_IS_BAD); 628 ASSERT_STATIC(LM_RX_FLAG_TCP_CKSUM_IS_GOOD == LM_RX_FLAG_IS_TCP_SEGMENT << SHIFT_IS_GOOD); 629 ASSERT_STATIC(LM_RX_FLAG_TCP_CKSUM_IS_BAD == LM_RX_FLAG_IS_TCP_SEGMENT << SHIFT_IS_BAD); 630 631 DbgMessage(pdev, INFORM, " Checksum validated.\n"); 632 633 /* tcp/udp cksum validated */ 634 if GET_FLAGS(cqe->type_error_flags, ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG) 635 { 636 /* invalid tcp/udp cksum */ 637 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) << SHIFT_IS_BAD ) ); 638 639 LM_COMMON_DRV_STATS_ATOMIC_INC_ETH(pdev, rx_tcp_cs_error_count); 640 DbgMessage(pdev, INFORM, " BAD checksum.\n"); 641 } 642 else if (GET_FLAGS(pkt->l2pkt_rx_info->flags , LM_RX_FLAG_IP_CKSUM_IS_BAD)) 643 { 644 /* invalid tcp/udp cksum due to invalid ip cksum */ 645 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) << SHIFT_IS_BAD ) ); 646 DbgMessage(pdev, INFORM, " BAD IP checksum\n"); 647 } 648 else 649 { 650 /* valid tcp/udp cksum */ 651 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) << SHIFT_IS_GOOD ) ); 652 DbgMessage(pdev, INFORM, " GOOD checksum.\n"); 653 } 654 } 655 else 656 { 657 DbgMessage(pdev, INFORM, " Checksum NOT validated.\n"); 658 /*Packets with invalid TCP options are reported with L4_XSUM_NO_VALIDATION due to HW limitation. In this case we assume that 659 their checksum is OK.*/ 660 if(GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) && 661 GET_FLAGS(cqe->status_flags, ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG) && 662 GET_FLAGS(cqe->pars_flags.flags, PARSING_FLAGS_TCP_OPTIONS_EXIST)) 663 { 664 DbgMessage(pdev, INFORM, " TCP Options exist - forcing return value.\n"); 665 if(GET_FLAGS(pkt->l2pkt_rx_info->flags , LM_RX_FLAG_IP_CKSUM_IS_BAD)) 666 { 667 DbgMessage(pdev, INFORM, " IP checksum invalid - reporting BAD checksum.\n"); 668 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) << SHIFT_IS_BAD ) ); 669 } 670 else 671 { 672 DbgMessage(pdev, INFORM, " IP checksum ok - reporting GOOD checksum.\n"); 673 SET_FLAGS(pkt->l2pkt_rx_info->flags , ( GET_FLAGS(pkt->l2pkt_rx_info->flags, (LM_RX_FLAG_IS_TCP_SEGMENT | LM_RX_FLAG_IS_UDP_DATAGRAM)) << SHIFT_IS_GOOD ) ); 674 } 675 } 676 } 677 } 678 679 __inline STATIC void 680 lm_recv_set_pkt_len( IN lm_device_t* pdev, 681 INOUT lm_packet_t* pkt, 682 IN const u16_t pkt_len, 683 IN const u32_t chain_idx) 684 { 685 //changed, as we dont have fhdr infrastructure 686 pkt->l2pkt_rx_info->size = pkt_len; //- 4; /* CRC32 */ 687 688 DbgMessage(pdev, VERBOSEl2, "pkt_size: %d\n",pkt->l2pkt_rx_info->size); 689 } 690 691 INLINE STATIC u32_t 692 calc_cksum(u16_t *hdr, u32_t len_in_bytes, u32_t sum) 693 { 694 // len_in_bytes - the length in bytes of the header 695 // sum - initial checksum 696 while (len_in_bytes > 1) 697 { 698 sum += NTOH16(*hdr); 699 len_in_bytes -= 2; 700 hdr++; 701 } 702 703 /* add left-over byte, if any */ 704 if (len_in_bytes) 705 { 706 sum += ((NTOH16(*hdr)) & 0xFF00); 707 } 708 709 return sum; 710 } 711 712 INLINE STATIC u8_t 713 validate_cksum(u32_t sum) 714 { 715 // len - the length in words of the header 716 // returns true iff the checksum (already written in the headr) is valid 717 718 // fold 32-bit sum to 16 bits 719 while (sum >> 16) 720 { 721 sum = (sum & 0xffff) + (sum >> 16); 722 } 723 724 return ((u16_t)(sum) == 0xffff); 725 } 726 727 INLINE STATIC u16_t 728 get_ip_hdr_len(u8_t *hdr) 729 { 730 // returns the ip header length in bytes 731 u16_t ip_hdr_len = 40; // ipv6 header length, we won't support ipv6 with extension header for now 732 733 if ((hdr[0] & 0xf0) == 0x40) 734 { 735 // ipv4, the lower 4 bit of the 1st byte of ip header 736 // contains the ip header length in unit of dword(32-bit) 737 ip_hdr_len = ((hdr[0] & 0xf) << 2); 738 } 739 return ip_hdr_len; 740 } 741 742 INLINE void 743 encap_pkt_parsing(struct _lm_device_t *pdev, 744 lm_packet_t *pkt) 745 { 746 u16_t tmp, inner_ip_hdr_len, tcp_length; 747 u32_t psuedo_cksum; 748 u8_t *hdr; 749 750 // encapsulated packet: 751 // outer mac | outer ip | gre | inner mac | inner ip | tcp 752 // minimum encapsultaed packet size is: 753 // two mac headers + gre header size + tcp header size + two ipv4 headers 754 if (pkt->l2pkt_rx_info->total_packet_size < (2*ETHERNET_PACKET_HEADER_SIZE + 2*20 + ETHERNET_GRE_SIZE + 20)) 755 { 756 return; 757 } 758 759 760 // set hdr to the outer ip header 761 hdr = pkt->l2pkt_rx_info->mem_virt + pdev->params.rcv_buffer_offset + ETHERNET_PACKET_HEADER_SIZE; 762 if (pkt->l2pkt_rx_info->flags & LM_RX_FLAG_VALID_VLAN_TAG) 763 { 764 hdr += ETHERNET_VLAN_TAG_SIZE; 765 } 766 767 // in case this is not standard ETH packet (e.g. managment, or in general non ipv4/ipv6), it is for sure 768 // not gre so we can end here 769 // if outer header is ipv4, protocol is the nine'th octet 770 // if outer header is ipv6, next header is the sixth octet 771 if (!(((pkt->l2pkt_rx_info->flags & LM_RX_FLAG_IS_IPV4_DATAGRAM) && (hdr[9] == 0x2f)) || 772 ((pkt->l2pkt_rx_info->flags & LM_RX_FLAG_IS_IPV6_DATAGRAM) && (hdr[6] == 0x2f)))) 773 { 774 // this is not encapsulated packet, no gre tunneling 775 // on ipv6 we don't support extension header 776 return; 777 } 778 779 // get the length of the outer ip header and set hdr to the gre header 780 hdr += get_ip_hdr_len(hdr); 781 782 /* GRE header 783 | Bits 0�4 | 5�7 | 8�12 | 13�15 | 16�31 | 784 | C|0|K|S | Recur | Flags | Version | Protocol Type | 785 | Checksum (optional) | Reserved | 786 | Key (optional) | 787 | Sequence Number (optional) | */ 788 789 // check that: 790 // checksum present bit is set to 0 791 // key present bit is set to 1 792 // sequence number present bit is set to 0 793 // protocol type should be always equal to 0x6558 (for encapsulating ethernet packets in GRE) 794 if (((hdr[0] & 0xb0) != 0x20) || (hdr[2] != 0x65) || (hdr[3] != 0x58)) 795 { 796 return; 797 } 798 // set hdr to the inner mac header 799 hdr += ETHERNET_GRE_SIZE; 800 801 // The first two octets of the tag are the Tag Protocol Identifier (TPID) value of 0x8100. 802 // This is located in the same place as the EtherType/Length field in untagged frames 803 if ((hdr[12] == 0x81) && (hdr[13] == 0x00)) 804 { 805 hdr += ETHERNET_VLAN_TAG_SIZE; 806 } 807 // set hdr to the inner ip header 808 hdr += ETHERNET_PACKET_HEADER_SIZE; 809 810 // get the length of the inner ip header 811 inner_ip_hdr_len = get_ip_hdr_len(hdr); 812 813 if ((hdr[0] & 0xf0) == 0x40) 814 { 815 // inner ip header is ipv4 816 // if the ip header checksum of the outer header is ok than validate the ip checksum of the inner header 817 if (pkt->l2pkt_rx_info->flags & LM_RX_FLAG_IP_CKSUM_IS_GOOD) 818 { 819 // validate the checksum 820 if (!validate_cksum(calc_cksum((u16_t*)hdr, inner_ip_hdr_len, 0))) 821 { 822 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IP_CKSUM_IS_BAD); 823 RESET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IP_CKSUM_IS_GOOD); 824 } 825 } 826 // check if protocol field is tcp 827 if (hdr[9] == 0x06) 828 { 829 // create the psuedo header 830 /* | Bit offset | 0�7 | 8�15 | 16�31 | 831 | 0 | Source address | 832 | 32 | Destination address | 833 | 64 | Zeros | Protocol | TCP length | */ 834 835 // adding 1 byte of zeros + protocol to the sum 836 // and adding source and destination address 837 psuedo_cksum = calc_cksum((u16_t*)&hdr[12], 8, 0x06); 838 // calculate the tcp length 839 mm_memcpy(&tmp, &hdr[2], sizeof(u16_t)); 840 tcp_length = NTOH16(tmp) - inner_ip_hdr_len; 841 // the TCP length field is the length of the TCP header and data (measured in octets). 842 psuedo_cksum += tcp_length; 843 } 844 else 845 { 846 // no tcp over ip 847 return; 848 } 849 } 850 else if ((hdr[0] & 0xf0) == 0x60) 851 { 852 // inner ip header is ipv6 853 // check if next header field is tcp 854 if (hdr[6] == 0x06) 855 { 856 // tcp over ipv6 857 // create the psuedo header 858 /* | Bit offset | 0�7 | 8�15 | 16�23 | 24�31 | 859 | 0 | Source address | 860 | 32 | | 861 | 64 | | 862 | 96 | | 863 | 128 | Destination address | 864 | 160 | | 865 | 192 | | 866 | 224 | | 867 | 256 | TCP length | 868 | 288 | Zeros |Next header |*/ 869 870 // adding 3 byte of zeros + protocol to the sum 871 // and adding source and destination address 872 psuedo_cksum = calc_cksum((u16_t*)&hdr[8], 32, 0x06); 873 // calculate the tcp length 874 // in the ip header: the size of the payload in octets, including any extension headers 875 mm_memcpy(&tmp, &hdr[4], sizeof(u16_t)); 876 // reduce the length of the extension headers 877 tcp_length = NTOH16(tmp) - (inner_ip_hdr_len - 40); 878 psuedo_cksum += tcp_length; 879 } 880 else 881 { 882 // no tcp over ip 883 return; 884 } 885 } 886 else 887 { 888 // no ipv4 or ipv6 889 return; 890 } 891 // set hdr to the tcp header 892 hdr += inner_ip_hdr_len; 893 894 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_IS_TCP_SEGMENT); 895 // claculate the checksum of the rest of the packet 896 // validate the checksum 897 if (validate_cksum(calc_cksum((u16_t*)hdr, tcp_length, psuedo_cksum))) 898 { 899 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_TCP_CKSUM_IS_GOOD); 900 RESET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_TCP_CKSUM_IS_BAD); 901 } 902 else 903 { 904 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_TCP_CKSUM_IS_BAD); 905 RESET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_TCP_CKSUM_IS_GOOD); 906 } 907 } 908 909 /******************************************************************************* 910 * Description: 911 * Here the RCQ chain is the chain coordinated with the status block, that is, 912 * the index in the status block describes the RCQ and NOT the rx_bd chain as in 913 * the case of Teton. We run on the delta between the new consumer index of the RCQ 914 * which we get from the sb and the old consumer index of the RCQ. 915 * In cases of both slow and fast path, the consumer of the RCQ is always incremented. 916 * 917 * The assumption which we must stick to all the way is: RCQ and Rx bd chain 918 * have the same size at all times! Otherwise, so help us Alan Bertkey! 919 * 920 * Return: 921 ******************************************************************************/ 922 u32_t 923 lm_get_packets_rcvd( struct _lm_device_t *pdev, 924 u32_t const chain_idx, 925 s_list_t *rcvd_list, 926 struct _sp_cqes_info *sp_cqes) 927 { 928 lm_rx_chain_t* rxq_chain = &LM_RXQ(pdev, chain_idx); //get a hold of the matching Rx bd chain according to index 929 lm_rcq_chain_t* rcq_chain = &LM_RCQ(pdev, chain_idx); //get a hold of the matching RCQ chain according to index 930 lm_bd_chain_t* rx_chain_bd = &LM_RXQ_CHAIN_BD(pdev, chain_idx); 931 lm_bd_chain_t* rx_chain_sge = LM_RXQ_SGE_PTR_IF_VALID(pdev, chain_idx); 932 lm_tpa_chain_t* tpa_chain = &LM_TPA(pdev, chain_idx); 933 union eth_rx_cqe* cqe = NULL; 934 lm_packet_t* pkt = NULL; 935 u32_t pkt_cnt = 0; 936 u16_t rx_old_idx = 0; 937 u16_t cq_new_idx = 0; 938 u16_t cq_old_idx = 0; 939 enum eth_rx_cqe_type cqe_type = MAX_ETH_RX_CQE_TYPE; 940 941 DbgMessage(pdev, INFORMl2 , "lm_get_packets_rcvd inside!\n"); 942 943 /* make sure to zeroize the sp_cqes... */ 944 mm_mem_zero( sp_cqes, sizeof(struct _sp_cqes_info) ); 945 946 /* Get the new consumer idx. The bd's between rcq_new_idx and rcq_old_idx 947 * are bd's containing receive packets. 948 */ 949 cq_new_idx = mm_le16_to_cpu(*(rcq_chain->hw_con_idx_ptr)); 950 951 /* The consumer index of the RCQ only, may stop at the end of a page boundary. In 952 * this case, we need to advance the next to the next one. 953 * In here we do not increase the cons_bd as well! this is since we're dealing here 954 * with the new cons index and not with the actual old one for which, as we progress, we 955 * need to maintain the bd_cons as well. 956 */ 957 if((cq_new_idx & lm_bd_chain_usable_bds_per_page(&rcq_chain->bd_chain)) == lm_bd_chain_usable_bds_per_page(&rcq_chain->bd_chain)) 958 { 959 cq_new_idx+= lm_bd_chain_bds_skip_eop(&rcq_chain->bd_chain); 960 } 961 962 DbgBreakIfFastPath( rx_chain_sge && !lm_bd_chains_are_consistent( rx_chain_sge, rx_chain_bd ) ); 963 964 rx_old_idx = lm_bd_chain_cons_idx(rx_chain_bd); 965 cq_old_idx = lm_bd_chain_cons_idx(&rcq_chain->bd_chain); 966 967 //there is no change in the RCQ consumer index so exit! 968 if (cq_old_idx == cq_new_idx) 969 { 970 DbgMessage(pdev, INFORMl2rx , "there is no change in the RCQ consumer index so exit!\n"); 971 return pkt_cnt; 972 } 973 974 while(cq_old_idx != cq_new_idx) 975 { 976 DbgBreakIfFastPath(S16_SUB(cq_new_idx, cq_old_idx) <= 0); 977 //get hold of the cqe, and find out what it's type corresponds to 978 cqe = (union eth_rx_cqe *)lm_bd_chain_consume_bd(&rcq_chain->bd_chain); 979 DbgBreakIfFastPath(cqe == NULL); 980 981 //update the cons of the RCQ and the bd_prod pointer of the RCQ as well! 982 //this holds both for slow and fast path! 983 cq_old_idx = lm_bd_chain_cons_idx(&rcq_chain->bd_chain); 984 985 cqe_type = GET_FLAGS_WITH_OFFSET(cqe->ramrod_cqe.ramrod_type, COMMON_RAMROD_ETH_RX_CQE_TYPE, COMMON_RAMROD_ETH_RX_CQE_TYPE_SHIFT); 986 DbgBreakIf(MAX_ETH_RX_CQE_TYPE <= cqe_type); 987 988 //the cqe is a ramrod, so do the ramrod and recycle the cqe. 989 //TODO: replace this with the #defines: 1- eth ramrod, 2- toe init ofld ramrod 990 switch(cqe_type) 991 { 992 case RX_ETH_CQE_TYPE_ETH_RAMROD: 993 { 994 /* 13/08/08 NirV: bugbug, temp workaround for dpc watch dog bug, 995 * ignore toe completions on L2 ring - initiate offload */ 996 if (cqe->ramrod_cqe.conn_type != TOE_CONNECTION_TYPE) 997 { 998 if (ERR_IF(sp_cqes->idx >= MAX_NUM_SPE)) 999 { 1000 DbgBreakMsgFastPath("too many spe completed\n"); 1001 /* we shouldn't get here - there is something very wrong if we did... in this case we will risk 1002 * completing the ramrods - even though we're holding a lock!!! */ 1003 /* bugbug... */ 1004 DbgBreakIfAll(sp_cqes->idx >= MAX_NUM_SPE); 1005 return pkt_cnt; 1006 } 1007 mm_memcpy((void*)(&(sp_cqes->sp_cqe[sp_cqes->idx++])), (const void*)cqe, sizeof(*cqe)); 1008 } 1009 1010 //update the prod of the RCQ - by this, we recycled the CQE. 1011 lm_bd_chain_bd_produced(&rcq_chain->bd_chain); 1012 1013 #if 0 1014 //in case of ramrod, pop out the Rx bd and push it to the free descriptors list 1015 pkt = (lm_packet_t *) s_list_pop_head(&rxq_chain->active_descq); 1016 1017 DbgBreakIfFastPath(pkt == NULL); 1018 1019 s_list_push_tail( &LM_RXQ(pdev, chain_idx).free_descq, 1020 &pkt->link); 1021 #endif 1022 break; 1023 } 1024 case RX_ETH_CQE_TYPE_ETH_FASTPATH: 1025 case RX_ETH_CQE_TYPE_ETH_START_AGG: //Fall through case 1026 { //enter here in case the cqe is a fast path type (data) 1027 u16_t parse_flags = 0; 1028 1029 DbgMessage(pdev, INFORMl2rx, "lm_get_packets_rcvd- it is fast path, func=%d\n", FUNC_ID(pdev)); 1030 1031 DbgBreakIf( (RX_ETH_CQE_TYPE_ETH_START_AGG == cqe_type)&& 1032 (lm_tpa_state_disable == tpa_chain->state)); 1033 1034 pkt = (lm_packet_t *) s_list_pop_head(&rxq_chain->active_descq); 1035 parse_flags = mm_le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags); 1036 1037 DbgBreakIfFastPath( NULL == pkt ); 1038 1039 #if DBG 1040 if CHK_NULL( pkt ) 1041 { 1042 return 0; 1043 } 1044 #endif // DBG 1045 1046 DbgBreakIfFastPath(SIG(pkt) != L2PACKET_RX_SIG); 1047 1048 #if L2_RX_BUF_SIG 1049 /* make sure signitures exist before and after the buffer */ 1050 DbgBreakIfFastPath(SIG(pkt->u1.rx.mem_virt - pdev->params.rcv_buffer_offset) != L2PACKET_RX_SIG); 1051 DbgBreakIfFastPath(END_SIG(pkt->u1.rx.mem_virt, MAX_L2_CLI_BUFFER_SIZE(pdev, chain_idx)) != L2PACKET_RX_SIG); 1052 #endif /* L2_RX_BUF_SIG */ 1053 1054 lm_bd_chain_bds_consumed(rx_chain_bd, 1); 1055 if( rx_chain_sge ) 1056 { 1057 lm_bd_chain_bds_consumed(rx_chain_sge, 1); 1058 } 1059 #if defined(_NTDDK_) 1060 //PreFast 28182 :Prefast reviewed and suppress this situation shouldn't occur. 1061 #pragma warning (push) 1062 #pragma warning( disable:28182 ) 1063 #endif // !_NTDDK_ 1064 /* Advance the rx_old_idx to the start bd_idx of the next packet. */ 1065 rx_old_idx = pkt->u1.rx.next_bd_idx; 1066 //cq_old_idx = pkt->u1.rx.next_bd_idx; 1067 1068 CLEAR_FLAGS( pkt->l2pkt_rx_info->flags ); 1069 1070 1071 if(RX_ETH_CQE_TYPE_ETH_START_AGG == cqe_type) 1072 { 1073 lm_recv_set_pkt_len(pdev, pkt, mm_le16_to_cpu(cqe->fast_path_cqe.len_on_bd), chain_idx); 1074 // total_packet_size is only known in stop_TPA 1075 1076 DbgBreakIf(0 != cqe->fast_path_cqe.pkt_len_or_gro_seg_len); 1077 1078 lm_tpa_start(pdev, 1079 pkt, 1080 chain_idx, 1081 cqe->fast_path_cqe.queue_index); 1082 1083 lm_tpa_start_flags_handle(pdev, 1084 &(cqe->fast_path_cqe), 1085 pkt, 1086 parse_flags); 1087 } 1088 else 1089 { 1090 lm_recv_set_pkt_len(pdev, pkt, mm_le16_to_cpu(cqe->fast_path_cqe.pkt_len_or_gro_seg_len), chain_idx); 1091 1092 // In regular mode pkt->l2pkt_rx_info->size == pkt->l2pkt_rx_info->total_packet_size 1093 // We need total_packet_size for Dynamic HC in order not to ask a question there if we are RSC or regular flow. 1094 pkt->l2pkt_rx_info->total_packet_size = pkt->l2pkt_rx_info->size; 1095 1096 /* make sure packet size if larger than header size and smaller than max packet size of the specific L2 client */ 1097 DbgBreakIfFastPath((pkt->l2pkt_rx_info->total_packet_size < MIN_ETHERNET_PACKET_SIZE) || (pkt->l2pkt_rx_info->total_packet_size > MAX_CLI_PACKET_SIZE(pdev, chain_idx))); 1098 1099 // ShayH:packet->size isn't useed anymore by windows we directly put the data on l2pkt_rx_info->size and l2pkt_rx_info->total_packet_size. 1100 // Need to ask if other UM clients use/need packet->size. 1101 pkt->size = pkt->l2pkt_rx_info->size; 1102 1103 if(OOO_CID(pdev) == chain_idx) 1104 { 1105 DbgBreakIfFastPath( ETH_FP_CQE_RAW != (GET_FLAGS( cqe->fast_path_cqe.type_error_flags, ETH_FAST_PATH_RX_CQE_SGL_RAW_SEL ) >> 1106 ETH_FAST_PATH_RX_CQE_SGL_RAW_SEL_SHIFT)); 1107 1108 //optimized 1109 /* make sure packet size if larger than header size and smaller than max packet size of the specific L2 client */ 1110 // TODO_OOO - check with flag 1111 ASSERT_STATIC( sizeof(pkt->u1.rx.sgl_or_raw_data.raw_data) == sizeof(cqe->fast_path_cqe.sgl_or_raw_data.raw_data) ); 1112 mm_memcpy( pkt->u1.rx.sgl_or_raw_data.raw_data, cqe->fast_path_cqe.sgl_or_raw_data.raw_data, sizeof(pkt->u1.rx.sgl_or_raw_data.raw_data) ); 1113 } 1114 else 1115 { 1116 DbgBreakIfFastPath( ETH_FP_CQE_REGULAR != (GET_FLAGS( cqe->fast_path_cqe.type_error_flags, ETH_FAST_PATH_RX_CQE_SGL_RAW_SEL )>> 1117 ETH_FAST_PATH_RX_CQE_SGL_RAW_SEL_SHIFT) ) ; 1118 } 1119 1120 lm_regular_flags_handle(pdev, 1121 &(cqe->fast_path_cqe), 1122 pkt, 1123 parse_flags); 1124 1125 if (GET_FLAGS(pdev->params.ofld_cap_to_ndis, LM_OFFLOAD_ENCAP_PACKET)) 1126 { 1127 // SW rx checksum for gre encapsulated packets 1128 encap_pkt_parsing(pdev, pkt); 1129 } 1130 1131 pkt_cnt++; 1132 s_list_push_tail(rcvd_list, &pkt->link); 1133 } 1134 1135 if GET_FLAGS(cqe->fast_path_cqe.status_flags, ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG) 1136 { 1137 SET_FLAGS(pkt->l2pkt_rx_info->flags, LM_RX_FLAG_VALID_HASH_VALUE ); 1138 *pkt->u1.rx.hash_val_ptr = mm_le32_to_cpu(cqe->fast_path_cqe.rss_hash_result); 1139 } 1140 1141 if(GET_FLAGS(parse_flags,PARSING_FLAGS_INNER_VLAN_EXIST)) 1142 { 1143 u16_t vlan_tag = mm_le16_to_cpu(cqe->fast_path_cqe.vlan_tag); 1144 1145 DbgMessage(pdev, INFORMl2, "vlan frame recieved: %x\n",vlan_tag); 1146 /* fw always set ETH_FAST_PATH_RX_CQE_VLAN_TAG_FLG and pass vlan tag when 1147 packet with vlan arrives but it remove the vlan from the packet only when 1148 it configured to remove vlan using params.vlan_removal_enable 1149 */ 1150 if ((!pdev->params.keep_vlan_tag) && 1151 ( OOO_CID(pdev) != chain_idx)) 1152 { 1153 SET_FLAGS(pkt->l2pkt_rx_info->flags , LM_RX_FLAG_VALID_VLAN_TAG); 1154 pkt->l2pkt_rx_info->vlan_tag = vlan_tag; 1155 DbgMessage(pdev, INFORMl2rx, "vlan removed from frame: %x\n",vlan_tag); 1156 } 1157 } 1158 1159 #if defined(_NTDDK_) 1160 #pragma warning (pop) 1161 #endif // !_NTDDK_ 1162 #if DBG 1163 if(GET_FLAGS(parse_flags,PARSING_FLAGS_FRAGMENTATION_STATUS)) 1164 { 1165 LM_COMMON_DRV_STATS_ATOMIC_INC_ETH(pdev, rx_ipv4_frag_count); 1166 } 1167 if(GET_FLAGS(parse_flags,PARSING_FLAGS_LLC_SNAP)) 1168 { 1169 LM_COMMON_DRV_STATS_ATOMIC_INC_ETH(pdev, rx_llc_snap_count); 1170 } 1171 if(GET_FLAGS(parse_flags,PARSING_FLAGS_IP_OPTIONS) && 1172 GET_FLAGS(pkt->l2pkt_rx_info->flags ,LM_RX_FLAG_IS_IPV6_DATAGRAM)) 1173 { 1174 LM_COMMON_DRV_STATS_ATOMIC_INC_ETH(pdev, rx_ipv6_ext_count); 1175 } 1176 #endif // DBG 1177 1178 /* We use to assert that if we got the PHY_DECODE_ERROR it was always a result of DROP_MAC_ERR, since we don't configure 1179 * DROP_MAC_ERR anymore, we don't expect this flag to ever be on.*/ 1180 DbgBreakIfFastPath( GET_FLAGS(cqe->fast_path_cqe.type_error_flags, ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG) ); 1181 1182 DbgBreakIfFastPath(cqe->fast_path_cqe.type_error_flags & 1183 ~(ETH_FAST_PATH_RX_CQE_TYPE | 1184 ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG | 1185 ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG | 1186 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG | 1187 ETH_FAST_PATH_RX_CQE_SGL_RAW_SEL)); 1188 1189 1190 break; 1191 } 1192 case RX_ETH_CQE_TYPE_ETH_STOP_AGG: 1193 {//TPA stop 1194 DbgBreakIf( lm_tpa_state_disable == tpa_chain->state); 1195 1196 pkt_cnt = lm_tpa_stop(pdev, 1197 rcvd_list, 1198 &(cqe->end_agg_cqe), 1199 chain_idx, 1200 pkt_cnt, 1201 cqe->end_agg_cqe.queue_index); 1202 1203 //update the prod of the RCQ - by this, we recycled the CQE. 1204 lm_bd_chain_bd_produced(&rcq_chain->bd_chain); 1205 break; 1206 } 1207 case MAX_ETH_RX_CQE_TYPE: 1208 default: 1209 { 1210 DbgBreakMsg("CQE type not supported"); 1211 } 1212 1213 } 1214 } 1215 1216 // TODO: Move index update to a more suitable place 1217 rx_chain_bd->cons_idx = rx_old_idx; 1218 if( rx_chain_sge ) 1219 { 1220 rx_chain_sge->cons_idx = rx_old_idx; 1221 } 1222 1223 //notify the fw of the prod 1224 lm_rx_set_prods(pdev, rcq_chain->iro_prod_offset, &rcq_chain->bd_chain, rx_chain_bd, rx_chain_sge ,chain_idx); 1225 1226 DbgMessage(pdev, INFORMl2rx, "lm_get_packets_rcvd- bd con: %d bd prod: %d \n", 1227 lm_bd_chain_cons_idx(rx_chain_bd), lm_bd_chain_prod_idx(rx_chain_bd)); 1228 DbgMessage(pdev, INFORMl2rx, "lm_get_packets_rcvd- cq con: %d cq prod: %d \n", 1229 lm_bd_chain_cons_idx(&rcq_chain->bd_chain), lm_bd_chain_prod_idx(&rcq_chain->bd_chain)); 1230 return pkt_cnt; 1231 } /* lm_get_packets_rcvd */ 1232 1233 lm_status_t lm_complete_ramrods( 1234 struct _lm_device_t *pdev, 1235 struct _sp_cqes_info *sp_cqes) 1236 { 1237 u8_t idx; 1238 1239 for (idx = 0; idx < sp_cqes->idx; idx++) { 1240 lm_eth_init_command_comp(pdev, &(sp_cqes->sp_cqe[idx].ramrod_cqe)); 1241 } 1242 1243 return LM_STATUS_SUCCESS; 1244 } 1245 1246 /* called by um whenever packets are returned by client 1247 rxq lock is taken by caller */ 1248 void 1249 lm_return_packet_bytes( struct _lm_device_t *pdev, 1250 u32_t const qidx, 1251 u32_t const returned_bytes) 1252 { 1253 lm_rx_chain_t *rxq = &LM_RXQ(pdev, qidx); 1254 1255 rxq->ret_bytes += returned_bytes; 1256 1257 /* aggregate updates over PCI */ 1258 1259 /* HC_RET_BYTES_TH = min(l2_hc_threshold0 / 2 , 16KB) */ 1260 #define HC_RET_BYTES_TH(pdev) (((pdev)->params.hc_threshold0[SM_RX_ID] < 32768) ? ((pdev)->params.hc_threshold0[SM_RX_ID] >> 1) : 16384) 1261 1262 /* TODO: Future: Add #updatesTH = 20 */ 1263 1264 /* time to update fw ? */ 1265 if(S32_SUB(rxq->ret_bytes, rxq->ret_bytes_last_fw_update + HC_RET_BYTES_TH(pdev)) >= 0) 1266 { 1267 /* 1268 !!DP 1269 The test below is to disable dynamic HC for the iSCSI chains 1270 */ 1271 // TODO: VF dhc 1272 if (qidx < LM_MAX_RSS_CHAINS(pdev) && IS_PFDEV(pdev)) /* should be fine, if not, you can go for less robust case of != LM_CLI_RX_CHAIN_IDX(pdev, LM_CLI_IDX_ISCSI) */ 1273 { 1274 /* There are HC_USTORM_SB_NUM_INDICES (4) index values for each SB to set and we're using the corresponding U indexes from the microcode consts */ 1275 LM_INTMEM_WRITE32(PFDEV(pdev), rxq->hc_sb_info.iro_dhc_offset, rxq->ret_bytes, BAR_CSTRORM_INTMEM); 1276 rxq->ret_bytes_last_fw_update = rxq->ret_bytes; 1277 } else if (IS_VFDEV(pdev)) { 1278 VF_REG_WR(pdev, VF_BAR0_CSDM_QUEUES_OFFSET + rxq->hc_sb_info.iro_dhc_offset, rxq->ret_bytes); 1279 rxq->ret_bytes_last_fw_update = rxq->ret_bytes; 1280 } 1281 } 1282 } 1283 1284