xref: /illumos-gate/usr/src/uts/common/io/bnxe/577xx/drivers/common/lm/l4/lm_l4tx.c (revision d14abf155341d55053c76eeec58b787a456b753b)
1 
2 #include "lm5710.h"
3 #include "bd_chain.h"
4 #include "command.h"
5 #include "context.h"
6 #include "lm_l4fp.h"
7 #include "lm_l4sp.h"
8 #include "mm_l4if.h"
9 
10 
11 /* TODO: remove this temporary solution for solaris / linux compilation conflict, linux needs the
12  * first option, solaris the latter */
13 #if defined(__LINUX)
14 #define TOE_TX_INIT_ZERO {{0}}
15 #else
16 #define TOE_TX_INIT_ZERO {0}
17 #endif
18 
19 #define TOE_TX_DOORBELL(pdev,cid) do{\
20     struct doorbell db = TOE_TX_INIT_ZERO;\
21     db.header.data |= (TOE_CONNECTION_TYPE << DOORBELL_HDR_T_CONN_TYPE_SHIFT);\
22     DOORBELL((pdev), (cid), *((u32_t *)&db));\
23     } while(0)
24 
_lm_tcp_tx_write_db(lm_device_t * pdev,lm_tcp_con_t * tx_con,u32_t cid,u32_t nbytes,u16_t nbds,u8_t fin)25 static __inline void _lm_tcp_tx_write_db(
26     lm_device_t  * pdev,
27     lm_tcp_con_t * tx_con,
28     u32_t cid,
29     u32_t nbytes,
30     u16_t nbds,
31     u8_t fin)
32 {
33     volatile struct toe_tx_db_data *db_data = tx_con->db_data.tx;
34 
35     db_data->bds_prod += nbds;       /* nbds should be written before nbytes (FW assumption) */
36     DbgBreakIf((db_data->bds_prod & 0xff) == 0);
37     db_data->bytes_prod_seq += nbytes;
38 
39     if(fin) {
40         DbgBreakIf(db_data->flags & (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT));
41         db_data->flags |= (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT);
42     }
43 
44     if (!(tx_con->flags & TCP_TX_DB_BLOCKED)) {
45         DbgMessage(pdev, INFORMl4tx,
46                     "ringing tx doorbell: cid=%d, (nbytes+=%d, nbds+=%d, fin=%d)\n",
47                     cid, nbytes, nbds, fin);
48         TOE_TX_DOORBELL(pdev, cid);
49     }
50 }
51 
lm_tcp_tx_write_db(lm_device_t * pdev,lm_tcp_state_t * tcp,u8_t post_end)52 static __inline void lm_tcp_tx_write_db(
53     lm_device_t *pdev,
54     lm_tcp_state_t *tcp,
55     u8_t post_end)
56 {
57     lm_tcp_con_t *tx_con = tcp->tx_con;
58 
59     /* define a policy for ringing the doorbell */
60     #define MAX_BYTES_PER_TX_DB    0xffff
61     #define MAX_BDS_PER_TX_DB      64
62 
63     if (post_end ||
64         tx_con->db_more_bytes >= MAX_BYTES_PER_TX_DB ||
65         tx_con->db_more_bds >= MAX_BDS_PER_TX_DB) {
66         _lm_tcp_tx_write_db(pdev, tx_con, tcp->cid, tx_con->db_more_bytes, tx_con->db_more_bds, 0);
67 
68         /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */
69         DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt + tx_con->db_more_bytes, tx_con->bytes_comp_cnt) < 0);
70         tx_con->bytes_post_cnt += tx_con->db_more_bytes;
71         tx_con->buffer_post_cnt += tx_con->db_more_bufs;
72         tx_con->db_more_bytes = tx_con->db_more_bds = tx_con->db_more_bufs = 0;
73         tx_con->fp_db_cnt++;
74     } else {
75         DbgMessage(pdev, INFORMl4tx,
76                     "skipped doorbell ringing for cid=%d\n", tcp->cid);
77     }
78 }
79 
80 /** Description:
81  *  Post a single tcp buffer to the Tx bd chain
82  * Assumptions:
83  *  - caller initiated tcp_buf->flags field with BUFFER_START/BUFFER_END appropriately
84  * Returns:
85  *  - SUCCESS - tcp buf was successfully attached to the bd chain
86  *  - RESOURCE - not enough available BDs on bd chain for given tcp buf
87  *  - CONNECTION_CLOSED - whenever connection's flag are marked as 'POST BLOCKED' */
lm_tcp_tx_post_buf(struct _lm_device_t * pdev,lm_tcp_state_t * tcp,lm_tcp_buffer_t * tcp_buf,lm_frag_list_t * frag_list)88 lm_status_t lm_tcp_tx_post_buf(
89     struct _lm_device_t *pdev,
90     lm_tcp_state_t      *tcp,
91     lm_tcp_buffer_t     *tcp_buf,
92     lm_frag_list_t      *frag_list)
93 {
94     lm_tcp_con_t *tx_con;
95     lm_bd_chain_t *tx_chain;
96     struct toe_tx_bd *tx_bd = NULL ;
97     lm_frag_t *frag;
98     u32_t i, dbg_buf_size = 0;
99     u32_t dbg_bytes_prod_seq;
100     u16_t old_prod, new_prod;
101 
102     DbgMessage(pdev, VERBOSEl4tx, "###lm_tcp_tx_post_buf\n");
103     DbgBreakIf(!(pdev && tcp && tcp_buf && frag_list));
104     DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid)));
105     DbgBreakIf(frag_list->cnt == 0);
106     tx_con = tcp->tx_con;
107     tx_chain = &tx_con->bd_chain;
108     frag = frag_list->frag_arr;
109 
110     DbgBreakIf(tx_con->flags & TCP_FIN_REQ_POSTED);
111 
112     /* check if tx con is already closed */
113     if(tx_con->flags & TCP_TX_POST_BLOCKED) {
114         DbgMessage(pdev, WARNl4tx, "post tx buf failed, posting is blocked (cid=%d, con->flags=%x)\n",
115                     tcp->cid, tx_con->flags);
116         return LM_STATUS_CONNECTION_CLOSED;
117     }
118     /* check bd chain availability (including additional bd that should
119      * be kept available for future fin request) */
120     if(lm_bd_chain_avail_bds(tx_chain) < frag_list->cnt + 1) {
121         DbgMessage(pdev, INFORMl4tx, "post tx buf failed, tx chain is full (cid=%d, avail bds=%d, buf nfrags=%d)\n",
122                     tcp->cid, lm_bd_chain_avail_bds(tx_chain), frag_list->cnt);
123 
124         LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, tx_no_l4_bd);
125 
126         if (tx_con->db_more_bds) {
127             /* if doorbell ringing was deferred (e.g. until an end of
128              * application buffer), it can no longer be deferred since
129              * the place in the bd chain is now required */
130             lm_tcp_tx_write_db(pdev, tcp, 1);
131         }
132         return LM_STATUS_RESOURCE;
133     }
134 
135     old_prod = lm_bd_chain_prod_idx(tx_chain);
136 
137     dbg_bytes_prod_seq = tx_con->db_data.tx->bytes_prod_seq + tx_con->db_more_bytes;
138      /* "attach" the frags to the bd chain */
139     for(i = 0; i < frag_list->cnt; i++, frag++) {
140         DbgBreakIf(frag->size > 0xffff || frag->size == 0); /* hw limit: each bd can point to a buffer with max size of 64KB */
141         tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(tx_chain);
142         tx_bd->addr_hi = frag->addr.as_u32.high;
143         tx_bd->addr_lo = frag->addr.as_u32.low;
144         tx_bd->flags = 0;
145         tx_bd->size = (u16_t)frag->size;
146         dbg_bytes_prod_seq += frag->size;
147         tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
148         dbg_buf_size += frag->size;
149 
150         /* Support for FW Nagle Algorithm:
151          * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
152          */
153         if ((u32_t)frag_list->size >= tx_con->u.tx.mss) {
154             tx_bd->flags |= TOE_TX_BD_LARGE_IO;
155         }
156 
157         DbgMessage(pdev, VERBOSEl4tx, "Setting Tx BD, addr_lo=0x%x, addr_hi=0x%x, size=%d\n",
158                     tx_bd->addr_lo, tx_bd->addr_hi, tx_bd->size);
159     }
160 
161     DbgBreakIf(frag_list->cnt > 0xffff);
162     tcp_buf->bd_used = frag_list->cnt & 0xffff;
163     tcp_buf->size = tcp_buf->more_to_comp = (u32_t)frag_list->size;
164     DbgBreakIf(tcp_buf->size != dbg_buf_size);
165 
166     DbgBreakIf(!(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START ?
167                  tx_con->app_buf_bytes_acc_post == 0 :
168                  tx_con->app_buf_bytes_acc_post > 0));
169     tx_con->app_buf_bytes_acc_post += tcp_buf->size;
170     tx_con->db_more_bytes += tcp_buf->size;
171     new_prod = lm_bd_chain_prod_idx(tx_chain);
172     DbgBreakIf(S16_SUB(new_prod, old_prod) < tcp_buf->bd_used);
173     tx_con->db_more_bds += S16_SUB(new_prod, old_prod);
174     tx_con->db_more_bufs++;
175 
176     /* Support for FW Nagle Algorithm:
177      * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
178      */
179     if (tcp_buf->size >= tx_con->u.tx.mss) {
180         tx_bd->flags |= TOE_TX_BD_LARGE_IO;
181     }
182 
183     /* special care in case of last tcp buffer of an application buffer */
184     if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) {
185         tcp_buf->app_buf_xferred = 0; /* just for safety */
186         tcp_buf->app_buf_size = tx_con->app_buf_bytes_acc_post;
187         tx_con->app_buf_bytes_acc_post = 0;
188 
189         /* special care for the last bd: */
190         tx_bd->flags |= TOE_TX_BD_NOTIFY;
191         tx_con->u.tx.bds_without_comp_flag = 0;
192         tx_bd->flags |= TOE_TX_BD_PUSH;
193 
194         DbgMessage(pdev, VERBOSEl4tx,
195                     "Setting Tx BD, last bd of app buf, flags=%d\n", tx_bd->flags);
196     } else {
197         /* make sure there aren't 'too many' bds without completion flag */
198         tx_con->u.tx.bds_without_comp_flag += tcp_buf->bd_used;
199         if (tx_con->u.tx.bds_without_comp_flag > (tx_chain->capacity - MAX_FRAG_CNT_PER_TB)) {
200             tx_bd->flags |= TOE_TX_BD_NOTIFY;
201             tx_con->u.tx.bds_without_comp_flag = 0;
202         }
203     }
204 
205     s_list_push_tail(&tx_con->active_tb_list, &tcp_buf->link);
206     tx_con->rq_nbytes += tcp_buf->size;
207     lm_tcp_tx_write_db(pdev, tcp, tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END);
208 
209     /* network reachability (NOT IMPLEMENTED):
210     if(lm_neigh_is_cache_entry_staled(tcp->path->neigh))
211        lm_neigh_indicate_staled_cache_entry(tcp->path->neigh);
212     */
213 
214     DbgMessage(pdev, VERBOSEl4tx, "posted tx buf for cid=%d, buf size=%d, bd used=%d, buf flags=%x, app_buf_size=%d\n",
215                 tcp->cid, tcp_buf->size, tcp_buf->bd_used, tcp_buf->flags, tcp_buf->app_buf_size);
216     DbgMessage(pdev, VERBOSEl4tx, "after posting tx buf, tx_con->active_tb_list=%d\n",
217                 s_list_entry_cnt(&tx_con->active_tb_list));
218 
219     return LM_STATUS_SUCCESS;
220 } /* lm_tcp_tx_post_buf */
221 
222 /** Description
223  *  indicates graceful disconnect completion to client.
224  * Assumtpions:
225  *  tx-lock is taken by caller
226  */
lm_tcp_tx_graceful_disconnect_complete(lm_device_t * pdev,lm_tcp_state_t * tcp)227 static __inline void lm_tcp_tx_graceful_disconnect_complete(lm_device_t * pdev, lm_tcp_state_t * tcp)
228 {
229     u8_t ip_version;
230     DbgBreakIf(!s_list_is_empty(&tcp->tx_con->active_tb_list));
231     DbgBreakIf(tcp->tx_con->flags & TCP_FIN_REQ_COMPLETED);
232     tcp->tx_con->flags |= TCP_FIN_REQ_COMPLETED;
233     DbgMessage(pdev, INFORMl4tx, "fin request completed (cid=%d)\n", tcp->cid);
234     tcp->tcp_state_calc.fin_completed_time = mm_get_current_time(pdev);
235     if (!(tcp->tx_con->u.tx.flags & TCP_CON_FIN_REQ_LM_INTERNAL)) {
236         ip_version = (tcp->path->path_const.ip_version == IP_VERSION_IPV4)? STATS_IP_4_IDX : STATS_IP_6_IDX;
237         LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, ipv[ip_version].out_fin);
238         mm_tcp_graceful_disconnect_done(pdev,tcp, LM_STATUS_SUCCESS);
239     }
240 }
241 
lm_tcp_tx_cmp_process(struct _lm_device_t * pdev,lm_tcp_state_t * tcp,u32_t completed_bytes)242 void lm_tcp_tx_cmp_process(
243     struct _lm_device_t *pdev,
244     lm_tcp_state_t      *tcp,
245     u32_t               completed_bytes
246     )
247 {
248     lm_tcp_con_t *tx_con = tcp->tx_con;
249     u32_t actual_completed; /* number of bytes actually completed (could be different than completed in case of fin) */
250     MM_INIT_TCP_LOCK_HANDLE();
251 
252     DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_app_cmp_process, cid=%d, completed_bytes=%d\n",
253                 tcp->cid, completed_bytes);
254 
255     DbgBreakIf(tx_con->flags & TCP_TX_COMP_BLOCKED);
256 
257     if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
258         mm_acquire_tcp_lock(pdev, tx_con);
259     }
260     tx_con->bytes_comp_cnt += completed_bytes;
261     DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt, tx_con->bytes_comp_cnt) < 0);
262 
263     DbgBreakIf(!completed_bytes);
264 
265     actual_completed = lm_tcp_complete_nbytes(pdev, tcp, tcp->tx_con, completed_bytes, FALSE);
266 
267     if (actual_completed != completed_bytes) {
268         DbgBreakIf(actual_completed > completed_bytes);
269         DbgBreakIf((completed_bytes - actual_completed) != 1);
270         DbgBreakIf(!(tx_con->flags & TCP_FIN_REQ_POSTED));
271         DbgBreakIf(tx_con->bytes_post_cnt != tx_con->bytes_comp_cnt);
272         /* fin completed */
273         tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_CMP;
274         tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_FIN_REQ_COMPLETED */
275     }
276 
277     /* network reachability (NOT IMPLEMENTED):
278     lm_neigh_update_nic_reachability_time(tcp->path->neigh)
279     */
280     if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
281         mm_release_tcp_lock(pdev, tx_con);
282     }
283 
284 } /* lm_tcp_tx_app_cmp_process */
285 
lm_toe_is_tx_completion(lm_device_t * pdev,u8_t drv_toe_rss_id)286 u8_t lm_toe_is_tx_completion(lm_device_t *pdev, u8_t drv_toe_rss_id)
287 {
288     u8_t result = FALSE;
289     lm_tcp_scq_t *scq = NULL;
290 
291     DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
292 
293     scq = &pdev->toe_info.scqs[drv_toe_rss_id];
294 
295     if ( scq->hw_con_idx_ptr &&
296         *scq->hw_con_idx_ptr != lm_bd_chain_cons_idx(&scq->bd_chain) )
297     {
298         result = TRUE;
299     }
300     DbgMessage(pdev, INFORMl4int, "lm_toe_is_tx_completion(): result is:%s\n", result? "TRUE" : "FALSE");
301 
302     return result;
303 }
304 
lm_tcp_tx_inc_trm_aborted_bytes(struct _lm_device_t * pdev,lm_tcp_state_t * tcp,u32_t aborted_bytes)305 void lm_tcp_tx_inc_trm_aborted_bytes(
306     struct _lm_device_t *pdev,
307     lm_tcp_state_t      *tcp,
308     u32_t               aborted_bytes
309     )
310 {
311     lm_tcp_con_t *tx_con = tcp->tx_con;
312     MM_INIT_TCP_LOCK_HANDLE();
313 
314     DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_inc_aborted_count, cid=%d, aborted_bytes=%d\n",
315                 tcp->cid, aborted_bytes);
316 
317     if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
318         mm_acquire_tcp_lock(pdev, tx_con);
319     }
320 
321     tx_con->bytes_trm_aborted_cnt += aborted_bytes;
322 
323     if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
324         mm_release_tcp_lock(pdev, tx_con);
325     }
326 
327 } /* lm_tcp_tx_inc_aborted_count */
328 
329 /** Description
330  *   completes the fast-path operations for a certain connection
331  *  Assumption:
332  *   fp-tx lock is taken
333  */
lm_tcp_tx_complete_tcp_fp(lm_device_t * pdev,lm_tcp_state_t * tcp,lm_tcp_con_t * con)334 void lm_tcp_tx_complete_tcp_fp(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con)
335 {
336     /**** Client completing :  may result in lock-release *****/
337     /* during lock-release, due to this function being called from service_deferred, more
338      * cqes can be processed. We don't want to mix. This function is mutually exclusive, so
339      * any processing makes it's way to being completed by calling this function.
340      * the following define a "fast-path completion"
341      * (i)   RQ buffers to be completed
342      *       defined by dpc_completed_tail and are collected during lm_tcp_complete_bufs BEFORE lock
343      *       is released, so no more buffer processing can make it's way into this buffer completion.
344      * (ii)  Fin to be completed
345      *       determined by the flags, since dpc_flags CAN be modified during processing we copy
346      *       them to a snapshot_flags parameter, which is initialized in this function only, so no fin
347      *       can can make its way in while we release the lock.
348      * (iv)  Remainders for sp
349      *       all sp operations are logged in dpc_flags. for the same reason as (iii) no sp commands can
350      *       make their way in during this fp-completion, all sp-processing after will relate to this point in time.
351      */
352 
353     con->dpc_info.snapshot_flags = con->dpc_info.dpc_flags;
354     con->dpc_info.dpc_flags = 0;
355 
356     /* complete buffers to client */
357     if (con->dpc_info.dpc_completed_tail != NULL) {
358         lm_tcp_complete_bufs(pdev, tcp, con);
359     }
360 
361     /* Graceful Disconnect */
362     if (con->dpc_info.snapshot_flags & LM_TCP_DPC_FIN_CMP) {
363         con->dpc_info.snapshot_flags &= ~LM_TCP_DPC_FIN_CMP;
364         lm_tcp_tx_graceful_disconnect_complete(pdev, con->tcp_state);
365     }
366 
367 }
368 
lm_tcp_tx_process_cqe(lm_device_t * pdev,struct toe_tx_cqe * cqe,lm_tcp_state_t * tcp)369 void lm_tcp_tx_process_cqe(
370     lm_device_t        * pdev,
371     struct toe_tx_cqe  * cqe,
372     lm_tcp_state_t     * tcp
373     )
374 {
375     enum toe_sq_opcode_type cmd;
376 
377     /* get the cmd from cqe */
378     cmd = ((cqe->params & TOE_TX_CQE_COMPLETION_OPCODE) >> TOE_TX_CQE_COMPLETION_OPCODE_SHIFT);
379 
380     DbgMessage(pdev, INFORMl4tx, "###lm_tcp_tx_process_cqe cid=%d cmd=%d\n", tcp->cid, cmd);
381     DbgBreakIf( ! (pdev && tcp) );
382     /* Check that the cqe len make sense, we could have got here by chance... */
383     DbgBreakIfAll(cqe->len & 0xc0000000); /* two upper bits on show a completion larger than 1GB - a bit odd...*/
384 
385     /* Three types of completios: fast-path, reset-recv, ramrod-cmp. All completions may have a
386      * fast-path part (nbytes completed) which will be handled in any case that cqe->len > 0 */
387 
388     /* complete data if anything needs to be complete */
389     if (cqe->len &&
390         ((tcp->tx_con->dpc_info.dpc_flags & LM_TCP_DPC_RESET_RECV /* RST recv on this DPC on a previous CQE */ ) ||
391          (tcp->tx_con->flags & TCP_REMOTE_RST_RECEIVED /* RST recv on previous DPC */ )))
392     {
393         /* 10/28/08 - Since in exterme cases current FW may not complete all sent+acked bytes
394            on RST recv cqe and do so only later on one of the following ramrod completions,
395            we need to ignore this too late completed bytes thus we nullify cqe->len */
396         DbgBreakIf((cmd != RAMROD_OPCODE_TOE_RESET_SEND) &&
397                    (cmd != RAMROD_OPCODE_TOE_INVALIDATE) &&
398                    (cmd != RAMROD_OPCODE_TOE_EMPTY_RAMROD) &&
399                    (cmd != RAMROD_OPCODE_TOE_TERMINATE));
400         lm_tcp_tx_inc_trm_aborted_bytes(pdev, tcp, cqe->len);
401         cqe->len = 0;
402     }
403     if (cqe->len) {
404         DbgBreakIf(tcp->tx_con->dpc_info.dpc_comp_blocked);
405         lm_tcp_tx_cmp_process(pdev, tcp, cqe->len);
406     }
407 
408     switch(cmd) {
409     case CMP_OPCODE_TOE_TX_CMP:
410         break;
411     case CMP_OPCODE_TOE_RST_RCV:
412         tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RESET_RECV;
413         tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_RST_RECEIVED */
414         break;
415     case RAMROD_OPCODE_TOE_RESET_SEND:
416         DbgBreakIf(! tcp->sp_request);
417         DbgBreakIf(tcp->sp_request->type != SP_REQUEST_ABORTIVE_DISCONNECT);
418         tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
419         tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_RST_REQ_COMPLETED */
420         break;
421     case RAMROD_OPCODE_TOE_INVALIDATE:
422         DbgBreakIf(! tcp->sp_request);
423         DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INVALIDATE);
424         tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
425         tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_INV_REQ_COMPLETED */
426         break;
427     case RAMROD_OPCODE_TOE_TERMINATE:
428         DbgBreakIf(! tcp->sp_request);
429         DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE1_OFFLOAD);
430         tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
431         tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_TRM_REQ_COMPLETED */
432         break;
433     case RAMROD_OPCODE_TOE_EMPTY_RAMROD:
434         DbgBreakIf(cqe->len);
435         DbgBreakIf(! tcp->sp_request );
436         DbgBreakIf((tcp->sp_request->type != SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT) &&
437                    (tcp->sp_request->type != SP_REQUEST_PENDING_REMOTE_DISCONNECT) &&
438                    (tcp->sp_request->type != SP_REQUEST_PENDING_TX_RST));
439         tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
440         break;
441     default:
442         DbgMessage(pdev, FATAL, "unexpected tx cqe opcode=%d\n", cmd);
443         DbgBreakIfAll(TRUE);
444     }
445 }
446 
447 /** Description
448  *
449  * Assumptions
450  *   connections is initialzed with a dummy head.
451  */
lm_tcp_tx_process_cqes(lm_device_t * pdev,u8_t drv_toe_rss_id,s_list_t * connections)452 void lm_tcp_tx_process_cqes(lm_device_t *pdev, u8_t drv_toe_rss_id, s_list_t * connections)
453 {
454     lm_tcp_scq_t *scq;
455     struct toe_tx_cqe *cqe, *hist_cqe;
456     lm_tcp_state_t *tcp;
457     u32_t cid;
458     u32_t avg_dpc_cnt;
459     u16_t cq_new_idx;
460     u16_t cq_old_idx;
461     u16_t num_to_reproduce = 0;
462     u8_t defer_cqe;
463     MM_INIT_TCP_LOCK_HANDLE();
464 
465     DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_tx_process_cqes\n");
466 
467     scq = &pdev->toe_info.scqs[drv_toe_rss_id];
468     cq_new_idx = *(scq->hw_con_idx_ptr);
469     cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
470     DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
471 
472     /* save statistics */
473     scq->num_cqes_last_dpc = S16_SUB(cq_new_idx, cq_old_idx);
474     if (scq->num_cqes_last_dpc) { /* Exclude zeroed value from statistics*/
475         if(scq->max_cqes_per_dpc < scq->num_cqes_last_dpc) {
476             scq->max_cqes_per_dpc = scq->num_cqes_last_dpc;
477         }
478         /* we don't want to wrap around...*/
479         if ((scq->sum_cqes_last_x_dpcs + scq->num_cqes_last_dpc) < scq->sum_cqes_last_x_dpcs) {
480             scq->avg_dpc_cnt = 0;
481             scq->sum_cqes_last_x_dpcs = 0;
482         }
483         scq->sum_cqes_last_x_dpcs += scq->num_cqes_last_dpc;
484         scq->avg_dpc_cnt++;
485         avg_dpc_cnt = scq->avg_dpc_cnt;
486         if (avg_dpc_cnt) {
487             scq->avg_cqes_per_dpc = scq->sum_cqes_last_x_dpcs / avg_dpc_cnt;
488         } else {
489             scq->sum_cqes_last_x_dpcs = 0;
490         }
491     }
492 
493     while(cq_old_idx != cq_new_idx) {
494         DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
495 
496         /* get next consumed cqe */
497         cqe = lm_toe_bd_chain_consume_bd(&scq->bd_chain);
498         DbgBreakIf(!cqe);
499         num_to_reproduce++;
500 
501         /* get tcp state from cqe */
502         cid = SW_CID(((cqe->params & TOE_TX_CQE_CID) >> TOE_TX_CQE_CID_SHIFT));
503         tcp = lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, cid);
504         DbgBreakIf(!tcp);
505         /* save cqe in history_cqes */
506         hist_cqe = (struct toe_tx_cqe *)lm_tcp_qe_buffer_next_cqe_override(&tcp->tx_con->history_cqes);
507         *hist_cqe = *cqe;
508 
509         defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
510         if (defer_cqe) {
511             /* if we're deferring completions - just store the cqe and continue to the next one */
512             /* Return if we are still deferred (may have changed since initial check was w/o a lock */
513             mm_acquire_tcp_lock(pdev, tcp->tx_con);
514             /* check again under lock if we're deferred */
515             defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
516             if (defer_cqe) {
517                 tcp->tx_con->flags |= TCP_DEFERRED_PROCESSING;
518                 lm_tcp_tx_process_cqe(pdev, cqe, tcp);
519             }
520             mm_release_tcp_lock(pdev, tcp->tx_con);
521         }
522         if (!defer_cqe) {
523             /* connections will always be initialized to a dummy, so once a tcp connection is added to the
524              * list, it's link will be initialized to point to another link other than NULL */
525             if (s_list_next_entry(&tcp->tx_con->dpc_info.link) == NULL) {
526                 s_list_push_head(connections, &tcp->tx_con->dpc_info.link);
527             }
528             lm_tcp_tx_process_cqe(pdev, cqe, tcp);
529         }
530         cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
531         /* GilR 5/12/2006 - TODO - decide with Alon if reading the hw_con again is required */
532         //cq_new_idx = *(scq->hw_con_idx_ptr);
533     }
534 
535     /* The fact that we post the producer here before we've handled any slow-path completions assures that
536      * the sp-ring will always be updated AFTER the producer was. */
537     if (num_to_reproduce) {
538         lm_toe_bd_chain_bds_produced(&scq->bd_chain, num_to_reproduce);
539 
540         /* GilR 5/13/2006 - TBA - save some stats? */
541 
542         /* notify the fw of the prod of the SCQ */
543         LM_INTMEM_WRITE16(pdev, CSTORM_TOE_CQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)),
544                           lm_bd_chain_prod_idx(&scq->bd_chain),  BAR_CSTRORM_INTMEM);
545     }
546 }
547 
lm_toe_service_tx_intr(lm_device_t * pdev,u8_t drv_toe_rss_id)548 void lm_toe_service_tx_intr(lm_device_t *pdev, u8_t drv_toe_rss_id)
549 {
550     s_list_t         connections;
551     s_list_entry_t   dummy;
552     lm_tcp_con_t   * con;
553     lm_tcp_state_t * tcp;
554 
555     MM_INIT_TCP_LOCK_HANDLE();
556 
557     DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_tx_intr\n");
558     DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
559 
560     s_list_clear(&connections);
561     s_list_push_head(&connections, &dummy);
562     /* process the cqes and initialize connections with all the connections that appeared
563      * in the DPC */
564     lm_tcp_tx_process_cqes(pdev,drv_toe_rss_id,&connections);
565 
566     /* complete the fp/sp parts of the connections remember to ignore the last one */
567     con = (lm_tcp_con_t *)s_list_peek_head(&connections);
568     tcp = con->tcp_state;
569     while (s_list_next_entry(&con->dpc_info.link) != NULL) {
570         mm_acquire_tcp_lock(pdev, con);
571         lm_tcp_tx_complete_tcp_fp(pdev, con->tcp_state, con);
572         mm_release_tcp_lock(pdev, con);
573         con = (lm_tcp_con_t *)s_list_next_entry(&con->dpc_info.link);
574         tcp = con->tcp_state;
575     }
576 
577     /* SP : traverse the connections. remember to ignore the last one */
578     con = (lm_tcp_con_t *)s_list_pop_head(&connections);
579     s_list_next_entry(&con->dpc_info.link) = NULL;
580     tcp = con->tcp_state;
581     while (s_list_entry_cnt(&connections) > 0) {
582         /* we access snapshot and not dpc, since once the dpc_flags were copied
583          * to snapshot they were zeroized */
584         if (con->dpc_info.snapshot_flags) {
585             lm_tcp_tx_complete_tcp_sp(pdev, tcp, con);
586         }
587         con = (lm_tcp_con_t *)s_list_pop_head(&connections);
588         s_list_next_entry(&con->dpc_info.link) = NULL;
589         tcp = con->tcp_state;
590     }
591 
592 }
593 
lm_tcp_graceful_disconnect(IN lm_device_t * pdev,IN lm_tcp_state_t * tcp_state)594 lm_status_t lm_tcp_graceful_disconnect(
595     IN lm_device_t          * pdev,
596     IN lm_tcp_state_t       * tcp_state
597 )
598 {
599     struct toe_tx_bd *tx_bd;
600     lm_tcp_con_t     *tcp_con = tcp_state->tx_con;
601     u16_t old_prod, new_prod;
602     u32_t dbg_bytes_prod_seq;
603 
604     DbgMessage(pdev, INFORMl4tx, "###lm_tcp_graceful_disconnect\n");
605 
606     if ( tcp_con->flags & TCP_TX_POST_BLOCKED ) {
607         return LM_STATUS_CONNECTION_CLOSED;
608     }
609 
610     DbgBreakIf( (tcp_con->app_buf_bytes_acc_post != 0) ||
611                 (tcp_con->db_more_bytes != 0) ||
612                 (tcp_con->db_more_bds != 0) ||
613                 (tcp_con->u.tx.bds_without_comp_flag != 0)
614                 );
615 
616     old_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
617 
618     /* Post FIN BD on Tx chain */
619     tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(&(tcp_con->bd_chain));
620     tx_bd->flags = TOE_TX_BD_FIN;  /* Vladz: Pay attention when u move this
621                                              line - there is an assignment to flags, NOT bitwise OR */
622     tx_bd->flags |= TOE_TX_BD_NOTIFY;
623     tx_bd->size = 1;
624     /* For a safety */
625     tx_bd->addr_hi = tx_bd->addr_lo = 0;
626 
627     dbg_bytes_prod_seq = tcp_con->db_data.tx->bytes_prod_seq + tcp_con->db_more_bytes;
628     dbg_bytes_prod_seq += tx_bd->size;
629     tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
630 
631     new_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
632     DbgBreakIf(S16_SUB(new_prod, old_prod) >= 3);
633     DbgBreakIf(S16_SUB(new_prod, old_prod) <= 0);
634 
635     DbgBreakIf(tcp_con->flags & TCP_FIN_REQ_POSTED);
636     tcp_con->flags |= TCP_FIN_REQ_POSTED;
637 
638     /* Update fin request time, if not already set by the caller */
639     if (!tcp_state->tcp_state_calc.fin_request_time) {
640         tcp_state->tcp_state_calc.fin_request_time = mm_get_current_time(pdev);
641         if (tcp_state->tcp_state_calc.fin_request_time == tcp_state->tcp_state_calc.fin_reception_time){
642             tcp_state->tcp_state_calc.fin_reception_time -= 1;
643         }
644     }
645 
646     /* Doorbell FIN */
647     _lm_tcp_tx_write_db(pdev, tcp_con, tcp_state->cid, 0, (u16_t)S16_SUB(new_prod, old_prod), 1);
648 
649     /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */
650     DbgBreakIf(S64_SUB(tcp_con->bytes_post_cnt + 1, tcp_con->bytes_comp_cnt) < 0);
651     tcp_con->bytes_post_cnt++;
652     tcp_con->fp_db_cnt++;
653 
654     return LM_STATUS_SUCCESS;
655 }
656 
657 
658