1
2 #include "lm5710.h"
3 #include "bd_chain.h"
4 #include "command.h"
5 #include "context.h"
6 #include "lm_l4fp.h"
7 #include "lm_l4sp.h"
8 #include "mm_l4if.h"
9
10
11 /* TODO: remove this temporary solution for solaris / linux compilation conflict, linux needs the
12 * first option, solaris the latter */
13 #if defined(__LINUX) || defined(__LITTLE_ENDIAN)
14 #define TOE_TX_INIT_ZERO {{0}}
15 #else
16 #define TOE_TX_INIT_ZERO {0}
17 #endif
18
19 #define TOE_TX_DOORBELL(pdev,cid) do{\
20 struct doorbell db = TOE_TX_INIT_ZERO;\
21 db.header.data |= (TOE_CONNECTION_TYPE << DOORBELL_HDR_T_CONN_TYPE_SHIFT);\
22 DOORBELL((pdev), (cid), *((u32_t *)&db));\
23 } while(0)
24
25 static __inline void _lm_tcp_tx_write_db(
26 lm_device_t * pdev,
27 lm_tcp_con_t * tx_con,
28 u32_t cid,
29 u32_t nbytes,
30 u16_t nbds,
31 u8_t fin)
32 {
33 volatile struct toe_tx_db_data *db_data = tx_con->db_data.tx;
34
35 db_data->bds_prod += nbds; /* nbds should be written before nbytes (FW assumption) */
36 DbgBreakIf((db_data->bds_prod & 0xff) == 0);
37 db_data->bytes_prod_seq += nbytes;
38
39 if(fin) {
40 DbgBreakIf(db_data->flags & (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT));
41 db_data->flags |= (TOE_TX_DB_DATA_FIN << TOE_TX_DB_DATA_FIN_SHIFT);
42 }
43
44 if (!(tx_con->flags & TCP_TX_DB_BLOCKED)) {
45 DbgMessage(pdev, INFORMl4tx,
46 "ringing tx doorbell: cid=%d, (nbytes+=%d, nbds+=%d, fin=%d)\n",
47 cid, nbytes, nbds, fin);
48 TOE_TX_DOORBELL(pdev, cid);
49 }
50 }
51
52 static __inline void lm_tcp_tx_write_db(
53 lm_device_t *pdev,
54 lm_tcp_state_t *tcp,
55 u8_t post_end)
56 {
57 lm_tcp_con_t *tx_con = tcp->tx_con;
58
59 /* define a policy for ringing the doorbell */
60 #define MAX_BYTES_PER_TX_DB 0xffff
61 #define MAX_BDS_PER_TX_DB 64
62
63 if (post_end ||
64 tx_con->db_more_bytes >= MAX_BYTES_PER_TX_DB ||
65 tx_con->db_more_bds >= MAX_BDS_PER_TX_DB) {
66 _lm_tcp_tx_write_db(pdev, tx_con, tcp->cid, tx_con->db_more_bytes, tx_con->db_more_bds, 0);
67
68 /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */
69 DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt + tx_con->db_more_bytes, tx_con->bytes_comp_cnt) < 0);
70 tx_con->bytes_post_cnt += tx_con->db_more_bytes;
71 tx_con->buffer_post_cnt += tx_con->db_more_bufs;
72 tx_con->db_more_bytes = tx_con->db_more_bds = tx_con->db_more_bufs = 0;
73 tx_con->fp_db_cnt++;
74 } else {
75 DbgMessage(pdev, INFORMl4tx,
76 "skipped doorbell ringing for cid=%d\n", tcp->cid);
77 }
78 }
79
80 /** Description:
81 * Post a single tcp buffer to the Tx bd chain
82 * Assumptions:
83 * - caller initiated tcp_buf->flags field with BUFFER_START/BUFFER_END appropriately
84 * Returns:
85 * - SUCCESS - tcp buf was successfully attached to the bd chain
86 * - RESOURCE - not enough available BDs on bd chain for given tcp buf
87 * - CONNECTION_CLOSED - whenever connection's flag are marked as 'POST BLOCKED' */
88 lm_status_t lm_tcp_tx_post_buf(
89 struct _lm_device_t *pdev,
90 lm_tcp_state_t *tcp,
91 lm_tcp_buffer_t *tcp_buf,
92 lm_frag_list_t *frag_list)
93 {
94 lm_tcp_con_t *tx_con;
95 lm_bd_chain_t *tx_chain;
96 struct toe_tx_bd *tx_bd = NULL ;
97 lm_frag_t *frag;
98 u32_t i, dbg_buf_size = 0;
99 u32_t dbg_bytes_prod_seq;
100 u16_t old_prod, new_prod;
101
102 DbgMessage(pdev, VERBOSEl4tx, "###lm_tcp_tx_post_buf\n");
103 DbgBreakIf(!(pdev && tcp && tcp_buf && frag_list));
104 DbgBreakIf(tcp->cid && (tcp != lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, tcp->cid)));
105 DbgBreakIf(frag_list->cnt == 0);
106 tx_con = tcp->tx_con;
107 tx_chain = &tx_con->bd_chain;
108 frag = frag_list->frag_arr;
109
110 DbgBreakIf(tx_con->flags & TCP_FIN_REQ_POSTED);
111
112 /* check if tx con is already closed */
113 if(tx_con->flags & TCP_TX_POST_BLOCKED) {
114 DbgMessage(pdev, WARNl4tx, "post tx buf failed, posting is blocked (cid=%d, con->flags=%x)\n",
115 tcp->cid, tx_con->flags);
116 return LM_STATUS_CONNECTION_CLOSED;
117 }
118 /* check bd chain availability (including additional bd that should
119 * be kept available for future fin request) */
120 if(lm_bd_chain_avail_bds(tx_chain) < frag_list->cnt + 1) {
121 DbgMessage(pdev, INFORMl4tx, "post tx buf failed, tx chain is full (cid=%d, avail bds=%d, buf nfrags=%d)\n",
122 tcp->cid, lm_bd_chain_avail_bds(tx_chain), frag_list->cnt);
123
124 LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, tx_no_l4_bd);
125
126 if (tx_con->db_more_bds) {
127 /* if doorbell ringing was deferred (e.g. until an end of
128 * application buffer), it can no longer be deferred since
129 * the place in the bd chain is now required */
130 lm_tcp_tx_write_db(pdev, tcp, 1);
131 }
132 return LM_STATUS_RESOURCE;
133 }
134
135 old_prod = lm_bd_chain_prod_idx(tx_chain);
136
137 dbg_bytes_prod_seq = tx_con->db_data.tx->bytes_prod_seq + tx_con->db_more_bytes;
138 /* "attach" the frags to the bd chain */
139 for(i = 0; i < frag_list->cnt; i++, frag++) {
140 DbgBreakIf(frag->size > 0xffff || frag->size == 0); /* hw limit: each bd can point to a buffer with max size of 64KB */
141 tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(tx_chain);
142 tx_bd->addr_hi = frag->addr.as_u32.high;
143 tx_bd->addr_lo = frag->addr.as_u32.low;
144 tx_bd->flags = 0;
145 tx_bd->size = (u16_t)frag->size;
146 dbg_bytes_prod_seq += frag->size;
147 tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
148 dbg_buf_size += frag->size;
149
150 /* Support for FW Nagle Algorithm:
151 * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
152 */
153 if ((u32_t)frag_list->size >= tx_con->u.tx.mss) {
154 tx_bd->flags |= TOE_TX_BD_LARGE_IO;
155 }
156
157 DbgMessage(pdev, VERBOSEl4tx, "Setting Tx BD, addr_lo=0x%x, addr_hi=0x%x, size=%d\n",
158 tx_bd->addr_lo, tx_bd->addr_hi, tx_bd->size);
159 }
160
161 DbgBreakIf(frag_list->cnt > 0xffff);
162 tcp_buf->bd_used = frag_list->cnt & 0xffff;
163 tcp_buf->size = tcp_buf->more_to_comp = (u32_t)frag_list->size;
164 DbgBreakIf(tcp_buf->size != dbg_buf_size);
165
166 DbgBreakIf(!(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_START ?
167 tx_con->app_buf_bytes_acc_post == 0 :
168 tx_con->app_buf_bytes_acc_post > 0));
169 tx_con->app_buf_bytes_acc_post += tcp_buf->size;
170 tx_con->db_more_bytes += tcp_buf->size;
171 new_prod = lm_bd_chain_prod_idx(tx_chain);
172 DbgBreakIf(S16_SUB(new_prod, old_prod) < tcp_buf->bd_used);
173 tx_con->db_more_bds += S16_SUB(new_prod, old_prod);
174 tx_con->db_more_bufs++;
175
176 /* Support for FW Nagle Algorithm:
177 * This bit, is to be set for every bd which is part of a tcp buffer which is equal to or larger than an mss.
178 */
179 if (tcp_buf->size >= tx_con->u.tx.mss) {
180 tx_bd->flags |= TOE_TX_BD_LARGE_IO;
181 }
182
183 /* special care in case of last tcp buffer of an application buffer */
184 if(tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END) {
185 tcp_buf->app_buf_xferred = 0; /* just for safety */
186 tcp_buf->app_buf_size = tx_con->app_buf_bytes_acc_post;
187 tx_con->app_buf_bytes_acc_post = 0;
188
189 /* special care for the last bd: */
190 tx_bd->flags |= TOE_TX_BD_NOTIFY;
191 tx_con->u.tx.bds_without_comp_flag = 0;
192 tx_bd->flags |= TOE_TX_BD_PUSH;
193
194 DbgMessage(pdev, VERBOSEl4tx,
195 "Setting Tx BD, last bd of app buf, flags=%d\n", tx_bd->flags);
196 } else {
197 /* make sure there aren't 'too many' bds without completion flag */
198 tx_con->u.tx.bds_without_comp_flag += tcp_buf->bd_used;
199 if (tx_con->u.tx.bds_without_comp_flag > (tx_chain->capacity - MAX_FRAG_CNT_PER_TB)) {
200 tx_bd->flags |= TOE_TX_BD_NOTIFY;
201 tx_con->u.tx.bds_without_comp_flag = 0;
202 }
203 }
204
205 s_list_push_tail(&tx_con->active_tb_list, &tcp_buf->link);
206 tx_con->rq_nbytes += tcp_buf->size;
207 lm_tcp_tx_write_db(pdev, tcp, tcp_buf->flags & TCP_BUF_FLAG_L4_POST_END);
208
209 /* network reachability (NOT IMPLEMENTED):
210 if(lm_neigh_is_cache_entry_staled(tcp->path->neigh))
211 lm_neigh_indicate_staled_cache_entry(tcp->path->neigh);
212 */
213
214 DbgMessage(pdev, VERBOSEl4tx, "posted tx buf for cid=%d, buf size=%d, bd used=%d, buf flags=%x, app_buf_size=%d\n",
215 tcp->cid, tcp_buf->size, tcp_buf->bd_used, tcp_buf->flags, tcp_buf->app_buf_size);
216 DbgMessage(pdev, VERBOSEl4tx, "after posting tx buf, tx_con->active_tb_list=%d\n",
217 s_list_entry_cnt(&tx_con->active_tb_list));
218
219 return LM_STATUS_SUCCESS;
220 } /* lm_tcp_tx_post_buf */
221
222 /** Description
223 * indicates graceful disconnect completion to client.
224 * Assumtpions:
225 * tx-lock is taken by caller
226 */
227 static __inline void lm_tcp_tx_graceful_disconnect_complete(lm_device_t * pdev, lm_tcp_state_t * tcp)
228 {
229 u8_t ip_version;
230 DbgBreakIf(!s_list_is_empty(&tcp->tx_con->active_tb_list));
231 DbgBreakIf(tcp->tx_con->flags & TCP_FIN_REQ_COMPLETED);
232 tcp->tx_con->flags |= TCP_FIN_REQ_COMPLETED;
233 DbgMessage(pdev, INFORMl4tx, "fin request completed (cid=%d)\n", tcp->cid);
234 tcp->tcp_state_calc.fin_completed_time = mm_get_current_time(pdev);
235 if (!(tcp->tx_con->u.tx.flags & TCP_CON_FIN_REQ_LM_INTERNAL)) {
236 ip_version = (tcp->path->path_const.ip_version == IP_VERSION_IPV4)? STATS_IP_4_IDX : STATS_IP_6_IDX;
237 LM_COMMON_DRV_STATS_ATOMIC_INC_TOE(pdev, ipv[ip_version].out_fin);
238 mm_tcp_graceful_disconnect_done(pdev,tcp, LM_STATUS_SUCCESS);
239 }
240 }
241
242 void lm_tcp_tx_cmp_process(
243 struct _lm_device_t *pdev,
244 lm_tcp_state_t *tcp,
245 u32_t completed_bytes
246 )
247 {
248 lm_tcp_con_t *tx_con = tcp->tx_con;
249 u32_t actual_completed; /* number of bytes actually completed (could be different than completed in case of fin) */
250 MM_INIT_TCP_LOCK_HANDLE();
251
252 DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_app_cmp_process, cid=%d, completed_bytes=%d\n",
253 tcp->cid, completed_bytes);
254
255 DbgBreakIf(tx_con->flags & TCP_TX_COMP_BLOCKED);
256
257 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
258 mm_acquire_tcp_lock(pdev, tx_con);
259 }
260 tx_con->bytes_comp_cnt += completed_bytes;
261 DbgBreakIf(S64_SUB(tx_con->bytes_post_cnt, tx_con->bytes_comp_cnt) < 0);
262
263 DbgBreakIf(!completed_bytes);
264
265 actual_completed = lm_tcp_complete_nbytes(pdev, tcp, tcp->tx_con, completed_bytes, FALSE);
266
267 if (actual_completed != completed_bytes) {
268 DbgBreakIf(actual_completed > completed_bytes);
269 DbgBreakIf((completed_bytes - actual_completed) != 1);
270 DbgBreakIf(!(tx_con->flags & TCP_FIN_REQ_POSTED));
271 DbgBreakIf(tx_con->bytes_post_cnt != tx_con->bytes_comp_cnt);
272 /* fin completed */
273 tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_FIN_CMP;
274 tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_FIN_REQ_COMPLETED */
275 }
276
277 /* network reachability (NOT IMPLEMENTED):
278 lm_neigh_update_nic_reachability_time(tcp->path->neigh)
279 */
280 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
281 mm_release_tcp_lock(pdev, tx_con);
282 }
283
284 } /* lm_tcp_tx_app_cmp_process */
285
286 u8_t lm_toe_is_tx_completion(lm_device_t *pdev, u8_t drv_toe_rss_id)
287 {
288 u8_t result = FALSE;
289 lm_tcp_scq_t *scq = NULL;
290
291 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
292
293 scq = &pdev->toe_info.scqs[drv_toe_rss_id];
294
295 if ( scq->hw_con_idx_ptr &&
296 *scq->hw_con_idx_ptr != lm_bd_chain_cons_idx(&scq->bd_chain) )
297 {
298 result = TRUE;
299 }
300 DbgMessage(pdev, INFORMl4int, "lm_toe_is_tx_completion(): result is:%s\n", result? "TRUE" : "FALSE");
301
302 return result;
303 }
304
305 void lm_tcp_tx_inc_trm_aborted_bytes(
306 struct _lm_device_t *pdev,
307 lm_tcp_state_t *tcp,
308 u32_t aborted_bytes
309 )
310 {
311 lm_tcp_con_t *tx_con = tcp->tx_con;
312 MM_INIT_TCP_LOCK_HANDLE();
313
314 DbgMessage(pdev, VERBOSEl4tx, "##lm_tcp_tx_inc_aborted_count, cid=%d, aborted_bytes=%d\n",
315 tcp->cid, aborted_bytes);
316
317 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
318 mm_acquire_tcp_lock(pdev, tx_con);
319 }
320
321 tx_con->bytes_trm_aborted_cnt += aborted_bytes;
322
323 if (!(tx_con->flags & TCP_DEFERRED_PROCESSING)) {
324 mm_release_tcp_lock(pdev, tx_con);
325 }
326
327 } /* lm_tcp_tx_inc_aborted_count */
328
329 /** Description
330 * completes the fast-path operations for a certain connection
331 * Assumption:
332 * fp-tx lock is taken
333 */
334 void lm_tcp_tx_complete_tcp_fp(lm_device_t * pdev, lm_tcp_state_t * tcp, lm_tcp_con_t * con)
335 {
336 /**** Client completing : may result in lock-release *****/
337 /* during lock-release, due to this function being called from service_deferred, more
338 * cqes can be processed. We don't want to mix. This function is mutually exclusive, so
339 * any processing makes it's way to being completed by calling this function.
340 * the following define a "fast-path completion"
341 * (i) RQ buffers to be completed
342 * defined by dpc_completed_tail and are collected during lm_tcp_complete_bufs BEFORE lock
343 * is released, so no more buffer processing can make it's way into this buffer completion.
344 * (ii) Fin to be completed
345 * determined by the flags, since dpc_flags CAN be modified during processing we copy
346 * them to a snapshot_flags parameter, which is initialized in this function only, so no fin
347 * can can make its way in while we release the lock.
348 * (iv) Remainders for sp
349 * all sp operations are logged in dpc_flags. for the same reason as (iii) no sp commands can
350 * make their way in during this fp-completion, all sp-processing after will relate to this point in time.
351 */
352
353 con->dpc_info.snapshot_flags = con->dpc_info.dpc_flags;
354 con->dpc_info.dpc_flags = 0;
355
356 /* complete buffers to client */
357 if (con->dpc_info.dpc_completed_tail != NULL) {
358 lm_tcp_complete_bufs(pdev, tcp, con);
359 }
360
361 /* Graceful Disconnect */
362 if (con->dpc_info.snapshot_flags & LM_TCP_DPC_FIN_CMP) {
363 con->dpc_info.snapshot_flags &= ~LM_TCP_DPC_FIN_CMP;
364 lm_tcp_tx_graceful_disconnect_complete(pdev, con->tcp_state);
365 }
366
367 }
368
369 void lm_tcp_tx_process_cqe(
370 lm_device_t * pdev,
371 struct toe_tx_cqe * cqe,
372 lm_tcp_state_t * tcp
373 )
374 {
375 enum toe_sq_opcode_type cmd;
376
377 /* get the cmd from cqe */
378 cmd = ((cqe->params & TOE_TX_CQE_COMPLETION_OPCODE) >> TOE_TX_CQE_COMPLETION_OPCODE_SHIFT);
379
380 DbgMessage(pdev, INFORMl4tx, "###lm_tcp_tx_process_cqe cid=%d cmd=%d\n", tcp->cid, cmd);
381 DbgBreakIf( ! (pdev && tcp) );
382 /* Check that the cqe len make sense, we could have got here by chance... */
383 DbgBreakIfAll(cqe->len & 0xc0000000); /* two upper bits on show a completion larger than 1GB - a bit odd...*/
384
385 /* Three types of completios: fast-path, reset-recv, ramrod-cmp. All completions may have a
386 * fast-path part (nbytes completed) which will be handled in any case that cqe->len > 0 */
387
388 /* complete data if anything needs to be complete */
389 if (cqe->len &&
390 ((tcp->tx_con->dpc_info.dpc_flags & LM_TCP_DPC_RESET_RECV /* RST recv on this DPC on a previous CQE */ ) ||
391 (tcp->tx_con->flags & TCP_REMOTE_RST_RECEIVED /* RST recv on previous DPC */ )))
392 {
393 /* 10/28/08 - Since in exterme cases current FW may not complete all sent+acked bytes
394 on RST recv cqe and do so only later on one of the following ramrod completions,
395 we need to ignore this too late completed bytes thus we nullify cqe->len */
396 DbgBreakIf((cmd != RAMROD_OPCODE_TOE_RESET_SEND) &&
397 (cmd != RAMROD_OPCODE_TOE_INVALIDATE) &&
398 (cmd != RAMROD_OPCODE_TOE_EMPTY_RAMROD) &&
399 (cmd != RAMROD_OPCODE_TOE_TERMINATE));
400 lm_tcp_tx_inc_trm_aborted_bytes(pdev, tcp, cqe->len);
401 cqe->len = 0;
402 }
403 if (cqe->len) {
404 DbgBreakIf(tcp->tx_con->dpc_info.dpc_comp_blocked);
405 lm_tcp_tx_cmp_process(pdev, tcp, cqe->len);
406 }
407
408 switch(cmd) {
409 case CMP_OPCODE_TOE_TX_CMP:
410 break;
411 case CMP_OPCODE_TOE_RST_RCV:
412 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RESET_RECV;
413 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_REMOTE_RST_RECEIVED */
414 break;
415 case RAMROD_OPCODE_TOE_RESET_SEND:
416 DbgBreakIf(! tcp->sp_request);
417 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_ABORTIVE_DISCONNECT);
418 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
419 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_RST_REQ_COMPLETED */
420 break;
421 case RAMROD_OPCODE_TOE_INVALIDATE:
422 DbgBreakIf(! tcp->sp_request);
423 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_INVALIDATE);
424 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
425 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_INV_REQ_COMPLETED */
426 break;
427 case RAMROD_OPCODE_TOE_TERMINATE:
428 DbgBreakIf(! tcp->sp_request);
429 DbgBreakIf(tcp->sp_request->type != SP_REQUEST_TERMINATE1_OFFLOAD);
430 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
431 tcp->tx_con->dpc_info.dpc_comp_blocked = TRUE; /* TCP_TRM_REQ_COMPLETED */
432 break;
433 case RAMROD_OPCODE_TOE_EMPTY_RAMROD:
434 DbgBreakIf(cqe->len);
435 DbgBreakIf(! tcp->sp_request );
436 DbgBreakIf((tcp->sp_request->type != SP_REQUEST_PENDING_LOCAL_FIN_DISCONNECT) &&
437 (tcp->sp_request->type != SP_REQUEST_PENDING_REMOTE_DISCONNECT) &&
438 (tcp->sp_request->type != SP_REQUEST_PENDING_TX_RST));
439 tcp->tx_con->dpc_info.dpc_flags |= LM_TCP_DPC_RAMROD_CMP;
440 break;
441 default:
442 DbgMessage(pdev, FATAL, "unexpected tx cqe opcode=%d\n", cmd);
443 DbgBreakIfAll(TRUE);
444 }
445 }
446
447 /** Description
448 *
449 * Assumptions
450 * connections is initialzed with a dummy head.
451 */
452 void lm_tcp_tx_process_cqes(lm_device_t *pdev, u8_t drv_toe_rss_id, s_list_t * connections)
453 {
454 lm_tcp_scq_t *scq;
455 struct toe_tx_cqe *cqe, *hist_cqe;
456 lm_tcp_state_t *tcp;
457 u32_t cid;
458 u32_t avg_dpc_cnt;
459 u16_t cq_new_idx;
460 u16_t cq_old_idx;
461 u16_t num_to_reproduce = 0;
462 u8_t defer_cqe;
463 MM_INIT_TCP_LOCK_HANDLE();
464
465 DbgMessage(pdev, VERBOSEl4int , "###lm_tcp_tx_process_cqes\n");
466
467 scq = &pdev->toe_info.scqs[drv_toe_rss_id];
468 cq_new_idx = *(scq->hw_con_idx_ptr);
469 cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
470 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
471
472 /* save statistics */
473 scq->num_cqes_last_dpc = S16_SUB(cq_new_idx, cq_old_idx);
474 if (scq->num_cqes_last_dpc) { /* Exclude zeroed value from statistics*/
475 if(scq->max_cqes_per_dpc < scq->num_cqes_last_dpc) {
476 scq->max_cqes_per_dpc = scq->num_cqes_last_dpc;
477 }
478 /* we don't want to wrap around...*/
479 if ((scq->sum_cqes_last_x_dpcs + scq->num_cqes_last_dpc) < scq->sum_cqes_last_x_dpcs) {
480 scq->avg_dpc_cnt = 0;
481 scq->sum_cqes_last_x_dpcs = 0;
482 }
483 scq->sum_cqes_last_x_dpcs += scq->num_cqes_last_dpc;
484 scq->avg_dpc_cnt++;
485 avg_dpc_cnt = scq->avg_dpc_cnt;
486 if (avg_dpc_cnt) {
487 scq->avg_cqes_per_dpc = scq->sum_cqes_last_x_dpcs / avg_dpc_cnt;
488 } else {
489 scq->sum_cqes_last_x_dpcs = 0;
490 }
491 }
492
493 while(cq_old_idx != cq_new_idx) {
494 DbgBreakIf(S16_SUB(cq_new_idx, cq_old_idx) <= 0);
495
496 /* get next consumed cqe */
497 cqe = lm_toe_bd_chain_consume_bd(&scq->bd_chain);
498 DbgBreakIf(!cqe);
499 num_to_reproduce++;
500
501 /* get tcp state from cqe */
502 cid = SW_CID(((cqe->params & TOE_TX_CQE_CID) >> TOE_TX_CQE_CID_SHIFT));
503 tcp = lm_cid_cookie(pdev, TOE_CONNECTION_TYPE, cid);
504 DbgBreakIf(!tcp);
505 /* save cqe in history_cqes */
506 hist_cqe = (struct toe_tx_cqe *)lm_tcp_qe_buffer_next_cqe_override(&tcp->tx_con->history_cqes);
507 *hist_cqe = *cqe;
508
509 defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
510 if (defer_cqe) {
511 /* if we're deferring completions - just store the cqe and continue to the next one */
512 /* Return if we are still deferred (may have changed since initial check was w/o a lock */
513 mm_acquire_tcp_lock(pdev, tcp->tx_con);
514 /* check again under lock if we're deferred */
515 defer_cqe = ((tcp->tx_con->flags & TCP_TX_COMP_DEFERRED) == TCP_TX_COMP_DEFERRED);
516 if (defer_cqe) {
517 tcp->tx_con->flags |= TCP_DEFERRED_PROCESSING;
518 lm_tcp_tx_process_cqe(pdev, cqe, tcp);
519 }
520 mm_release_tcp_lock(pdev, tcp->tx_con);
521 }
522 if (!defer_cqe) {
523 /* connections will always be initialized to a dummy, so once a tcp connection is added to the
524 * list, it's link will be initialized to point to another link other than NULL */
525 if (s_list_next_entry(&tcp->tx_con->dpc_info.link) == NULL) {
526 s_list_push_head(connections, &tcp->tx_con->dpc_info.link);
527 }
528 lm_tcp_tx_process_cqe(pdev, cqe, tcp);
529 }
530 cq_old_idx = lm_bd_chain_cons_idx(&scq->bd_chain);
531 /* GilR 5/12/2006 - TODO - decide with Alon if reading the hw_con again is required */
532 //cq_new_idx = *(scq->hw_con_idx_ptr);
533 }
534
535 /* The fact that we post the producer here before we've handled any slow-path completions assures that
536 * the sp-ring will always be updated AFTER the producer was. */
537 if (num_to_reproduce) {
538 lm_toe_bd_chain_bds_produced(&scq->bd_chain, num_to_reproduce);
539
540 /* GilR 5/13/2006 - TBA - save some stats? */
541
542 /* notify the fw of the prod of the SCQ */
543 LM_INTMEM_WRITE16(pdev, CSTORM_TOE_CQ_PROD_OFFSET(LM_TOE_FW_RSS_ID(pdev,drv_toe_rss_id) , PORT_ID(pdev)),
544 lm_bd_chain_prod_idx(&scq->bd_chain), BAR_CSTRORM_INTMEM);
545 }
546 }
547
548 void lm_toe_service_tx_intr(lm_device_t *pdev, u8_t drv_toe_rss_id)
549 {
550 s_list_t connections;
551 s_list_entry_t dummy;
552 lm_tcp_con_t * con;
553 lm_tcp_state_t * tcp;
554
555 MM_INIT_TCP_LOCK_HANDLE();
556
557 DbgMessage(pdev, VERBOSEl4int , "###lm_toe_service_tx_intr\n");
558 DbgBreakIf(!(pdev && ARRSIZE(pdev->toe_info.scqs) > drv_toe_rss_id));
559
560 s_list_clear(&connections);
561 s_list_push_head(&connections, &dummy);
562 /* process the cqes and initialize connections with all the connections that appeared
563 * in the DPC */
564 lm_tcp_tx_process_cqes(pdev,drv_toe_rss_id,&connections);
565
566 /* complete the fp/sp parts of the connections remember to ignore the last one */
567 con = (lm_tcp_con_t *)s_list_peek_head(&connections);
568 tcp = con->tcp_state;
569 while (s_list_next_entry(&con->dpc_info.link) != NULL) {
570 mm_acquire_tcp_lock(pdev, con);
571 lm_tcp_tx_complete_tcp_fp(pdev, con->tcp_state, con);
572 mm_release_tcp_lock(pdev, con);
573 con = (lm_tcp_con_t *)s_list_next_entry(&con->dpc_info.link);
574 tcp = con->tcp_state;
575 }
576
577 /* SP : traverse the connections. remember to ignore the last one */
578 con = (lm_tcp_con_t *)s_list_pop_head(&connections);
579 s_list_next_entry(&con->dpc_info.link) = NULL;
580 tcp = con->tcp_state;
581 while (s_list_entry_cnt(&connections) > 0) {
582 /* we access snapshot and not dpc, since once the dpc_flags were copied
583 * to snapshot they were zeroized */
584 if (con->dpc_info.snapshot_flags) {
585 lm_tcp_tx_complete_tcp_sp(pdev, tcp, con);
586 }
587 con = (lm_tcp_con_t *)s_list_pop_head(&connections);
588 s_list_next_entry(&con->dpc_info.link) = NULL;
589 tcp = con->tcp_state;
590 }
591
592 }
593
594 lm_status_t lm_tcp_graceful_disconnect(
595 IN lm_device_t * pdev,
596 IN lm_tcp_state_t * tcp_state
597 )
598 {
599 struct toe_tx_bd *tx_bd;
600 lm_tcp_con_t *tcp_con = tcp_state->tx_con;
601 u16_t old_prod, new_prod;
602 u32_t dbg_bytes_prod_seq;
603
604 DbgMessage(pdev, INFORMl4tx, "###lm_tcp_graceful_disconnect\n");
605
606 if ( tcp_con->flags & TCP_TX_POST_BLOCKED ) {
607 return LM_STATUS_CONNECTION_CLOSED;
608 }
609
610 DbgBreakIf( (tcp_con->app_buf_bytes_acc_post != 0) ||
611 (tcp_con->db_more_bytes != 0) ||
612 (tcp_con->db_more_bds != 0) ||
613 (tcp_con->u.tx.bds_without_comp_flag != 0)
614 );
615
616 old_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
617
618 /* Post FIN BD on Tx chain */
619 tx_bd = (struct toe_tx_bd *)lm_toe_bd_chain_produce_bd(&(tcp_con->bd_chain));
620 tx_bd->flags = TOE_TX_BD_FIN; /* Vladz: Pay attention when u move this
621 line - there is an assignment to flags, NOT bitwise OR */
622 tx_bd->flags |= TOE_TX_BD_NOTIFY;
623 tx_bd->size = 1;
624 /* For a safety */
625 tx_bd->addr_hi = tx_bd->addr_lo = 0;
626
627 dbg_bytes_prod_seq = tcp_con->db_data.tx->bytes_prod_seq + tcp_con->db_more_bytes;
628 dbg_bytes_prod_seq += tx_bd->size;
629 tx_bd->nextBdStartSeq = dbg_bytes_prod_seq;
630
631 new_prod = lm_bd_chain_prod_idx(&(tcp_con->bd_chain));
632 DbgBreakIf(S16_SUB(new_prod, old_prod) >= 3);
633 DbgBreakIf(S16_SUB(new_prod, old_prod) <= 0);
634
635 DbgBreakIf(tcp_con->flags & TCP_FIN_REQ_POSTED);
636 tcp_con->flags |= TCP_FIN_REQ_POSTED;
637
638 /* Update fin request time, if not already set by the caller */
639 if (!tcp_state->tcp_state_calc.fin_request_time) {
640 tcp_state->tcp_state_calc.fin_request_time = mm_get_current_time(pdev);
641 if (tcp_state->tcp_state_calc.fin_request_time == tcp_state->tcp_state_calc.fin_reception_time){
642 tcp_state->tcp_state_calc.fin_reception_time -= 1;
643 }
644 }
645
646 /* Doorbell FIN */
647 _lm_tcp_tx_write_db(pdev, tcp_con, tcp_state->cid, 0, (u16_t)S16_SUB(new_prod, old_prod), 1);
648
649 /* assert if the new addition will make the cyclic counter post_cnt smaller than comp_cnt */
650 DbgBreakIf(S64_SUB(tcp_con->bytes_post_cnt + 1, tcp_con->bytes_comp_cnt) < 0);
651 tcp_con->bytes_post_cnt++;
652 tcp_con->fp_db_cnt++;
653
654 return LM_STATUS_SUCCESS;
655 }
656
657