238
239 /*
240 * In the error case, wc->opcode sometimes contains
241 * garbage
242 */
243 switch (send->s_opcode) {
244 case IBT_WRC_SEND:
245 if (send->s_rm)
246 rdsv3_ib_send_unmap_rm(ic, send,
247 wc->wc_status);
248 break;
249 case IBT_WRC_RDMAW:
250 case IBT_WRC_RDMAR:
251 /*
252 * Nothing to be done - the SG list will
253 * be unmapped
254 * when the SEND completes.
255 */
256 break;
257 default:
258 #ifndef __lock_lint
259 RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
260 "RDS/IB: %s: unexpected opcode "
261 "0x%x in WR!",
262 __func__, send->s_opcode);
263 #endif
264 break;
265 }
266
267 send->s_opcode = 0xdd;
268 if (send->s_queued + HZ/2 < jiffies)
269 rdsv3_ib_stats_inc(s_ib_tx_stalled);
270
271 /*
272 * If a RDMA operation produced an error, signal
273 * this right
274 * away. If we don't, the subsequent SEND that goes
275 * with this
276 * RDMA will be canceled with ERR_WFLUSH, and the
277 * application
278 * never learn that the RDMA failed.
279 */
280 if (wc->wc_status ==
281 IBT_WC_REMOTE_ACCESS_ERR && send->s_op) {
282 struct rdsv3_message *rm;
283
570 uint32_t credit_alloc;
571 uint32_t posted;
572 uint32_t adv_credits = 0;
573 int send_flags = 0;
574 int sent;
575 int ret;
576 int flow_controlled = 0;
577
578 RDSV3_DPRINTF4("rdsv3_ib_xmit", "conn: %p, rm: %p", conn, rm);
579
580 ASSERT(!(off % RDSV3_FRAG_SIZE));
581 ASSERT(!(hdr_off != 0 && hdr_off != sizeof (struct rdsv3_header)));
582
583 /* Do not send cong updates to IB loopback */
584 if (conn->c_loopback &&
585 rm->m_inc.i_hdr.h_flags & RDSV3_FLAG_CONG_BITMAP) {
586 rdsv3_cong_map_updated(conn->c_fcong, ~(uint64_t)0);
587 return (sizeof (struct rdsv3_header) + RDSV3_CONG_MAP_BYTES);
588 }
589
590 #ifndef __lock_lint
591 /* FIXME we may overallocate here */
592 if (ntohl(rm->m_inc.i_hdr.h_len) == 0)
593 i = 1;
594 else
595 i = ceil(ntohl(rm->m_inc.i_hdr.h_len), RDSV3_FRAG_SIZE);
596 #endif
597
598 work_alloc = rdsv3_ib_ring_alloc(&ic->i_send_ring, i, &pos);
599 if (work_alloc != i) {
600 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
601 set_bit(RDSV3_LL_SEND_FULL, &conn->c_flags);
602 rdsv3_ib_stats_inc(s_ib_tx_ring_full);
603 ret = -ENOMEM;
604 goto out;
605 }
606
607 credit_alloc = work_alloc;
608 if (ic->i_flowctl) {
609 credit_alloc = rdsv3_ib_send_grab_credits(ic, work_alloc,
610 &posted, 0);
611 adv_credits += posted;
612 if (credit_alloc < work_alloc) {
613 rdsv3_ib_ring_unalloc(&ic->i_send_ring,
614 work_alloc - credit_alloc);
615 work_alloc = credit_alloc;
616 flow_controlled++;
|
238
239 /*
240 * In the error case, wc->opcode sometimes contains
241 * garbage
242 */
243 switch (send->s_opcode) {
244 case IBT_WRC_SEND:
245 if (send->s_rm)
246 rdsv3_ib_send_unmap_rm(ic, send,
247 wc->wc_status);
248 break;
249 case IBT_WRC_RDMAW:
250 case IBT_WRC_RDMAR:
251 /*
252 * Nothing to be done - the SG list will
253 * be unmapped
254 * when the SEND completes.
255 */
256 break;
257 default:
258 RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
259 "RDS/IB: %s: unexpected opcode "
260 "0x%x in WR!",
261 __func__, send->s_opcode);
262 break;
263 }
264
265 send->s_opcode = 0xdd;
266 if (send->s_queued + HZ/2 < jiffies)
267 rdsv3_ib_stats_inc(s_ib_tx_stalled);
268
269 /*
270 * If a RDMA operation produced an error, signal
271 * this right
272 * away. If we don't, the subsequent SEND that goes
273 * with this
274 * RDMA will be canceled with ERR_WFLUSH, and the
275 * application
276 * never learn that the RDMA failed.
277 */
278 if (wc->wc_status ==
279 IBT_WC_REMOTE_ACCESS_ERR && send->s_op) {
280 struct rdsv3_message *rm;
281
568 uint32_t credit_alloc;
569 uint32_t posted;
570 uint32_t adv_credits = 0;
571 int send_flags = 0;
572 int sent;
573 int ret;
574 int flow_controlled = 0;
575
576 RDSV3_DPRINTF4("rdsv3_ib_xmit", "conn: %p, rm: %p", conn, rm);
577
578 ASSERT(!(off % RDSV3_FRAG_SIZE));
579 ASSERT(!(hdr_off != 0 && hdr_off != sizeof (struct rdsv3_header)));
580
581 /* Do not send cong updates to IB loopback */
582 if (conn->c_loopback &&
583 rm->m_inc.i_hdr.h_flags & RDSV3_FLAG_CONG_BITMAP) {
584 rdsv3_cong_map_updated(conn->c_fcong, ~(uint64_t)0);
585 return (sizeof (struct rdsv3_header) + RDSV3_CONG_MAP_BYTES);
586 }
587
588 /* FIXME we may overallocate here */
589 if (ntohl(rm->m_inc.i_hdr.h_len) == 0)
590 i = 1;
591 else
592 i = ceil(ntohl(rm->m_inc.i_hdr.h_len), RDSV3_FRAG_SIZE);
593
594 work_alloc = rdsv3_ib_ring_alloc(&ic->i_send_ring, i, &pos);
595 if (work_alloc != i) {
596 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
597 set_bit(RDSV3_LL_SEND_FULL, &conn->c_flags);
598 rdsv3_ib_stats_inc(s_ib_tx_ring_full);
599 ret = -ENOMEM;
600 goto out;
601 }
602
603 credit_alloc = work_alloc;
604 if (ic->i_flowctl) {
605 credit_alloc = rdsv3_ib_send_grab_credits(ic, work_alloc,
606 &posted, 0);
607 adv_credits += posted;
608 if (credit_alloc < work_alloc) {
609 rdsv3_ib_ring_unalloc(&ic->i_send_ring,
610 work_alloc - credit_alloc);
611 work_alloc = credit_alloc;
612 flow_controlled++;
|