Print this page
8368 remove warlock leftovers from usr/src/uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/clients/rdsv3/ib_send.c
+++ new/usr/src/uts/common/io/ib/clients/rdsv3/ib_send.c
1 1 /*
2 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
3 3 */
4 4
5 5 /*
6 6 * This file contains code imported from the OFED rds source file ib_send.c
7 7 * Oracle elects to have and use the contents of ib_send.c under and governed
8 8 * by the OpenIB.org BSD license (see below for full license text). However,
9 9 * the following notice accompanied the original version of this file:
10 10 */
11 11
12 12 /*
13 13 * Copyright (c) 2006 Oracle. All rights reserved.
14 14 *
15 15 * This software is available to you under a choice of one of two
16 16 * licenses. You may choose to be licensed under the terms of the GNU
17 17 * General Public License (GPL) Version 2, available from the file
18 18 * COPYING in the main directory of this source tree, or the
19 19 * OpenIB.org BSD license below:
20 20 *
21 21 * Redistribution and use in source and binary forms, with or
22 22 * without modification, are permitted provided that the following
23 23 * conditions are met:
24 24 *
25 25 * - Redistributions of source code must retain the above
26 26 * copyright notice, this list of conditions and the following
27 27 * disclaimer.
28 28 *
29 29 * - Redistributions in binary form must reproduce the above
30 30 * copyright notice, this list of conditions and the following
31 31 * disclaimer in the documentation and/or other materials
32 32 * provided with the distribution.
33 33 *
34 34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35 35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36 36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37 37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38 38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39 39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40 40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41 41 * SOFTWARE.
42 42 *
43 43 */
44 44 #include <sys/rds.h>
45 45
46 46 #include <sys/ib/clients/rdsv3/rdsv3.h>
47 47 #include <sys/ib/clients/rdsv3/rdma.h>
48 48 #include <sys/ib/clients/rdsv3/ib.h>
49 49 #include <sys/ib/clients/rdsv3/rdsv3_debug.h>
50 50
51 51 static void
52 52 rdsv3_ib_send_rdma_complete(struct rdsv3_message *rm,
53 53 int wc_status)
54 54 {
55 55 int notify_status;
56 56
57 57 RDSV3_DPRINTF4("rdsv3_ib_send_rdma_complete", "rm: %p, wc_status: %d",
58 58 rm, wc_status);
59 59
60 60 switch (wc_status) {
61 61 case IBT_WC_WR_FLUSHED_ERR:
62 62 return;
63 63
64 64 case IBT_WC_SUCCESS:
65 65 notify_status = RDS_RDMA_SUCCESS;
66 66 break;
67 67
68 68 case IBT_WC_REMOTE_ACCESS_ERR:
69 69 notify_status = RDS_RDMA_REMOTE_ERROR;
70 70 break;
71 71
72 72 default:
73 73 notify_status = RDS_RDMA_OTHER_ERROR;
74 74 break;
75 75 }
76 76 rdsv3_rdma_send_complete(rm, notify_status);
77 77
78 78 RDSV3_DPRINTF4("rdsv3_ib_send_rdma_complete", "rm: %p, wc_status: %d",
79 79 rm, wc_status);
80 80 }
81 81
82 82 static void rdsv3_ib_dma_unmap_sg_rdma(struct ib_device *dev,
83 83 uint_t num, struct rdsv3_rdma_sg scat[]);
84 84
85 85 void
86 86 rdsv3_ib_send_unmap_rdma(struct rdsv3_ib_connection *ic,
87 87 struct rdsv3_rdma_op *op)
88 88 {
89 89 RDSV3_DPRINTF4("rdsv3_ib_send_unmap_rdma", "ic: %p, op: %p", ic, op);
90 90 if (op->r_mapped) {
91 91 op->r_mapped = 0;
92 92 if (ic->i_cm_id) {
93 93 rdsv3_ib_dma_unmap_sg_rdma(ic->i_cm_id->device,
94 94 op->r_nents, op->r_rdma_sg);
95 95 } else {
96 96 rdsv3_ib_dma_unmap_sg_rdma((struct ib_device *)NULL,
97 97 op->r_nents, op->r_rdma_sg);
98 98 }
99 99 }
100 100 }
101 101
102 102 static void
103 103 rdsv3_ib_send_unmap_rm(struct rdsv3_ib_connection *ic,
104 104 struct rdsv3_ib_send_work *send,
105 105 int wc_status)
106 106 {
107 107 struct rdsv3_message *rm = send->s_rm;
108 108
109 109 RDSV3_DPRINTF4("rdsv3_ib_send_unmap_rm", "ic %p send %p rm %p\n",
110 110 ic, send, rm);
111 111
112 112 mutex_enter(&rm->m_rs_lock);
113 113 if (rm->m_count) {
114 114 rdsv3_ib_dma_unmap_sg(ic->i_cm_id->device,
115 115 rm->m_sg, rm->m_count);
116 116 rm->m_count = 0;
117 117 }
118 118 mutex_exit(&rm->m_rs_lock);
119 119
120 120 if (rm->m_rdma_op != NULL) {
121 121 rdsv3_ib_send_unmap_rdma(ic, rm->m_rdma_op);
122 122
123 123 /*
124 124 * If the user asked for a completion notification on this
125 125 * message, we can implement three different semantics:
126 126 * 1. Notify when we received the ACK on the RDS message
127 127 * that was queued with the RDMA. This provides reliable
128 128 * notification of RDMA status at the expense of a one-way
129 129 * packet delay.
130 130 * 2. Notify when the IB stack gives us the completion
131 131 * event for the RDMA operation.
132 132 * 3. Notify when the IB stack gives us the completion
133 133 * event for the accompanying RDS messages.
134 134 * Here, we implement approach #3. To implement approach #2,
135 135 * call rdsv3_rdma_send_complete from the cq_handler.
136 136 * To implement #1,
137 137 * don't call rdsv3_rdma_send_complete at all, and fall back to
138 138 * the notify
139 139 * handling in the ACK processing code.
140 140 *
141 141 * Note: There's no need to explicitly sync any RDMA buffers
142 142 * using
143 143 * ib_dma_sync_sg_for_cpu - the completion for the RDMA
144 144 * operation itself unmapped the RDMA buffers, which takes care
145 145 * of synching.
146 146 */
147 147 rdsv3_ib_send_rdma_complete(rm, wc_status);
148 148
149 149 if (rm->m_rdma_op->r_write)
150 150 rdsv3_stats_add(s_send_rdma_bytes,
151 151 rm->m_rdma_op->r_bytes);
152 152 else
153 153 rdsv3_stats_add(s_recv_rdma_bytes,
154 154 rm->m_rdma_op->r_bytes);
155 155 }
156 156
157 157 /*
158 158 * If anyone waited for this message to get flushed out, wake
159 159 * them up now
160 160 */
161 161 rdsv3_message_unmapped(rm);
162 162
163 163 rdsv3_message_put(rm);
164 164 send->s_rm = NULL;
165 165 }
166 166
167 167 void
168 168 rdsv3_ib_send_init_ring(struct rdsv3_ib_connection *ic)
169 169 {
170 170 struct rdsv3_ib_send_work *send;
171 171 uint32_t i;
172 172
173 173 RDSV3_DPRINTF4("rdsv3_ib_send_init_ring", "ic: %p", ic);
174 174
175 175 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
176 176 send->s_rm = NULL;
177 177 send->s_op = NULL;
178 178 }
179 179 }
180 180
181 181 void
182 182 rdsv3_ib_send_clear_ring(struct rdsv3_ib_connection *ic)
183 183 {
184 184 struct rdsv3_ib_send_work *send;
185 185 uint32_t i;
186 186
187 187 RDSV3_DPRINTF4("rdsv3_ib_send_clear_ring", "ic: %p", ic);
188 188
189 189 for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {
190 190 if (send->s_opcode == 0xdd)
191 191 continue;
192 192 if (send->s_rm)
193 193 rdsv3_ib_send_unmap_rm(ic, send, IBT_WC_WR_FLUSHED_ERR);
194 194 if (send->s_op)
195 195 rdsv3_ib_send_unmap_rdma(ic, send->s_op);
196 196 }
197 197
198 198 RDSV3_DPRINTF4("rdsv3_ib_send_clear_ring", "Return: ic: %p", ic);
199 199 }
200 200
201 201 /*
202 202 * The _oldest/_free ring operations here race cleanly with the alloc/unalloc
203 203 * operations performed in the send path. As the sender allocs and potentially
204 204 * unallocs the next free entry in the ring it doesn't alter which is
205 205 * the next to be freed, which is what this is concerned with.
206 206 */
207 207 void
208 208 rdsv3_ib_send_cqe_handler(struct rdsv3_ib_connection *ic, ibt_wc_t *wc)
209 209 {
210 210 struct rdsv3_connection *conn = ic->conn;
211 211 struct rdsv3_ib_send_work *send;
212 212 uint32_t completed, polled;
213 213 uint32_t oldest;
214 214 uint32_t i = 0;
215 215 int ret;
216 216
217 217 RDSV3_DPRINTF4("rdsv3_ib_send_cqe_handler",
218 218 "wc wc_id 0x%llx status %u byte_len %u imm_data %u\n",
219 219 (unsigned long long)wc->wc_id, wc->wc_status,
220 220 wc->wc_bytes_xfer, ntohl(wc->wc_immed_data));
221 221
222 222 rdsv3_ib_stats_inc(s_ib_tx_cq_event);
223 223
224 224 if (wc->wc_id == RDSV3_IB_ACK_WR_ID) {
225 225 if (ic->i_ack_queued + HZ/2 < jiffies)
226 226 rdsv3_ib_stats_inc(s_ib_tx_stalled);
227 227 rdsv3_ib_ack_send_complete(ic);
228 228 return;
229 229 }
230 230
231 231 oldest = rdsv3_ib_ring_oldest(&ic->i_send_ring);
232 232
233 233 completed = rdsv3_ib_ring_completed(&ic->i_send_ring,
234 234 (wc->wc_id & ~RDSV3_IB_SEND_OP), oldest);
235 235
236 236 for (i = 0; i < completed; i++) {
237 237 send = &ic->i_sends[oldest];
238 238
239 239 /*
240 240 * In the error case, wc->opcode sometimes contains
241 241 * garbage
242 242 */
243 243 switch (send->s_opcode) {
244 244 case IBT_WRC_SEND:
245 245 if (send->s_rm)
246 246 rdsv3_ib_send_unmap_rm(ic, send,
247 247 wc->wc_status);
|
↓ open down ↓ |
247 lines elided |
↑ open up ↑ |
248 248 break;
249 249 case IBT_WRC_RDMAW:
250 250 case IBT_WRC_RDMAR:
251 251 /*
252 252 * Nothing to be done - the SG list will
253 253 * be unmapped
254 254 * when the SEND completes.
255 255 */
256 256 break;
257 257 default:
258 -#ifndef __lock_lint
259 258 RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
260 259 "RDS/IB: %s: unexpected opcode "
261 260 "0x%x in WR!",
262 261 __func__, send->s_opcode);
263 -#endif
264 262 break;
265 263 }
266 264
267 265 send->s_opcode = 0xdd;
268 266 if (send->s_queued + HZ/2 < jiffies)
269 267 rdsv3_ib_stats_inc(s_ib_tx_stalled);
270 268
271 269 /*
272 270 * If a RDMA operation produced an error, signal
273 271 * this right
274 272 * away. If we don't, the subsequent SEND that goes
275 273 * with this
276 274 * RDMA will be canceled with ERR_WFLUSH, and the
277 275 * application
278 276 * never learn that the RDMA failed.
279 277 */
280 278 if (wc->wc_status ==
281 279 IBT_WC_REMOTE_ACCESS_ERR && send->s_op) {
282 280 struct rdsv3_message *rm;
283 281
284 282 rm = rdsv3_send_get_message(conn, send->s_op);
285 283 if (rm) {
286 284 if (rm->m_rdma_op != NULL)
287 285 rdsv3_ib_send_unmap_rdma(ic,
288 286 rm->m_rdma_op);
289 287 rdsv3_ib_send_rdma_complete(rm,
290 288 wc->wc_status);
291 289 rdsv3_message_put(rm);
292 290 }
293 291 }
294 292
295 293 oldest = (oldest + 1) % ic->i_send_ring.w_nr;
296 294 }
297 295
298 296 rdsv3_ib_ring_free(&ic->i_send_ring, completed);
299 297
300 298 clear_bit(RDSV3_LL_SEND_FULL, &conn->c_flags);
301 299
302 300 /* We expect errors as the qp is drained during shutdown */
303 301 if (wc->wc_status != IBT_WC_SUCCESS && rdsv3_conn_up(conn)) {
304 302 RDSV3_DPRINTF2("rdsv3_ib_send_cqe_handler",
305 303 "send completion on %u.%u.%u.%u "
306 304 "had status %u, disconnecting and reconnecting\n",
307 305 NIPQUAD(conn->c_faddr), wc->wc_status);
308 306 rdsv3_conn_drop(conn);
309 307 }
310 308
311 309 RDSV3_DPRINTF4("rdsv3_ib_send_cqe_handler", "Return: conn: %p", ic);
312 310 }
313 311
314 312 /*
315 313 * This is the main function for allocating credits when sending
316 314 * messages.
317 315 *
318 316 * Conceptually, we have two counters:
319 317 * - send credits: this tells us how many WRs we're allowed
320 318 * to submit without overruning the reciever's queue. For
321 319 * each SEND WR we post, we decrement this by one.
322 320 *
323 321 * - posted credits: this tells us how many WRs we recently
324 322 * posted to the receive queue. This value is transferred
325 323 * to the peer as a "credit update" in a RDS header field.
326 324 * Every time we transmit credits to the peer, we subtract
327 325 * the amount of transferred credits from this counter.
328 326 *
329 327 * It is essential that we avoid situations where both sides have
330 328 * exhausted their send credits, and are unable to send new credits
331 329 * to the peer. We achieve this by requiring that we send at least
332 330 * one credit update to the peer before exhausting our credits.
333 331 * When new credits arrive, we subtract one credit that is withheld
334 332 * until we've posted new buffers and are ready to transmit these
335 333 * credits (see rdsv3_ib_send_add_credits below).
336 334 *
337 335 * The RDS send code is essentially single-threaded; rdsv3_send_xmit
338 336 * grabs c_send_lock to ensure exclusive access to the send ring.
339 337 * However, the ACK sending code is independent and can race with
340 338 * message SENDs.
341 339 *
342 340 * In the send path, we need to update the counters for send credits
343 341 * and the counter of posted buffers atomically - when we use the
344 342 * last available credit, we cannot allow another thread to race us
345 343 * and grab the posted credits counter. Hence, we have to use a
346 344 * spinlock to protect the credit counter, or use atomics.
347 345 *
348 346 * Spinlocks shared between the send and the receive path are bad,
349 347 * because they create unnecessary delays. An early implementation
350 348 * using a spinlock showed a 5% degradation in throughput at some
351 349 * loads.
352 350 *
353 351 * This implementation avoids spinlocks completely, putting both
354 352 * counters into a single atomic, and updating that atomic using
355 353 * atomic_add (in the receive path, when receiving fresh credits),
356 354 * and using atomic_cmpxchg when updating the two counters.
357 355 */
358 356 int
359 357 rdsv3_ib_send_grab_credits(struct rdsv3_ib_connection *ic,
360 358 uint32_t wanted, uint32_t *adv_credits, int need_posted)
361 359 {
362 360 unsigned int avail, posted, got = 0, advertise;
363 361 long oldval, newval;
364 362
365 363 RDSV3_DPRINTF4("rdsv3_ib_send_grab_credits", "ic: %p, %d %d %d",
366 364 ic, wanted, *adv_credits, need_posted);
367 365
368 366 *adv_credits = 0;
369 367 if (!ic->i_flowctl)
370 368 return (wanted);
371 369
372 370 try_again:
373 371 advertise = 0;
374 372 oldval = newval = atomic_get(&ic->i_credits);
375 373 posted = IB_GET_POST_CREDITS(oldval);
376 374 avail = IB_GET_SEND_CREDITS(oldval);
377 375
378 376 RDSV3_DPRINTF5("rdsv3_ib_send_grab_credits",
379 377 "wanted (%u): credits=%u posted=%u\n", wanted, avail, posted);
380 378
381 379 /* The last credit must be used to send a credit update. */
382 380 if (avail && !posted)
383 381 avail--;
384 382
385 383 if (avail < wanted) {
386 384 struct rdsv3_connection *conn = ic->i_cm_id->context;
387 385
388 386 /* Oops, there aren't that many credits left! */
389 387 set_bit(RDSV3_LL_SEND_FULL, &conn->c_flags);
390 388 got = avail;
391 389 } else {
392 390 /* Sometimes you get what you want, lalala. */
393 391 got = wanted;
394 392 }
395 393 newval -= IB_SET_SEND_CREDITS(got);
396 394
397 395 /*
398 396 * If need_posted is non-zero, then the caller wants
399 397 * the posted regardless of whether any send credits are
400 398 * available.
401 399 */
402 400 if (posted && (got || need_posted)) {
403 401 advertise = min(posted, RDSV3_MAX_ADV_CREDIT);
404 402 newval -= IB_SET_POST_CREDITS(advertise);
405 403 }
406 404
407 405 /* Finally bill everything */
408 406 if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)
409 407 goto try_again;
410 408
411 409 *adv_credits = advertise;
412 410
413 411 RDSV3_DPRINTF4("rdsv3_ib_send_grab_credits", "ic: %p, %d %d %d",
414 412 ic, got, *adv_credits, need_posted);
415 413
416 414 return (got);
417 415 }
418 416
419 417 void
420 418 rdsv3_ib_send_add_credits(struct rdsv3_connection *conn, unsigned int credits)
421 419 {
422 420 struct rdsv3_ib_connection *ic = conn->c_transport_data;
423 421
424 422 if (credits == 0)
425 423 return;
426 424
427 425 RDSV3_DPRINTF5("rdsv3_ib_send_add_credits",
428 426 "credits (%u): current=%u%s\n",
429 427 credits,
430 428 IB_GET_SEND_CREDITS(atomic_get(&ic->i_credits)),
431 429 test_bit(RDSV3_LL_SEND_FULL, &conn->c_flags) ?
432 430 ", ll_send_full" : "");
433 431
434 432 atomic_add_32(&ic->i_credits, IB_SET_SEND_CREDITS(credits));
435 433 if (test_and_clear_bit(RDSV3_LL_SEND_FULL, &conn->c_flags))
436 434 rdsv3_queue_delayed_work(rdsv3_wq, &conn->c_send_w, 0);
437 435
438 436 ASSERT(!(IB_GET_SEND_CREDITS(credits) >= 16384));
439 437
440 438 rdsv3_ib_stats_inc(s_ib_rx_credit_updates);
441 439
442 440 RDSV3_DPRINTF4("rdsv3_ib_send_add_credits",
443 441 "Return: conn: %p, credits: %d",
444 442 conn, credits);
445 443 }
446 444
447 445 void
448 446 rdsv3_ib_advertise_credits(struct rdsv3_connection *conn, unsigned int posted)
449 447 {
450 448 struct rdsv3_ib_connection *ic = conn->c_transport_data;
451 449
452 450 RDSV3_DPRINTF4("rdsv3_ib_advertise_credits", "conn: %p, posted: %d",
453 451 conn, posted);
454 452
455 453 if (posted == 0)
456 454 return;
457 455
458 456 atomic_add_32(&ic->i_credits, IB_SET_POST_CREDITS(posted));
459 457
460 458 /*
461 459 * Decide whether to send an update to the peer now.
462 460 * If we would send a credit update for every single buffer we
463 461 * post, we would end up with an ACK storm (ACK arrives,
464 462 * consumes buffer, we refill the ring, send ACK to remote
465 463 * advertising the newly posted buffer... ad inf)
466 464 *
467 465 * Performance pretty much depends on how often we send
468 466 * credit updates - too frequent updates mean lots of ACKs.
469 467 * Too infrequent updates, and the peer will run out of
470 468 * credits and has to throttle.
471 469 * For the time being, 16 seems to be a good compromise.
472 470 */
473 471 if (IB_GET_POST_CREDITS(atomic_get(&ic->i_credits)) >= 16)
474 472 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
475 473 }
476 474
477 475 static inline void
478 476 rdsv3_ib_xmit_populate_wr(struct rdsv3_ib_connection *ic,
479 477 ibt_send_wr_t *wr, unsigned int pos,
480 478 struct rdsv3_scatterlist *scat, unsigned int off, unsigned int length,
481 479 int send_flags)
482 480 {
483 481 ibt_wr_ds_t *sge;
484 482
485 483 RDSV3_DPRINTF4("rdsv3_ib_xmit_populate_wr",
486 484 "ic: %p, wr: %p scat: %p %d %d %d %d",
487 485 ic, wr, scat, pos, off, length, send_flags);
488 486
489 487 wr->wr_id = pos | RDSV3_IB_SEND_OP;
490 488 wr->wr_trans = IBT_RC_SRV;
491 489 wr->wr_flags = send_flags;
492 490 wr->wr_opcode = IBT_WRC_SEND;
493 491
494 492 if (length != 0) {
495 493 int ix, len, assigned;
496 494 ibt_wr_ds_t *sgl;
497 495
498 496 ASSERT(length <= scat->length - off);
499 497
500 498 sgl = scat->sgl;
501 499 if (off != 0) {
502 500 /* find the right sgl to begin with */
503 501 while (sgl->ds_len <= off) {
504 502 off -= sgl->ds_len;
505 503 sgl++;
506 504 }
507 505 }
508 506
509 507 ix = 1; /* first data sgl is at 1 */
510 508 assigned = 0;
511 509 len = length;
512 510 do {
513 511 sge = &wr->wr_sgl[ix++];
514 512 sge->ds_va = sgl->ds_va + off;
515 513 assigned = min(len, sgl->ds_len - off);
516 514 sge->ds_len = assigned;
517 515 sge->ds_key = sgl->ds_key;
518 516 len -= assigned;
519 517 if (len != 0) {
520 518 sgl++;
521 519 off = 0;
522 520 }
523 521 } while (len > 0);
524 522
525 523 wr->wr_nds = ix;
526 524 } else {
527 525 /*
528 526 * We're sending a packet with no payload. There is only
529 527 * one SGE
530 528 */
531 529 wr->wr_nds = 1;
532 530 }
533 531
534 532 sge = &wr->wr_sgl[0];
535 533 sge->ds_va = ic->i_send_hdrs_dma + (pos * sizeof (struct rdsv3_header));
536 534 sge->ds_len = sizeof (struct rdsv3_header);
537 535 sge->ds_key = ic->i_mr->lkey;
538 536
539 537 RDSV3_DPRINTF4("rdsv3_ib_xmit_populate_wr",
540 538 "Return: ic: %p, wr: %p scat: %p", ic, wr, scat);
541 539 }
542 540
543 541 /*
544 542 * This can be called multiple times for a given message. The first time
545 543 * we see a message we map its scatterlist into the IB device so that
546 544 * we can provide that mapped address to the IB scatter gather entries
547 545 * in the IB work requests. We translate the scatterlist into a series
548 546 * of work requests that fragment the message. These work requests complete
549 547 * in order so we pass ownership of the message to the completion handler
550 548 * once we send the final fragment.
551 549 *
552 550 * The RDS core uses the c_send_lock to only enter this function once
553 551 * per connection. This makes sure that the tx ring alloc/unalloc pairs
554 552 * don't get out of sync and confuse the ring.
555 553 */
556 554 int
557 555 rdsv3_ib_xmit(struct rdsv3_connection *conn, struct rdsv3_message *rm,
558 556 unsigned int hdr_off, unsigned int sg, unsigned int off)
559 557 {
560 558 struct rdsv3_ib_connection *ic = conn->c_transport_data;
561 559 struct ib_device *dev = ic->i_cm_id->device;
562 560 struct rdsv3_ib_send_work *send = NULL;
563 561 struct rdsv3_ib_send_work *first;
564 562 struct rdsv3_ib_send_work *prev;
565 563 ibt_send_wr_t *wr;
566 564 struct rdsv3_scatterlist *scat;
567 565 uint32_t pos;
568 566 uint32_t i;
569 567 uint32_t work_alloc;
570 568 uint32_t credit_alloc;
571 569 uint32_t posted;
572 570 uint32_t adv_credits = 0;
573 571 int send_flags = 0;
574 572 int sent;
575 573 int ret;
576 574 int flow_controlled = 0;
577 575
578 576 RDSV3_DPRINTF4("rdsv3_ib_xmit", "conn: %p, rm: %p", conn, rm);
579 577
|
↓ open down ↓ |
306 lines elided |
↑ open up ↑ |
580 578 ASSERT(!(off % RDSV3_FRAG_SIZE));
581 579 ASSERT(!(hdr_off != 0 && hdr_off != sizeof (struct rdsv3_header)));
582 580
583 581 /* Do not send cong updates to IB loopback */
584 582 if (conn->c_loopback &&
585 583 rm->m_inc.i_hdr.h_flags & RDSV3_FLAG_CONG_BITMAP) {
586 584 rdsv3_cong_map_updated(conn->c_fcong, ~(uint64_t)0);
587 585 return (sizeof (struct rdsv3_header) + RDSV3_CONG_MAP_BYTES);
588 586 }
589 587
590 -#ifndef __lock_lint
591 588 /* FIXME we may overallocate here */
592 589 if (ntohl(rm->m_inc.i_hdr.h_len) == 0)
593 590 i = 1;
594 591 else
595 592 i = ceil(ntohl(rm->m_inc.i_hdr.h_len), RDSV3_FRAG_SIZE);
596 -#endif
597 593
598 594 work_alloc = rdsv3_ib_ring_alloc(&ic->i_send_ring, i, &pos);
599 595 if (work_alloc != i) {
600 596 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
601 597 set_bit(RDSV3_LL_SEND_FULL, &conn->c_flags);
602 598 rdsv3_ib_stats_inc(s_ib_tx_ring_full);
603 599 ret = -ENOMEM;
604 600 goto out;
605 601 }
606 602
607 603 credit_alloc = work_alloc;
608 604 if (ic->i_flowctl) {
609 605 credit_alloc = rdsv3_ib_send_grab_credits(ic, work_alloc,
610 606 &posted, 0);
611 607 adv_credits += posted;
612 608 if (credit_alloc < work_alloc) {
613 609 rdsv3_ib_ring_unalloc(&ic->i_send_ring,
614 610 work_alloc - credit_alloc);
615 611 work_alloc = credit_alloc;
616 612 flow_controlled++;
617 613 }
618 614 if (work_alloc == 0) {
619 615 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
620 616 rdsv3_ib_stats_inc(s_ib_tx_throttle);
621 617 ret = -ENOMEM;
622 618 goto out;
623 619 }
624 620 }
625 621
626 622 /* map the message the first time we see it */
627 623 if (ic->i_rm == NULL) {
628 624 /*
629 625 * printk(KERN_NOTICE
630 626 * "rdsv3_ib_xmit prep msg dport=%u flags=0x%x len=%d\n",
631 627 * be16_to_cpu(rm->m_inc.i_hdr.h_dport),
632 628 * rm->m_inc.i_hdr.h_flags,
633 629 * be32_to_cpu(rm->m_inc.i_hdr.h_len));
634 630 */
635 631 if (rm->m_nents) {
636 632 rm->m_count = rdsv3_ib_dma_map_sg(dev,
637 633 rm->m_sg, rm->m_nents);
638 634 RDSV3_DPRINTF5("rdsv3_ib_xmit",
639 635 "ic %p mapping rm %p: %d\n", ic, rm, rm->m_count);
640 636 if (rm->m_count == 0) {
641 637 rdsv3_ib_stats_inc(s_ib_tx_sg_mapping_failure);
642 638 rdsv3_ib_ring_unalloc(&ic->i_send_ring,
643 639 work_alloc);
644 640 ret = -ENOMEM; /* XXX ? */
645 641 RDSV3_DPRINTF2("rdsv3_ib_xmit",
646 642 "fail: ic %p mapping rm %p: %d\n",
647 643 ic, rm, rm->m_count);
648 644 goto out;
649 645 }
650 646 } else {
651 647 rm->m_count = 0;
652 648 }
653 649
654 650 ic->i_unsignaled_wrs = rdsv3_ib_sysctl_max_unsig_wrs;
655 651 ic->i_unsignaled_bytes = rdsv3_ib_sysctl_max_unsig_bytes;
656 652 rdsv3_message_addref(rm);
657 653 ic->i_rm = rm;
658 654
659 655 /* Finalize the header */
660 656 if (test_bit(RDSV3_MSG_ACK_REQUIRED, &rm->m_flags))
661 657 rm->m_inc.i_hdr.h_flags |= RDSV3_FLAG_ACK_REQUIRED;
662 658 if (test_bit(RDSV3_MSG_RETRANSMITTED, &rm->m_flags))
663 659 rm->m_inc.i_hdr.h_flags |= RDSV3_FLAG_RETRANSMITTED;
664 660
665 661 /*
666 662 * If it has a RDMA op, tell the peer we did it. This is
667 663 * used by the peer to release use-once RDMA MRs.
668 664 */
669 665 if (rm->m_rdma_op) {
670 666 struct rdsv3_ext_header_rdma ext_hdr;
671 667
672 668 ext_hdr.h_rdma_rkey = htonl(rm->m_rdma_op->r_key);
673 669 (void) rdsv3_message_add_extension(&rm->m_inc.i_hdr,
674 670 RDSV3_EXTHDR_RDMA, &ext_hdr,
675 671 sizeof (ext_hdr));
676 672 }
677 673 if (rm->m_rdma_cookie) {
678 674 (void) rdsv3_message_add_rdma_dest_extension(
679 675 &rm->m_inc.i_hdr,
680 676 rdsv3_rdma_cookie_key(rm->m_rdma_cookie),
681 677 rdsv3_rdma_cookie_offset(rm->m_rdma_cookie));
682 678 }
683 679
684 680 /*
685 681 * Note - rdsv3_ib_piggyb_ack clears the ACK_REQUIRED bit, so
686 682 * we should not do this unless we have a chance of at least
687 683 * sticking the header into the send ring. Which is why we
688 684 * should call rdsv3_ib_ring_alloc first.
689 685 */
690 686 rm->m_inc.i_hdr.h_ack = htonll(rdsv3_ib_piggyb_ack(ic));
691 687 rdsv3_message_make_checksum(&rm->m_inc.i_hdr);
692 688
693 689 /*
694 690 * Update adv_credits since we reset the ACK_REQUIRED bit.
695 691 */
696 692 (void) rdsv3_ib_send_grab_credits(ic, 0, &posted, 1);
697 693 adv_credits += posted;
698 694 ASSERT(adv_credits <= 255);
699 695 }
700 696
701 697 send = &ic->i_sends[pos];
702 698 first = send;
703 699 prev = NULL;
704 700 scat = &rm->m_sg[sg];
705 701 sent = 0;
706 702 i = 0;
707 703
708 704 /*
709 705 * Sometimes you want to put a fence between an RDMA
710 706 * READ and the following SEND.
711 707 * We could either do this all the time
712 708 * or when requested by the user. Right now, we let
713 709 * the application choose.
714 710 */
715 711 if (rm->m_rdma_op && rm->m_rdma_op->r_fence)
716 712 send_flags = IBT_WR_SEND_FENCE;
717 713
718 714 /*
719 715 * We could be copying the header into the unused tail of the page.
720 716 * That would need to be changed in the future when those pages might
721 717 * be mapped userspace pages or page cache pages. So instead we always
722 718 * use a second sge and our long-lived ring of mapped headers. We send
723 719 * the header after the data so that the data payload can be aligned on
724 720 * the receiver.
725 721 */
726 722
727 723 /* handle a 0-len message */
728 724 if (ntohl(rm->m_inc.i_hdr.h_len) == 0) {
729 725 wr = &ic->i_send_wrs[0];
730 726 rdsv3_ib_xmit_populate_wr(ic, wr, pos, NULL, 0, 0, send_flags);
731 727 send->s_queued = jiffies;
732 728 send->s_op = NULL;
733 729 send->s_opcode = wr->wr_opcode;
734 730 goto add_header;
735 731 }
736 732
737 733 /* if there's data reference it with a chain of work reqs */
738 734 for (; i < work_alloc && scat != &rm->m_sg[rm->m_count]; i++) {
739 735 unsigned int len;
740 736
741 737 send = &ic->i_sends[pos];
742 738
743 739 wr = &ic->i_send_wrs[i];
744 740 len = min(RDSV3_FRAG_SIZE,
745 741 rdsv3_ib_sg_dma_len(dev, scat) - off);
746 742 rdsv3_ib_xmit_populate_wr(ic, wr, pos, scat, off, len,
747 743 send_flags);
748 744 send->s_queued = jiffies;
749 745 send->s_op = NULL;
750 746 send->s_opcode = wr->wr_opcode;
751 747
752 748 /*
753 749 * We want to delay signaling completions just enough to get
754 750 * the batching benefits but not so much that we create dead
755 751 * time
756 752 * on the wire.
757 753 */
758 754 if (ic->i_unsignaled_wrs-- == 0) {
759 755 ic->i_unsignaled_wrs = rdsv3_ib_sysctl_max_unsig_wrs;
760 756 wr->wr_flags |=
761 757 IBT_WR_SEND_SIGNAL | IBT_WR_SEND_SOLICIT;
762 758 }
763 759
764 760 ic->i_unsignaled_bytes -= len;
765 761 if (ic->i_unsignaled_bytes <= 0) {
766 762 ic->i_unsignaled_bytes =
767 763 rdsv3_ib_sysctl_max_unsig_bytes;
768 764 wr->wr_flags |=
769 765 IBT_WR_SEND_SIGNAL | IBT_WR_SEND_SOLICIT;
770 766 }
771 767
772 768 /*
773 769 * Always signal the last one if we're stopping due to flow
774 770 * control.
775 771 */
776 772 if (flow_controlled && i == (work_alloc-1)) {
777 773 wr->wr_flags |=
778 774 IBT_WR_SEND_SIGNAL | IBT_WR_SEND_SOLICIT;
779 775 }
780 776
781 777 RDSV3_DPRINTF5("rdsv3_ib_xmit", "send %p wr %p num_sge %u \n",
782 778 send, wr, wr->wr_nds);
783 779
784 780 sent += len;
785 781 off += len;
786 782 if (off == rdsv3_ib_sg_dma_len(dev, scat)) {
787 783 scat++;
788 784 off = 0;
789 785 }
790 786
791 787 add_header:
792 788 /*
793 789 * Tack on the header after the data. The header SGE
794 790 * should already
795 791 * have been set up to point to the right header buffer.
796 792 */
797 793 (void) memcpy(&ic->i_send_hdrs[pos], &rm->m_inc.i_hdr,
798 794 sizeof (struct rdsv3_header));
799 795
800 796 if (0) {
801 797 struct rdsv3_header *hdr = &ic->i_send_hdrs[pos];
802 798
803 799 RDSV3_DPRINTF2("rdsv3_ib_xmit",
804 800 "send WR dport=%u flags=0x%x len=%d",
805 801 ntohs(hdr->h_dport),
806 802 hdr->h_flags,
807 803 ntohl(hdr->h_len));
808 804 }
809 805 if (adv_credits) {
810 806 struct rdsv3_header *hdr = &ic->i_send_hdrs[pos];
811 807
812 808 /* add credit and redo the header checksum */
813 809 hdr->h_credit = adv_credits;
814 810 rdsv3_message_make_checksum(hdr);
815 811 adv_credits = 0;
816 812 rdsv3_ib_stats_inc(s_ib_tx_credit_updates);
817 813 }
818 814
819 815 prev = send;
820 816
821 817 pos = (pos + 1) % ic->i_send_ring.w_nr;
822 818 }
823 819
824 820 /*
825 821 * Account the RDS header in the number of bytes we sent, but just once.
826 822 * The caller has no concept of fragmentation.
827 823 */
828 824 if (hdr_off == 0)
829 825 sent += sizeof (struct rdsv3_header);
830 826
831 827 /* if we finished the message then send completion owns it */
832 828 if (scat == &rm->m_sg[rm->m_count]) {
833 829 prev->s_rm = ic->i_rm;
834 830 wr->wr_flags |= IBT_WR_SEND_SIGNAL | IBT_WR_SEND_SOLICIT;
835 831 ic->i_rm = NULL;
836 832 }
837 833
838 834 if (i < work_alloc) {
839 835 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);
840 836 work_alloc = i;
841 837 }
842 838 if (ic->i_flowctl && i < credit_alloc)
843 839 rdsv3_ib_send_add_credits(conn, credit_alloc - i);
844 840
845 841 /* XXX need to worry about failed_wr and partial sends. */
846 842 ret = ibt_post_send(ib_get_ibt_channel_hdl(ic->i_cm_id),
847 843 ic->i_send_wrs, i, &posted);
848 844 if (posted != i) {
849 845 RDSV3_DPRINTF2("rdsv3_ib_xmit",
850 846 "ic %p first %p nwr: %d ret %d:%d",
851 847 ic, first, i, ret, posted);
852 848 }
853 849 if (ret) {
854 850 RDSV3_DPRINTF2("rdsv3_ib_xmit",
855 851 "RDS/IB: ib_post_send to %u.%u.%u.%u "
856 852 "returned %d\n", NIPQUAD(conn->c_faddr), ret);
857 853 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
858 854 if (prev->s_rm) {
859 855 ic->i_rm = prev->s_rm;
860 856 prev->s_rm = NULL;
861 857 }
862 858 RDSV3_DPRINTF2("rdsv3_ib_xmit", "ibt_post_send failed\n");
863 859 rdsv3_conn_drop(ic->conn);
864 860 ret = -EAGAIN;
865 861 goto out;
866 862 }
867 863
868 864 ret = sent;
869 865
870 866 RDSV3_DPRINTF4("rdsv3_ib_xmit", "Return: conn: %p, rm: %p", conn, rm);
871 867 out:
872 868 ASSERT(!adv_credits);
873 869 return (ret);
874 870 }
875 871
876 872 static void
877 873 rdsv3_ib_dma_unmap_sg_rdma(struct ib_device *dev, uint_t num,
878 874 struct rdsv3_rdma_sg scat[])
879 875 {
880 876 ibt_hca_hdl_t hca_hdl;
881 877 int i;
882 878 int num_sgl;
883 879
884 880 RDSV3_DPRINTF4("rdsv3_ib_dma_unmap_sg", "rdma_sg: %p", scat);
885 881
886 882 if (dev) {
887 883 hca_hdl = ib_get_ibt_hca_hdl(dev);
888 884 } else {
889 885 hca_hdl = scat[0].hca_hdl;
890 886 RDSV3_DPRINTF2("rdsv3_ib_dma_unmap_sg_rdma",
891 887 "NULL dev use cached hca_hdl %p", hca_hdl);
892 888 }
893 889
894 890 if (hca_hdl == NULL)
895 891 return;
896 892 scat[0].hca_hdl = NULL;
897 893
898 894 for (i = 0; i < num; i++) {
899 895 if (scat[i].mihdl != NULL) {
900 896 num_sgl = (scat[i].iovec.bytes / PAGESIZE) + 2;
901 897 kmem_free(scat[i].swr.wr_sgl,
902 898 (num_sgl * sizeof (ibt_wr_ds_t)));
903 899 scat[i].swr.wr_sgl = NULL;
904 900 (void) ibt_unmap_mem_iov(hca_hdl, scat[i].mihdl);
905 901 scat[i].mihdl = NULL;
906 902 } else
907 903 break;
908 904 }
909 905 }
910 906
911 907 /* ARGSUSED */
912 908 uint_t
913 909 rdsv3_ib_dma_map_sg_rdma(struct ib_device *dev, struct rdsv3_rdma_sg scat[],
914 910 uint_t num, struct rdsv3_scatterlist **scatl)
915 911 {
916 912 ibt_hca_hdl_t hca_hdl;
917 913 ibt_iov_attr_t iov_attr;
918 914 struct buf *bp;
919 915 uint_t i, j, k;
920 916 uint_t count;
921 917 struct rdsv3_scatterlist *sg;
922 918 int ret;
923 919
924 920 RDSV3_DPRINTF4("rdsv3_ib_dma_map_sg_rdma", "scat: %p, num: %d",
925 921 scat, num);
926 922
927 923 hca_hdl = ib_get_ibt_hca_hdl(dev);
928 924 scat[0].hca_hdl = hca_hdl;
929 925 bzero(&iov_attr, sizeof (ibt_iov_attr_t));
930 926 iov_attr.iov_flags = IBT_IOV_BUF;
931 927 iov_attr.iov_lso_hdr_sz = 0;
932 928
933 929 for (i = 0, count = 0; i < num; i++) {
934 930 /* transpose umem_cookie to buf structure */
935 931 bp = ddi_umem_iosetup(scat[i].umem_cookie,
936 932 scat[i].iovec.addr & PAGEOFFSET, scat[i].iovec.bytes,
937 933 B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP);
938 934 if (bp == NULL) {
939 935 /* free resources and return error */
940 936 goto out;
941 937 }
942 938 /* setup ibt_map_mem_iov() attributes */
943 939 iov_attr.iov_buf = bp;
944 940 iov_attr.iov_wr_nds = (scat[i].iovec.bytes / PAGESIZE) + 2;
945 941 scat[i].swr.wr_sgl =
946 942 kmem_zalloc(iov_attr.iov_wr_nds * sizeof (ibt_wr_ds_t),
947 943 KM_SLEEP);
948 944
949 945 ret = ibt_map_mem_iov(hca_hdl, &iov_attr,
950 946 (ibt_all_wr_t *)&scat[i].swr, &scat[i].mihdl);
951 947 freerbuf(bp);
952 948 if (ret != IBT_SUCCESS) {
953 949 RDSV3_DPRINTF2("rdsv3_ib_dma_map_sg_rdma",
954 950 "ibt_map_mem_iov returned: %d", ret);
955 951 /* free resources and return error */
956 952 kmem_free(scat[i].swr.wr_sgl,
957 953 iov_attr.iov_wr_nds * sizeof (ibt_wr_ds_t));
958 954 goto out;
959 955 }
960 956 count += scat[i].swr.wr_nds;
961 957
962 958 #ifdef DEBUG
963 959 for (j = 0; j < scat[i].swr.wr_nds; j++) {
964 960 RDSV3_DPRINTF5("rdsv3_ib_dma_map_sg_rdma",
965 961 "sgl[%d] va %llx len %x", j,
966 962 scat[i].swr.wr_sgl[j].ds_va,
967 963 scat[i].swr.wr_sgl[j].ds_len);
968 964 }
969 965 #endif
970 966 RDSV3_DPRINTF4("rdsv3_ib_dma_map_sg_rdma",
971 967 "iovec.bytes: 0x%x scat[%d]swr.wr_nds: %d",
972 968 scat[i].iovec.bytes, i, scat[i].swr.wr_nds);
973 969 }
974 970
975 971 count = ((count - 1) / RDSV3_IB_MAX_SGE) + 1;
976 972 RDSV3_DPRINTF4("rdsv3_ib_dma_map_sg_rdma", "Ret: num: %d", count);
977 973 return (count);
978 974
979 975 out:
980 976 rdsv3_ib_dma_unmap_sg_rdma(dev, num, scat);
981 977 return (0);
982 978 }
983 979
984 980 int
985 981 rdsv3_ib_xmit_rdma(struct rdsv3_connection *conn, struct rdsv3_rdma_op *op)
986 982 {
987 983 struct rdsv3_ib_connection *ic = conn->c_transport_data;
988 984 struct rdsv3_ib_send_work *send = NULL;
989 985 struct rdsv3_rdma_sg *scat;
990 986 uint64_t remote_addr;
991 987 uint32_t pos;
992 988 uint32_t work_alloc;
993 989 uint32_t i, j, k, idx;
994 990 uint32_t left, count;
995 991 uint32_t posted;
996 992 int sent;
997 993 ibt_status_t status;
998 994 ibt_send_wr_t *wr;
999 995 ibt_wr_ds_t *sge;
1000 996
1001 997 RDSV3_DPRINTF4("rdsv3_ib_xmit_rdma", "rdsv3_ib_conn: %p", ic);
1002 998
1003 999 /* map the message the first time we see it */
1004 1000 if (!op->r_mapped) {
1005 1001 op->r_count = rdsv3_ib_dma_map_sg_rdma(ic->i_cm_id->device,
1006 1002 op->r_rdma_sg, op->r_nents, &op->r_sg);
1007 1003 RDSV3_DPRINTF5("rdsv3_ib_xmit_rdma", "ic %p mapping op %p: %d",
1008 1004 ic, op, op->r_count);
1009 1005 if (op->r_count == 0) {
1010 1006 rdsv3_ib_stats_inc(s_ib_tx_sg_mapping_failure);
1011 1007 RDSV3_DPRINTF2("rdsv3_ib_xmit_rdma",
1012 1008 "fail: ic %p mapping op %p: %d",
1013 1009 ic, op, op->r_count);
1014 1010 return (-ENOMEM); /* XXX ? */
1015 1011 }
1016 1012 op->r_mapped = 1;
1017 1013 }
1018 1014
1019 1015 /*
1020 1016 * Instead of knowing how to return a partial rdma read/write
1021 1017 * we insist that there
1022 1018 * be enough work requests to send the entire message.
1023 1019 */
1024 1020 work_alloc = rdsv3_ib_ring_alloc(&ic->i_send_ring, op->r_count, &pos);
1025 1021 if (work_alloc != op->r_count) {
1026 1022 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
1027 1023 rdsv3_ib_stats_inc(s_ib_tx_ring_full);
1028 1024 return (-ENOMEM);
1029 1025 }
1030 1026
1031 1027 RDSV3_DPRINTF4("rdsv3_ib_xmit_rdma", "pos %u cnt %u", pos, op->r_count);
1032 1028 /*
1033 1029 * take the scatter list and transpose into a list of
1034 1030 * send wr's each with a scatter list of RDSV3_IB_MAX_SGE
1035 1031 */
1036 1032 scat = &op->r_rdma_sg[0];
1037 1033 sent = 0;
1038 1034 remote_addr = op->r_remote_addr;
1039 1035
1040 1036 for (i = 0, k = 0; i < op->r_nents; i++) {
1041 1037 left = scat[i].swr.wr_nds;
1042 1038 for (idx = 0; left > 0; k++) {
1043 1039 send = &ic->i_sends[pos];
1044 1040 send->s_queued = jiffies;
1045 1041 send->s_opcode = op->r_write ? IBT_WRC_RDMAW :
1046 1042 IBT_WRC_RDMAR;
1047 1043 send->s_op = op;
1048 1044
1049 1045 wr = &ic->i_send_wrs[k];
1050 1046 wr->wr_flags = 0;
1051 1047 wr->wr_id = pos | RDSV3_IB_SEND_OP;
1052 1048 wr->wr_trans = IBT_RC_SRV;
1053 1049 wr->wr_opcode = op->r_write ? IBT_WRC_RDMAW :
1054 1050 IBT_WRC_RDMAR;
1055 1051 wr->wr.rc.rcwr.rdma.rdma_raddr = remote_addr;
1056 1052 wr->wr.rc.rcwr.rdma.rdma_rkey = op->r_key;
1057 1053
1058 1054 if (left > RDSV3_IB_MAX_SGE) {
1059 1055 count = RDSV3_IB_MAX_SGE;
1060 1056 left -= RDSV3_IB_MAX_SGE;
1061 1057 } else {
1062 1058 count = left;
1063 1059 left = 0;
1064 1060 }
1065 1061 wr->wr_nds = count;
1066 1062
1067 1063 for (j = 0; j < count; j++) {
1068 1064 sge = &wr->wr_sgl[j];
1069 1065 *sge = scat[i].swr.wr_sgl[idx];
1070 1066 remote_addr += scat[i].swr.wr_sgl[idx].ds_len;
1071 1067 sent += scat[i].swr.wr_sgl[idx].ds_len;
1072 1068 idx++;
1073 1069 RDSV3_DPRINTF5("xmit_rdma",
1074 1070 "send_wrs[%d]sgl[%d] va %llx len %x",
1075 1071 k, j, sge->ds_va, sge->ds_len);
1076 1072 }
1077 1073 RDSV3_DPRINTF5("rdsv3_ib_xmit_rdma",
1078 1074 "wr[%d] %p key: %x code: %d tlen: %d",
1079 1075 k, wr, wr->wr.rc.rcwr.rdma.rdma_rkey,
1080 1076 wr->wr_opcode, sent);
1081 1077
1082 1078 /*
1083 1079 * We want to delay signaling completions just enough
1084 1080 * to get the batching benefits but not so much that
1085 1081 * we create dead time on the wire.
1086 1082 */
1087 1083 if (ic->i_unsignaled_wrs-- == 0) {
1088 1084 ic->i_unsignaled_wrs =
1089 1085 rdsv3_ib_sysctl_max_unsig_wrs;
1090 1086 wr->wr_flags = IBT_WR_SEND_SIGNAL;
1091 1087 }
1092 1088
1093 1089 pos = (pos + 1) % ic->i_send_ring.w_nr;
1094 1090 }
1095 1091 }
1096 1092
1097 1093 status = ibt_post_send(ib_get_ibt_channel_hdl(ic->i_cm_id),
1098 1094 ic->i_send_wrs, k, &posted);
1099 1095 if (status != IBT_SUCCESS) {
1100 1096 RDSV3_DPRINTF2("rdsv3_ib_xmit_rdma",
1101 1097 "RDS/IB: rdma ib_post_send to %u.%u.%u.%u "
1102 1098 "returned %d", NIPQUAD(conn->c_faddr), status);
1103 1099 rdsv3_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
1104 1100 }
1105 1101 RDSV3_DPRINTF4("rdsv3_ib_xmit_rdma", "Ret: %p", ic);
1106 1102 return (status);
1107 1103 }
1108 1104
1109 1105 void
1110 1106 rdsv3_ib_xmit_complete(struct rdsv3_connection *conn)
1111 1107 {
1112 1108 struct rdsv3_ib_connection *ic = conn->c_transport_data;
1113 1109
1114 1110 RDSV3_DPRINTF4("rdsv3_ib_xmit_complete", "conn: %p", conn);
1115 1111
1116 1112 /*
1117 1113 * We may have a pending ACK or window update we were unable
1118 1114 * to send previously (due to flow control). Try again.
1119 1115 */
1120 1116 rdsv3_ib_attempt_ack(ic);
1121 1117 }
|
↓ open down ↓ |
515 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX