From f1d92fa8865c281cb7da31f2e3b62ece3f2f9ca1 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Thu, 9 Jul 2009 15:52:04 +0100 Subject: [infiniband] Allow external QPN to differ from real QPN Most IB hardware seems not to allow allocation of the genuine QPNs 0 and 1, so allow for the externally-visible QPN (as constructed and parsed by ib_packet, where used) to differ from the real hardware-allocated QPN. --- src/include/gpxe/infiniband.h | 9 ++++++++- src/net/infiniband.c | 21 +++++++++++++++++++-- src/net/infiniband/ib_packet.c | 10 +++++----- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h index 9c1f57a08..dacb13e4b 100644 --- a/src/include/gpxe/infiniband.h +++ b/src/include/gpxe/infiniband.h @@ -96,8 +96,15 @@ struct ib_queue_pair { struct ib_device *ibdev; /** List of queue pairs on this Infiniband device */ struct list_head list; - /** Queue Pair Number */ + /** Queue pair number */ unsigned long qpn; + /** Externally-visible queue pair number + * + * This may differ from the real queue pair number (e.g. when + * the HCA cannot use the management QPNs 0 and 1 as hardware + * QPNs and needs to remap them). + */ + unsigned long ext_qpn; /** Queue pair type */ enum ib_queue_pair_type type; /** Queue key */ diff --git a/src/net/infiniband.c b/src/net/infiniband.c index 1444a12c6..b15dcc61a 100644 --- a/src/net/infiniband.c +++ b/src/net/infiniband.c @@ -196,7 +196,6 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev, "%s\n", ibdev, strerror ( rc ) ); goto err_dev_create_qp; } - DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n", ibdev, qp, ib_qp_get_drvdata ( qp ), qp->qpn ); DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n", @@ -205,6 +204,24 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev, DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n", ibdev, qp->qpn, num_recv_wqes, qp->recv.iobufs, ( ( ( void * ) qp ) + total_size ) ); + + /* Calculate externally-visible QPN */ + switch ( type ) { + case IB_QPT_SMA: + qp->ext_qpn = IB_QPN_SMA; + break; + case IB_QPT_GMA: + qp->ext_qpn = IB_QPN_GMA; + break; + default: + qp->ext_qpn = qp->qpn; + break; + } + if ( qp->ext_qpn != qp->qpn ) { + DBGC ( ibdev, "IBDEV %p QPN %#lx has external QPN %#lx\n", + ibdev, qp->qpn, qp->ext_qpn ); + } + return qp; ibdev->op->destroy_qp ( ibdev, qp ); @@ -295,7 +312,7 @@ struct ib_queue_pair * ib_find_qp_qpn ( struct ib_device *ibdev, struct ib_queue_pair *qp; list_for_each_entry ( qp, &ibdev->qps, list ) { - if ( qp->qpn == qpn ) + if ( ( qpn == qp->qpn ) || ( qpn == qp->ext_qpn ) ) return qp; } return NULL; diff --git a/src/net/infiniband/ib_packet.c b/src/net/infiniband/ib_packet.c index 409ef2f8b..689e00dce 100644 --- a/src/net/infiniband/ib_packet.c +++ b/src/net/infiniband/ib_packet.c @@ -58,7 +58,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf, unsigned int lnh; DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n", - ibdev, ibdev->lid, qp->qpn, av->lid, av->qpn, av->qkey ); + ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey ); /* Calculate packet length */ pad_len = ( (-payload_len) & 0x3 ); @@ -76,7 +76,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf, lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len ); /* Construct LRH */ - vl = ( ( av->qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT ); + vl = ( ( qp->ext_qpn == IB_QPN_SMA ) ? IB_VL_SMP : IB_VL_DEFAULT ); lrh->vl__lver = ( vl << 4 ); lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH ); lrh->sl__lnh = ( ( av->sl << 4 ) | lnh ); @@ -104,7 +104,7 @@ int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf, /* Construct DETH */ deth->qkey = htonl ( av->qkey ); - deth->src_qp = htonl ( qp->qpn ); + deth->src_qp = htonl ( qp->ext_qpn ); DBGCP_HDA ( ibdev, 0, iobuf->data, ( iob_len ( iobuf ) - orig_iob_len ) ); @@ -233,8 +233,8 @@ int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf, } DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n", - ibdev, lid, - ( IB_LID_MULTICAST( lid ) ? ( qp ? (*qp)->qpn : -1UL ) : qpn ), + ibdev, lid, ( IB_LID_MULTICAST( lid ) ? + ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ), av->lid, av->qpn, ntohl ( deth->qkey ) ); DBGCP_HDA ( ibdev, 0, ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ), -- cgit