[infiniband] Add infrastructure for RC queue pairs

Queue pairs are now assumed to be created in the INIT state, with a
call to ib_modify_qp() required to bring the queue pair to the RTS
state.

ib_modify_qp() no longer takes a modification list; callers should
modify the relevant queue pair parameters (e.g. qkey) directly and
then call ib_modify_qp() to synchronise the changes to the hardware.

The packet sequence number is now a property of the queue pair, rather
than of the device.

Each queue pair may have an associated address vector.  For RC queue
pairs, this is the address vector that will be programmed in to the
hardware as the remote address.  For UD queue pairs, it will be used
as the default address vector if none is supplied to ib_post_send().

4 files changed, 19 insertions, 24 deletions

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index a9ca34575..5bf357431 100644
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -855,7 +855,6 @@ static int arbel_create_qp ( struct ib_device *ibdev,
 		     ( virt_to_bus ( arbel_qp->recv.wqe ) >> 6 ) );
 	MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.rcv_db_record_index,
 		     arbel_qp->recv.doorbell_idx );
-	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
 	if ( ( rc = arbel_cmd_rst2init_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
 		DBGC ( arbel, "Arbel %p RST2INIT_QPEE failed: %s\n",
 		       arbel, strerror ( rc ) );
@@ -908,24 +907,17 @@ static int arbel_create_qp ( struct ib_device *ibdev,
  *
  * @v ibdev		Infiniband device
  * @v qp		Queue pair
- * @v mod_list		Modification list
  * @ret rc		Return status code
  */
 static int arbel_modify_qp ( struct ib_device *ibdev,
-			     struct ib_queue_pair *qp,
-			     unsigned long mod_list ) {
+			     struct ib_queue_pair *qp ) {
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
 	struct arbelprm_qp_ee_state_transitions qpctx;
-	unsigned long optparammask = 0;
 	int rc;
 
-	/* Construct optparammask */
-	if ( mod_list & IB_MODIFY_QKEY )
-		optparammask |= ARBEL_QPEE_OPT_PARAM_QKEY;
-
 	/* Issue RTS2RTS_QP */
 	memset ( &qpctx, 0, sizeof ( qpctx ) );
-	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, optparammask );
+	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, ARBEL_QPEE_OPT_PARAM_QKEY );
 	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
 	if ( ( rc = arbel_cmd_rts2rts_qp ( arbel, qp->qpn, &qpctx ) ) != 0 ){
 		DBGC ( arbel, "Arbel %p RTS2RTS_QP failed: %s\n",
diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index 4ece7a4c2..26f623142 100644
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -885,7 +885,6 @@ static int hermon_create_qp ( struct ib_device *ibdev,
 	MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
 	MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.db_record_addr_l,
 		     ( virt_to_phys ( &hermon_qp->recv.doorbell ) >> 2 ) );
-	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
 	MLX_FILL_1 ( &qpctx, 53, qpc_eec_data.mtt_base_addr_l,
 		     ( hermon_qp->mtt.mtt_base_addr >> 3 ) );
 	if ( ( rc = hermon_cmd_rst2init_qp ( hermon, qp->qpn,
@@ -946,24 +945,17 @@ static int hermon_create_qp ( struct ib_device *ibdev,
  *
  * @v ibdev		Infiniband device
  * @v qp		Queue pair
- * @v mod_list		Modification list
  * @ret rc		Return status code
  */
 static int hermon_modify_qp ( struct ib_device *ibdev,
-			      struct ib_queue_pair *qp,
-			      unsigned long mod_list ) {
+			      struct ib_queue_pair *qp ) {
 	struct hermon *hermon = ib_get_drvdata ( ibdev );
 	struct hermonprm_qp_ee_state_transitions qpctx;
-	unsigned long optparammask = 0;
 	int rc;
 
-	/* Construct optparammask */
-	if ( mod_list & IB_MODIFY_QKEY )
-		optparammask |= HERMON_QP_OPT_PARAM_QKEY;
-
 	/* Issue RTS2RTS_QP */
 	memset ( &qpctx, 0, sizeof ( qpctx ) );
-	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, optparammask );
+	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, HERMON_QP_OPT_PARAM_QKEY );
 	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
 	if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
 		DBGC ( hermon, "Hermon %p RTS2RTS_QP failed: %s\n",
diff --git a/src/drivers/infiniband/linda.c b/src/drivers/infiniband/linda.c
index 02b5d8563..c8e44bf8a 100644
--- a/src/drivers/infiniband/linda.c
+++ b/src/drivers/infiniband/linda.c
@@ -268,6 +268,18 @@ static int linda_set_port_info ( struct ib_device *ibdev, union ib_mad *mad ) {
 	return 0;
 }
 
+/**
+ * Set partition key table
+ *
+ * @v ibdev		Infiniband device
+ * @v mad		Set partition key table MAD
+ */
+static int linda_set_pkey_table ( struct ib_device *ibdev __unused,
+				  union ib_mad *mad __unused ) {
+	/* Nothing to do */
+	return 0;
+}
+
 /***************************************************************************
  *
  * Context allocation
@@ -852,12 +864,10 @@ static int linda_create_qp ( struct ib_device *ibdev,
  *
  * @v ibdev		Infiniband device
  * @v qp		Queue pair
- * @v mod_list		Modification list
  * @ret rc		Return status code
  */
 static int linda_modify_qp ( struct ib_device *ibdev,
-			     struct ib_queue_pair *qp,
-			     unsigned long mod_list __unused ) {
+			     struct ib_queue_pair *qp ) {
 	struct linda *linda = ib_get_drvdata ( ibdev );
 
 	/* Nothing to do; the hardware doesn't have a notion of queue
@@ -1456,6 +1466,7 @@ static struct ib_device_operations linda_ib_operations = {
 	.mcast_attach	= linda_mcast_attach,
 	.mcast_detach	= linda_mcast_detach,
 	.set_port_info	= linda_set_port_info,
+	.set_pkey_table	= linda_set_pkey_table,
 };
 
 /***************************************************************************
diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 4d9f21d7a..9110784c0 100644
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -521,7 +521,7 @@ static int ipoib_open ( struct net_device *netdev ) {
 	/* Allocate queue pair */
 	ipoib->qp = ib_create_qp ( ibdev, IB_QPT_UD,
 				   IPOIB_NUM_SEND_WQES, ipoib->cq,
-				   IPOIB_NUM_RECV_WQES, ipoib->cq, 0 );
+				   IPOIB_NUM_RECV_WQES, ipoib->cq );
 	if ( ! ipoib->qp ) {
 		DBGC ( ipoib, "IPoIB %p could not allocate queue pair\n",
 		       ipoib );