[infiniband] Split subnet management agent client out into ib_smc.c

Not all Infiniband cards have embedded subnet management agents.
Split out the code that communicates with such an embedded SMA into a
separate ib_smc.c file, and have drivers call ib_smc_update()
explicitly when they suspect that the answers given by the embedded
SMA may have changed.

1 files changed, 52 insertions, 271 deletions

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index ed0d1a666..2691ffc95 100644
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -10,6 +10,8 @@
 #include <stdint.h>
 #include <gpxe/refcnt.h>
 #include <gpxe/device.h>
+#include <gpxe/ib_packet.h>
+#include <gpxe/ib_mad.h>
 
 /** Subnet administrator QPN */
 #define IB_SA_QPN 1
@@ -20,36 +22,6 @@
 /** Subnet administrator queue key */
 #define IB_GLOBAL_QKEY 0x80010000UL
 
-/** An Infiniband Global Identifier */
-struct ib_gid {
-	union {
-		uint8_t bytes[16];
-		uint16_t words[8];
-		uint32_t dwords[4];
-	} u;
-};
-
-/** An Infiniband Global Route Header */
-struct ib_global_route_header {
-	/** IP version, traffic class, and flow label
-	 *
-	 *  4 bits : Version of the GRH
-	 *  8 bits : Traffic class
-	 * 20 bits : Flow label
-	 */
-	uint32_t ipver_tclass_flowlabel;
-	/** Payload length */
-	uint16_t paylen;
-	/** Next header */
-	uint8_t nxthdr;
-	/** Hop limit */
-	uint8_t hoplmt;
-	/** Source GID */
-	struct ib_gid sgid;
-	/** Destiniation GID */
-	struct ib_gid dgid;
-} __attribute__ (( packed ));
-
 struct ib_device;
 struct ib_queue_pair;
 struct ib_address_vector;
@@ -178,8 +150,6 @@ struct ib_completion_queue {
 	void *drv_priv;
 };
 
-struct ib_mad_hdr;
-
 /**
  * Infiniband device operations
  *
@@ -306,16 +276,6 @@ struct ib_device_operations {
 	void ( * mcast_detach ) ( struct ib_device *ibdev,
 				  struct ib_queue_pair *qp,
 				  struct ib_gid *gid );
-	/**
-	 * Issue management datagram
-	 *
-	 * @v ibdev		Infiniband device
-	 * @v mad		Management datagram
-	 * @v len		Length of management datagram
-	 * @ret rc		Return status code
-	 */
-	int ( * mad ) ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
-			size_t len );
 };
 
 /** An Infiniband device */
@@ -330,14 +290,24 @@ struct ib_device {
 	struct ib_device_operations *op;
 	/** Port number */
 	unsigned int port;
-	/** Link state */
-	int link_up;
+
+	/** Port state */
+	uint8_t port_state;
+	/** Link width */
+	uint8_t link_width;
+	/** Link speed */
+	uint8_t link_speed;
 	/** Port GID */
-	struct ib_gid port_gid;
+	struct ib_gid gid;
+	/** Port LID */
+	uint16_t lid;
 	/** Subnet manager LID */
-	unsigned long sm_lid;
+	uint16_t sm_lid;
+	/** Subnet manager SL */
+	uint8_t sm_sl;
 	/** Partition key */
-	unsigned int pkey;
+	uint16_t pkey;
+
 	/** Driver private data */
 	void *drv_priv;
 	/** Owner private data */
@@ -375,6 +345,11 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
 extern int register_ibdev ( struct ib_device *ibdev );
 extern void unregister_ibdev ( struct ib_device *ibdev );
 extern void ib_link_state_changed ( struct ib_device *ibdev );
+extern struct list_head ib_devices;
+
+/** Iterate over all network devices */
+#define for_each_ibdev( ibdev ) \
+	list_for_each_entry ( (ibdev), &ib_devices, list )
 
 /**
  * Poll completion queue
@@ -382,7 +357,7 @@ extern void ib_link_state_changed ( struct ib_device *ibdev );
  * @v ibdev		Infiniband device
  * @v cq		Completion queue
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
 	ibdev->op->poll_cq ( ibdev, cq );
 }
@@ -393,7 +368,7 @@ ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
  * @v ibdev		Infiniband device
  * @ret rc		Return status code
  */
-static inline __attribute__ (( always_inline )) int
+static inline __always_inline int
 ib_open ( struct ib_device *ibdev ) {
 	return ibdev->op->open ( ibdev );
 }
@@ -403,12 +378,23 @@ ib_open ( struct ib_device *ibdev ) {
  *
  * @v ibdev		Infiniband device
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_close ( struct ib_device *ibdev ) {
 	ibdev->op->close ( ibdev );
 }
 
 /**
+ * Check link state
+ *
+ * @v ibdev		Infiniband device
+ * @ret link_up		Link is up
+ */
+static inline __always_inline int
+ib_link_ok ( struct ib_device *ibdev ) {
+	return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
+}
+
+/**
  * Attach to multicast group
  *
  * @v ibdev		Infiniband device
@@ -416,7 +402,7 @@ ib_close ( struct ib_device *ibdev ) {
  * @v gid		Multicast GID
  * @ret rc		Return status code
  */
-static inline __attribute__ (( always_inline )) int
+static inline __always_inline int
 ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		  struct ib_gid *gid ) {
 	return ibdev->op->mcast_attach ( ibdev, qp, gid );
@@ -429,32 +415,19 @@ ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
  * @v qp		Queue pair
  * @v gid		Multicast GID
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		  struct ib_gid *gid ) {
 	ibdev->op->mcast_detach ( ibdev, qp, gid );
 }
 
 /**
- * Issue management datagram
- *
- * @v ibdev		Infiniband device
- * @v mad		Management datagram
- * @v len		Length of management datagram
- * @ret rc		Return status code
- */
-static inline __attribute__ (( always_inline )) int
-ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
-	return ibdev->op->mad ( ibdev, mad, len );
-}
-
-/**
  * Get reference to Infiniband device
  *
  * @v ibdev		Infiniband device
  * @ret ibdev		Infiniband device
  */
-static inline __attribute__ (( always_inline )) struct ib_device *
+static inline __always_inline struct ib_device *
 ibdev_get ( struct ib_device *ibdev ) {
 	ref_get ( &ibdev->refcnt );
 	return ibdev;
@@ -465,7 +438,7 @@ ibdev_get ( struct ib_device *ibdev ) {
  *
  * @v ibdev		Infiniband device
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ibdev_put ( struct ib_device *ibdev ) {
 	ref_put ( &ibdev->refcnt );
 }
@@ -476,7 +449,7 @@ ibdev_put ( struct ib_device *ibdev ) {
  * @v wq		Work queue
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
 	wq->drv_priv = priv;
 }
@@ -487,7 +460,7 @@ ib_wq_set_drvdata ( struct ib_work_queue *wq, void *priv ) {
  * @v wq		Work queue
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
 	return wq->drv_priv;
 }
@@ -498,7 +471,7 @@ ib_wq_get_drvdata ( struct ib_work_queue *wq ) {
  * @v qp		Queue pair
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
 	qp->drv_priv = priv;
 }
@@ -509,7 +482,7 @@ ib_qp_set_drvdata ( struct ib_queue_pair *qp, void *priv ) {
  * @v qp		Queue pair
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
 	return qp->drv_priv;
 }
@@ -520,7 +493,7 @@ ib_qp_get_drvdata ( struct ib_queue_pair *qp ) {
  * @v qp		Queue pair
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
 	qp->owner_priv = priv;
 }
@@ -531,7 +504,7 @@ ib_qp_set_ownerdata ( struct ib_queue_pair *qp, void *priv ) {
  * @v qp		Queue pair
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
 	return qp->owner_priv;
 }
@@ -542,7 +515,7 @@ ib_qp_get_ownerdata ( struct ib_queue_pair *qp ) {
  * @v cq		Completion queue
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
 	cq->drv_priv = priv;
 }
@@ -553,7 +526,7 @@ ib_cq_set_drvdata ( struct ib_completion_queue *cq, void *priv ) {
  * @v cq		Completion queue
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
 	return cq->drv_priv;
 }
@@ -564,7 +537,7 @@ ib_cq_get_drvdata ( struct ib_completion_queue *cq ) {
  * @v ibdev		Infiniband device
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
 	ibdev->drv_priv = priv;
 }
@@ -575,7 +548,7 @@ ib_set_drvdata ( struct ib_device *ibdev, void *priv ) {
  * @v ibdev		Infiniband device
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_get_drvdata ( struct ib_device *ibdev ) {
 	return ibdev->drv_priv;
 }
@@ -586,7 +559,7 @@ ib_get_drvdata ( struct ib_device *ibdev ) {
  * @v ibdev		Infiniband device
  * @v priv		Private data
  */
-static inline __attribute__ (( always_inline )) void
+static inline __always_inline void
 ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
 	ibdev->owner_priv = priv;
 }
@@ -597,201 +570,9 @@ ib_set_ownerdata ( struct ib_device *ibdev, void *priv ) {
  * @v ibdev		Infiniband device
  * @ret priv		Private data
  */
-static inline __attribute__ (( always_inline )) void *
+static inline __always_inline void *
 ib_get_ownerdata ( struct ib_device *ibdev ) {
 	return ibdev->owner_priv;
 }
 
-/*****************************************************************************
- *
- * Management datagrams
- *
- * Portions Copyright (c) 2004 Mellanox Technologies Ltd.  All rights
- * reserved.
- *
- */
-
-/* Management base version */
-#define IB_MGMT_BASE_VERSION			1
-
-/* Management classes */
-#define IB_MGMT_CLASS_SUBN_LID_ROUTED		0x01
-#define IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE	0x81
-#define IB_MGMT_CLASS_SUBN_ADM			0x03
-#define IB_MGMT_CLASS_PERF_MGMT			0x04
-#define IB_MGMT_CLASS_BM			0x05
-#define IB_MGMT_CLASS_DEVICE_MGMT		0x06
-#define IB_MGMT_CLASS_CM			0x07
-#define IB_MGMT_CLASS_SNMP			0x08
-#define IB_MGMT_CLASS_VENDOR_RANGE2_START	0x30
-#define IB_MGMT_CLASS_VENDOR_RANGE2_END		0x4F
-
-/* Management methods */
-#define IB_MGMT_METHOD_GET			0x01
-#define IB_MGMT_METHOD_SET			0x02
-#define IB_MGMT_METHOD_GET_RESP			0x81
-#define IB_MGMT_METHOD_SEND			0x03
-#define IB_MGMT_METHOD_TRAP			0x05
-#define IB_MGMT_METHOD_REPORT			0x06
-#define IB_MGMT_METHOD_REPORT_RESP		0x86
-#define IB_MGMT_METHOD_TRAP_REPRESS		0x07
-#define IB_MGMT_METHOD_DELETE			0x15
-#define IB_MGMT_METHOD_RESP			0x80
-
-/* Subnet management attributes */
-#define IB_SMP_ATTR_NOTICE			0x0002
-#define IB_SMP_ATTR_NODE_DESC			0x0010
-#define IB_SMP_ATTR_NODE_INFO			0x0011
-#define IB_SMP_ATTR_SWITCH_INFO			0x0012
-#define IB_SMP_ATTR_GUID_INFO			0x0014
-#define IB_SMP_ATTR_PORT_INFO			0x0015
-#define IB_SMP_ATTR_PKEY_TABLE			0x0016
-#define IB_SMP_ATTR_SL_TO_VL_TABLE		0x0017
-#define IB_SMP_ATTR_VL_ARB_TABLE		0x0018
-#define IB_SMP_ATTR_LINEAR_FORWARD_TABLE	0x0019
-#define IB_SMP_ATTR_RANDOM_FORWARD_TABLE	0x001A
-#define IB_SMP_ATTR_MCAST_FORWARD_TABLE		0x001B
-#define IB_SMP_ATTR_SM_INFO			0x0020
-#define IB_SMP_ATTR_VENDOR_DIAG			0x0030
-#define IB_SMP_ATTR_LED_INFO			0x0031
-#define IB_SMP_ATTR_VENDOR_MASK			0xFF00
-
-#define IB_SA_ATTR_MC_MEMBER_REC		0x38
-#define IB_SA_ATTR_PATH_REC			0x35
-
-#define IB_SA_MCMEMBER_REC_MGID			(1<<0)
-#define IB_SA_MCMEMBER_REC_PORT_GID		(1<<1)
-#define IB_SA_MCMEMBER_REC_QKEY			(1<<2)
-#define IB_SA_MCMEMBER_REC_MLID			(1<<3)
-#define IB_SA_MCMEMBER_REC_MTU_SELECTOR		(1<<4)
-#define IB_SA_MCMEMBER_REC_MTU			(1<<5)
-#define IB_SA_MCMEMBER_REC_TRAFFIC_CLASS	(1<<6)
-#define IB_SA_MCMEMBER_REC_PKEY			(1<<7)
-#define IB_SA_MCMEMBER_REC_RATE_SELECTOR	(1<<8)
-#define IB_SA_MCMEMBER_REC_RATE			(1<<9)
-#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR	(1<<10)
-#define IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME	(1<<11)
-#define IB_SA_MCMEMBER_REC_SL			(1<<12)
-#define IB_SA_MCMEMBER_REC_FLOW_LABEL		(1<<13)
-#define IB_SA_MCMEMBER_REC_HOP_LIMIT		(1<<14)
-#define IB_SA_MCMEMBER_REC_SCOPE		(1<<15)
-#define IB_SA_MCMEMBER_REC_JOIN_STATE		(1<<16)
-#define IB_SA_MCMEMBER_REC_PROXY_JOIN		(1<<17)
-
-#define IB_SA_PATH_REC_DGID			(1<<2)
-#define IB_SA_PATH_REC_SGID			(1<<3)
-
-struct ib_mad_hdr {
-	uint8_t base_version;
-	uint8_t mgmt_class;
-	uint8_t class_version;
-	uint8_t method;
-	uint16_t status;
-	uint16_t class_specific;
-	uint32_t tid[2];
-	uint16_t attr_id;
-	uint16_t resv;
-	uint32_t attr_mod;
-} __attribute__ (( packed ));
-
-struct ib_sa_hdr {
-	uint32_t sm_key[2];
-	uint16_t reserved;
-	uint16_t attrib_offset;
-	uint32_t comp_mask[2];
-} __attribute__ (( packed ));
-
-struct ib_rmpp_hdr {
-	uint32_t raw[3];
-} __attribute__ (( packed ));
-
-struct ib_mad_data {
-	struct ib_mad_hdr mad_hdr;
-	uint8_t data[232];
-} __attribute__ (( packed ));
-
-struct ib_mad_guid_info {
-	struct ib_mad_hdr mad_hdr;
-	uint32_t mkey[2];
-	uint32_t reserved[8];
-	uint8_t gid_local[8];
-} __attribute__ (( packed ));
-
-struct ib_mad_port_info {
-	struct ib_mad_hdr mad_hdr;
-	uint32_t mkey[2];
-	uint32_t reserved[8];
-	uint32_t mkey2[2];
-	uint8_t gid_prefix[8];
-	uint16_t lid;
-	uint16_t mastersm_lid;
-	uint32_t cap_mask;
-	uint16_t diag_code;
-	uint16_t mkey_lease_period;
-	uint8_t local_port_num;
-	uint8_t link_width_enabled;
-	uint8_t link_width_supported;
-	uint8_t link_width_active;
-	uint8_t port_state__link_speed_supported;
-	uint8_t link_down_def_state__port_phys_state;
-	uint8_t lmc__r1__mkey_prot_bits;
-	uint8_t link_speed_enabled__link_speed_active;
-} __attribute__ (( packed ));
-
-struct ib_mad_pkey_table {
-	struct ib_mad_hdr mad_hdr;
-	uint32_t mkey[2];
-	uint32_t reserved[8];
-	uint16_t pkey[16][2];
-} __attribute__ (( packed ));
-
-struct ib_mad_path_record {
-	struct ib_mad_hdr mad_hdr;
-	struct ib_rmpp_hdr rmpp_hdr;
-	struct ib_sa_hdr sa_hdr;
-	uint32_t reserved0[2];
-	struct ib_gid dgid;
-	struct ib_gid sgid;
-	uint16_t dlid;
-	uint16_t slid;
-	uint32_t hop_limit__flow_label__raw_traffic;
-	uint32_t pkey__numb_path__reversible__tclass;
-	uint8_t reserved1;
-	uint8_t reserved__sl;
-	uint8_t mtu_selector__mtu;
-	uint8_t rate_selector__rate;
-	uint32_t preference__packet_lifetime__packet_lifetime_selector;
-	uint32_t reserved2[35];
-} __attribute__ (( packed ));
-
-struct ib_mad_mc_member_record {
-	struct ib_mad_hdr mad_hdr;
-	struct ib_rmpp_hdr rmpp_hdr;
-	struct ib_sa_hdr sa_hdr;
-	struct ib_gid mgid;
-	struct ib_gid port_gid;
-	uint32_t qkey;
-	uint16_t mlid;
-	uint8_t mtu_selector__mtu;
-	uint8_t tclass;
-	uint16_t pkey;
-	uint8_t rate_selector__rate;
-	uint8_t packet_lifetime_selector__packet_lifetime;
-	uint32_t sl__flow_label__hop_limit;
-	uint8_t scope__join_state;
-	uint8_t proxy_join__reserved;
-	uint16_t reserved0;
-	uint32_t reserved1[37];
-} __attribute__ (( packed ));
-
-union ib_mad {
-	struct ib_mad_hdr mad_hdr;
-	struct ib_mad_data data;
-	struct ib_mad_guid_info guid_info;
-	struct ib_mad_port_info port_info;
-	struct ib_mad_pkey_table pkey_table;
-	struct ib_mad_path_record path_record;
-	struct ib_mad_mc_member_record mc_member_record;
-} __attribute__ (( packed ));
-
 #endif /* _GPXE_INFINIBAND_H */