From 6b90a6d66b17bfe09351e18c705cb4a2ed147300 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:18 +0200 Subject: IB/Verbs: Implement new callback query_protocol() Add new callback query_protocol() and implement for each HW. Mapping List: node-type link-layer transport protocol nes RNIC ETH IWARP IWARP amso1100 RNIC ETH IWARP IWARP cxgb3 RNIC ETH IWARP IWARP cxgb4 RNIC ETH IWARP IWARP usnic USNIC_UDP ETH USNIC_UDP USNIC_UDP ocrdma IB_CA ETH IB IBOE mlx4 IB_CA IB/ETH IB IB/IBOE mlx5 IB_CA IB IB IB ehca IB_CA IB IB IB ipath IB_CA IB IB IB mthca IB_CA IB IB IB qib IB_CA IB IB IB Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 65994a19e840..080f204273e4 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -81,6 +81,13 @@ enum rdma_transport_type { RDMA_TRANSPORT_USNIC_UDP }; +enum rdma_protocol_type { + RDMA_PROTOCOL_IB, + RDMA_PROTOCOL_IBOE, + RDMA_PROTOCOL_IWARP, + RDMA_PROTOCOL_USNIC_UDP +}; + __attribute_const__ enum rdma_transport_type rdma_node_get_transport(enum rdma_node_type node_type); @@ -1501,6 +1508,8 @@ struct ib_device { int (*query_port)(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); + enum rdma_protocol_type (*query_protocol)(struct ib_device *device, + u8 port_num); enum rdma_link_layer (*get_link_layer)(struct ib_device *device, u8 port_num); int (*query_gid)(struct ib_device *device, -- cgit v1.2.3 From de66be94749d75c2f3578f3d01f91d31a8eb85ef Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:19 +0200 Subject: IB/Verbs: Implement raw management helpers Add raw helpers: rdma_protocol_ib rdma_protocol_iboe rdma_protocol_iwarp rdma_ib_or_iboe (transition, clean up later) To help us detect which technology the port supported. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 080f204273e4..e6dd9846b6c2 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1752,6 +1752,28 @@ int ib_query_port(struct ib_device *device, enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num); +static inline bool rdma_protocol_ib(struct ib_device *device, u8 port_num) +{ + return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IB; +} + +static inline bool rdma_protocol_iboe(struct ib_device *device, u8 port_num) +{ + return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IBOE; +} + +static inline bool rdma_protocol_iwarp(struct ib_device *device, u8 port_num) +{ + return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IWARP; +} + +static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) +{ + enum rdma_protocol_type pt = device->query_protocol(device, port_num); + + return (pt == RDMA_PROTOCOL_IB || pt == RDMA_PROTOCOL_IBOE); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From c757dea816407dc472452091e3ea941cef6638a2 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:32 +0200 Subject: IB/Verbs: Use management helper rdma_cap_ib_mad() Introduce helper rdma_cap_ib_mad() to help us check if the port of an IB device support Infiniband Management Datagrams. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index e6dd9846b6c2..23ba66e25b7f 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1774,6 +1774,21 @@ static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) return (pt == RDMA_PROTOCOL_IB || pt == RDMA_PROTOCOL_IBOE); } +/** + * rdma_cap_ib_mad - Check if the port of device has the capability Infiniband + * Management Datagrams. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support Infiniband + * Management Datagrams. + */ +static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) +{ + return rdma_ib_or_iboe(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From 29541e3add4d03682b78311823a5426e82019cda Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:33 +0200 Subject: IB/Verbs: Use management helper rdma_cap_ib_smi() Introduce helper rdma_cap_ib_smi() to help us check if the port of an IB device support Infiniband Subnet Management Interface. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 23ba66e25b7f..e983e335af21 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1789,6 +1789,21 @@ static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) return rdma_ib_or_iboe(device, port_num); } +/** + * rdma_cap_ib_smi - Check if the port of device has the capability Infiniband + * Subnet Management Interface. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support Infiniband + * Subnet Management Interface. + */ +static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) +{ + return rdma_protocol_ib(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From 72219cea8e246a55bff92e5ff6ec21f331a8791e Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:34 +0200 Subject: IB/Verbs: Use management helper rdma_cap_ib_cm() Introduce helper rdma_cap_ib_cm() to help us check if the port of an IB device support Infiniband Communication Manager. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index e983e335af21..e349596fd500 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1804,6 +1804,21 @@ static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) return rdma_protocol_ib(device, port_num); } +/** + * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband + * Communication Manager. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support Infiniband + * Communication Manager. + */ +static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) +{ + return rdma_ib_or_iboe(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From 042153306d9d08da67459f187d63a68aefd97388 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:35 +0200 Subject: IB/Verbs: Use management helper rdma_cap_iw_cm() Introduce helper rdma_cap_iw_cm() to help us check if the port of an IB device support IWARP Communication Manager. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index e349596fd500..cc92a6489136 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1819,6 +1819,21 @@ static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) return rdma_ib_or_iboe(device, port_num); } +/** + * rdma_cap_iw_cm - Check if the port of device has the capability IWARP + * Communication Manager. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support IWARP + * Communication Manager. + */ +static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) +{ + return rdma_protocol_iwarp(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From fe53ba2f0c3de0416422407bab2c1982a2e85b6a Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:36 +0200 Subject: IB/Verbs: Use management helper rdma_cap_ib_sa() Introduce helper rdma_cap_ib_sa() to help us check if the port of an IB device support Infiniband Subnet Administration. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index cc92a6489136..c3a561e891b1 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1834,6 +1834,21 @@ static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) return rdma_protocol_iwarp(device, port_num); } +/** + * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband + * Subnet Administration. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support Infiniband + * Subnet Administration. + */ +static inline bool rdma_cap_ib_sa(struct ib_device *device, u8 port_num) +{ + return rdma_protocol_ib(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From a31ad3b0e35f7e340c1ab6668080cff91d48c90a Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:37 +0200 Subject: IB/Verbs: Use management helper rdma_cap_ib_mcast() Introduce helper rdma_cap_ib_mcast() to help us check if the port of an IB device support Infiniband Multicast. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index c3a561e891b1..6bbbc86d39d9 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1849,6 +1849,21 @@ static inline bool rdma_cap_ib_sa(struct ib_device *device, u8 port_num) return rdma_protocol_ib(device, port_num); } +/** + * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband + * Multicast. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support Infiniband + * Multicast. + */ +static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) +{ + return rdma_cap_ib_sa(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From bc0f1d71536063f8b2df966625e0136bca03b3e6 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:38 +0200 Subject: IB/Verbs: Use management helper rdma_cap_read_multi_sge() Introduce helper rdma_cap_read_multi_sge() to help us check if the port of an IB device support RDMA Read Multiple Scatter-Gather Entries. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 6bbbc86d39d9..2cf23b130f9f 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1864,6 +1864,22 @@ static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) return rdma_cap_ib_sa(device, port_num); } +/** + * rdma_cap_read_multi_sge - Check if the port of device has the capability + * RDMA Read Multiple Scatter-Gather Entries. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support + * RDMA Read Multiple Scatter-Gather Entries. + */ +static inline bool rdma_cap_read_multi_sge(struct ib_device *device, + u8 port_num) +{ + return !rdma_protocol_iwarp(device, port_num); +} + int ib_query_gid(struct ib_device *device, u8 port_num, int index, union ib_gid *gid); -- cgit v1.2.3 From 30a74ef41d5293cb2f85fcce120fe869a672ade4 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:39 +0200 Subject: IB/Verbs: Use management helper rdma_cap_af_ib() Introduce helper rdma_cap_af_ib() to help us check if the port of an IB device support Native Infiniband Address. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 2cf23b130f9f..349d1216564c 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1864,6 +1864,21 @@ static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) return rdma_cap_ib_sa(device, port_num); } +/** + * rdma_cap_af_ib - Check if the port of device has the capability + * Native Infiniband Address. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support + * Native Infiniband Address. + */ +static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) +{ + return rdma_ib_or_iboe(device, port_num); +} + /** * rdma_cap_read_multi_sge - Check if the port of device has the capability * RDMA Read Multiple Scatter-Gather Entries. -- cgit v1.2.3 From 227128fc68401d8e36b660ffeef4320c5fb492d7 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 5 May 2015 14:50:40 +0200 Subject: IB/Verbs: Use management helper rdma_cap_eth_ah() Introduce helper rdma_cap_eth_ah() to help us check if the port of an IB device support Ethernet Address Handler. Signed-off-by: Michael Wang Reviewed-by: Ira Weiny Tested-by: Ira Weiny Reviewed-by: Sean Hefty Reviewed-by: Jason Gunthorpe Tested-by: Doug Ledford Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 349d1216564c..8d59479eea4d 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1879,6 +1879,21 @@ static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) return rdma_ib_or_iboe(device, port_num); } +/** + * rdma_cap_eth_ah - Check if the port of device has the capability + * Ethernet Address Handler. + * + * @device: Device to be checked + * @port_num: Port number of the device + * + * Return false when port of the device don't support + * Ethernet Address Handler. + */ +static inline bool rdma_cap_eth_ah(struct ib_device *device, u8 port_num) +{ + return rdma_protocol_iboe(device, port_num); +} + /** * rdma_cap_read_multi_sge - Check if the port of device has the capability * RDMA Read Multiple Scatter-Gather Entries. -- cgit v1.2.3 From 296ec00995fb28c4e34b41f80b5a876f3a25c134 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Mon, 18 May 2015 10:41:45 +0200 Subject: IB/Verbs: Improve docs for rdma-helpers Increase the level of documentation for the rdma_cap_* helpers introduced by Michael Wang . This patch is loosely based on a patch Michael wrote to enhance the documentation of these functions, but has been significantly modified in terms of verbiage. In addition, the comments were moved from a kernel Documentation/infiniband/ file to being inline in the header file itself for the functions in question. Finally, the documentation was formated in proper kdoc format. Signed-off-by: Michael Wang Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 132 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 40 deletions(-) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 8d59479eea4d..81740c14fdb1 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1775,14 +1775,16 @@ static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) } /** - * rdma_cap_ib_mad - Check if the port of device has the capability Infiniband + * rdma_cap_ib_mad - Check if the port of a device supports Infiniband * Management Datagrams. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * Management Datagrams (MAD) are a required part of the InfiniBand + * specification and are supported on all InfiniBand devices. A slightly + * extended version are also supported on OPA interfaces. * - * Return false when port of the device don't support Infiniband - * Management Datagrams. + * Return: true if the port supports sending/receiving of MAD packets. */ static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) { @@ -1790,14 +1792,24 @@ static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) } /** - * rdma_cap_ib_smi - Check if the port of device has the capability Infiniband - * Subnet Management Interface. + * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband + * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI). + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * Each InfiniBand node is required to provide a Subnet Management Agent + * that the subnet manager can access. Prior to the fabric being fully + * configured by the subnet manager, the SMA is accessed via a well known + * interface called the Subnet Management Interface (SMI). This interface + * uses directed route packets to communicate with the SM to get around the + * chicken and egg problem of the SM needing to know what's on the fabric + * in order to configure the fabric, and needing to configure the fabric in + * order to send packets to the devices on the fabric. These directed + * route packets do not need the fabric fully configured in order to reach + * their destination. The SMI is the only method allowed to send + * directed route packets on an InfiniBand fabric. * - * Return false when port of the device don't support Infiniband - * Subnet Management Interface. + * Return: true if the port provides an SMI. */ static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) { @@ -1807,12 +1819,17 @@ static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) /** * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband * Communication Manager. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * The InfiniBand Communication Manager is one of many pre-defined General + * Service Agents (GSA) that are accessed via the General Service + * Interface (GSI). It's role is to facilitate establishment of connections + * between nodes as well as other management related tasks for established + * connections. * - * Return false when port of the device don't support Infiniband - * Communication Manager. + * Return: true if the port supports an IB CM (this does not guarantee that + * a CM is actually running however). */ static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) { @@ -1822,12 +1839,14 @@ static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) /** * rdma_cap_iw_cm - Check if the port of device has the capability IWARP * Communication Manager. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * Similar to above, but specific to iWARP connections which have a different + * managment protocol than InfiniBand. * - * Return false when port of the device don't support IWARP - * Communication Manager. + * Return: true if the port supports an iWARP CM (this does not guarantee that + * a CM is actually running however). */ static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) { @@ -1837,12 +1856,17 @@ static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) /** * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband * Subnet Administration. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * An InfiniBand Subnet Administration (SA) service is a pre-defined General + * Service Agent (GSA) provided by the Subnet Manager (SM). On InfiniBand + * fabrics, devices should resolve routes to other hosts by contacting the + * SA to query the proper route. * - * Return false when port of the device don't support Infiniband - * Subnet Administration. + * Return: true if the port should act as a client to the fabric Subnet + * Administration interface. This does not imply that the SA service is + * running locally. */ static inline bool rdma_cap_ib_sa(struct ib_device *device, u8 port_num) { @@ -1852,12 +1876,19 @@ static inline bool rdma_cap_ib_sa(struct ib_device *device, u8 port_num) /** * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband * Multicast. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * InfiniBand multicast registration is more complex than normal IPv4 or + * IPv6 multicast registration. Each Host Channel Adapter must register + * with the Subnet Manager when it wishes to join a multicast group. It + * should do so only once regardless of how many queue pairs it subscribes + * to this group. And it should leave the group only after all queue pairs + * attached to the group have been detached. * - * Return false when port of the device don't support Infiniband - * Multicast. + * Return: true if the port must undertake the additional adminstrative + * overhead of registering/unregistering with the SM and tracking of the + * total number of queue pairs attached to the multicast group. */ static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) { @@ -1867,12 +1898,15 @@ static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) /** * rdma_cap_af_ib - Check if the port of device has the capability * Native Infiniband Address. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default + * GID. RoCE uses a different mechanism, but still generates a GID via + * a prescribed mechanism and port specific data. * - * Return false when port of the device don't support - * Native Infiniband Address. + * Return: true if the port uses a GID address to identify devices on the + * network. */ static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) { @@ -1881,13 +1915,19 @@ static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) /** * rdma_cap_eth_ah - Check if the port of device has the capability - * Ethernet Address Handler. + * Ethernet Address Handle. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * RoCE is InfiniBand over Ethernet, and it uses a well defined technique + * to fabricate GIDs over Ethernet/IP specific addresses native to the + * port. Normally, packet headers are generated by the sending host + * adapter, but when sending connectionless datagrams, we must manually + * inject the proper headers for the fabric we are communicating over. * - * Return false when port of the device don't support - * Ethernet Address Handler. + * Return: true if we are running as a RoCE port and must force the + * addition of a Global Route Header built from our Ethernet Address + * Handle into our header list for connectionless packets. */ static inline bool rdma_cap_eth_ah(struct ib_device *device, u8 port_num) { @@ -1897,12 +1937,24 @@ static inline bool rdma_cap_eth_ah(struct ib_device *device, u8 port_num) /** * rdma_cap_read_multi_sge - Check if the port of device has the capability * RDMA Read Multiple Scatter-Gather Entries. + * @device: Device to check + * @port_num: Port number to check * - * @device: Device to be checked - * @port_num: Port number of the device + * iWARP has a restriction that RDMA READ requests may only have a single + * Scatter/Gather Entry (SGE) in the work request. * - * Return false when port of the device don't support - * RDMA Read Multiple Scatter-Gather Entries. + * NOTE: although the linux kernel currently assumes all devices are either + * single SGE RDMA READ devices or identical SGE maximums for RDMA READs and + * WRITEs, according to Tom Talpey, this is not accurate. There are some + * devices out there that support more than a single SGE on RDMA READ + * requests, but do not support the same number of SGEs as they do on + * RDMA WRITE requests. The linux kernel would need rearchitecting to + * support these imbalanced READ/WRITE SGEs allowed devices. So, for now, + * suffice with either the device supports the same READ/WRITE SGEs, or + * it only gets one READ sge. + * + * Return: true for any device that allows more than one SGE in RDMA READ + * requests. */ static inline bool rdma_cap_read_multi_sge(struct ib_device *device, u8 port_num) -- cgit v1.2.3 From 0cf18d7723055709faf51b50f5a33253b480637f Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 13 May 2015 20:02:55 -0400 Subject: IB/core: Create common start/end port functions Previously start_port and end_port were defined in 2 places, cache.c and device.c and this prevented their use in other modules. Make these common functions, change the name to reflect the rdma name space, and update existing users. Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 81740c14fdb1..be4465b5df7b 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1752,6 +1752,33 @@ int ib_query_port(struct ib_device *device, enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num); +/** + * rdma_start_port - Return the first valid port number for the device + * specified + * + * @device: Device to be checked + * + * Return start port number + */ +static inline u8 rdma_start_port(const struct ib_device *device) +{ + return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1; +} + +/** + * rdma_end_port - Return the last valid port number for the device + * specified + * + * @device: Device to be checked + * + * Return last port number + */ +static inline u8 rdma_end_port(const struct ib_device *device) +{ + return (device->node_type == RDMA_NODE_IB_SWITCH) ? + 0 : device->phys_port_cnt; +} + static inline bool rdma_protocol_ib(struct ib_device *device, u8 port_num) { return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IB; -- cgit v1.2.3 From 2b1b5b601230ae4356be4724ea7a058ed7203c63 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 18 May 2015 13:40:28 +0300 Subject: IB/core, cma: Nice log-friendly string helpers Some of us keep revisiting the code to decode enumerations that appear in out logs. Let's borrow the nice logging helpers that exists in xprtrdma and rds for CMA events, IB events and WC statuses. Reviewd-by: Sean Hefty Signed-off-by: Sagi Grimberg Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 4 ++++ include/rdma/rdma_cm.h | 2 ++ 2 files changed, 6 insertions(+) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 65994a19e840..672fc8f20409 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -412,6 +412,8 @@ enum ib_event_type { IB_EVENT_GID_CHANGE, }; +__attribute_const__ const char *ib_event_msg(enum ib_event_type event); + struct ib_event { struct ib_device *device; union { @@ -663,6 +665,8 @@ enum ib_wc_status { IB_WC_GENERAL_ERR }; +__attribute_const__ const char *ib_wc_status_msg(enum ib_wc_status status); + enum ib_wc_opcode { IB_WC_SEND, IB_WC_RDMA_WRITE, diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h index 1ed2088dc9f5..c92522c192d2 100644 --- a/include/rdma/rdma_cm.h +++ b/include/rdma/rdma_cm.h @@ -62,6 +62,8 @@ enum rdma_cm_event_type { RDMA_CM_EVENT_TIMEWAIT_EXIT }; +__attribute_const__ const char *rdma_event_msg(enum rdma_cm_event_type event); + enum rdma_port_space { RDMA_PS_SDP = 0x0001, RDMA_PS_IPOIB = 0x0002, -- cgit v1.2.3 From 96909308542006215980874ce496101ea51aa031 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Fri, 8 May 2015 14:27:22 -0400 Subject: IB/mad: Change ib_response_mad signature arguments ib_response_mad only needs read access to the MAD header, not write access to the entire mad struct, so replace struct ib_mad with const struct ib_mad_hdr Reviewed-By: Jason Gunthorpe Reviewed-by: Sean Hefty Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_mad.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h index 9bb99e983f58..69e3c45e7d46 100644 --- a/include/rdma/ib_mad.h +++ b/include/rdma/ib_mad.h @@ -263,7 +263,7 @@ struct ib_mad_send_buf { * ib_response_mad - Returns if the specified MAD has been generated in * response to a sent request or trap. */ -int ib_response_mad(struct ib_mad *mad); +int ib_response_mad(const struct ib_mad_hdr *hdr); /** * ib_get_rmpp_resptime - Returns the RMPP response time. -- cgit v1.2.3 From f766c58fa3ea38a30912df8b5af3ca40637fe5e9 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Fri, 8 May 2015 14:27:24 -0400 Subject: IB/mad: Add const qualifiers to query only functions The following functions only need read access to the data passed to them. ib_mad_kernel_rmpp_agent is_rmpp_data_mad rcv_has_same_gid ib_find_send_mad Clarify with const specifiers Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_mad.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h index 69e3c45e7d46..c0ea51f90a03 100644 --- a/include/rdma/ib_mad.h +++ b/include/rdma/ib_mad.h @@ -675,6 +675,6 @@ void ib_free_send_mad(struct ib_mad_send_buf *send_buf); * @agent: the agent in question * @return: true if agent is performing rmpp, false otherwise. */ -int ib_mad_kernel_rmpp_agent(struct ib_mad_agent *agent); +int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent); #endif /* IB_MAD_H */ -- cgit v1.2.3 From 7738613e7cb419179545910744b1777d87edac5c Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 13 May 2015 20:02:58 -0400 Subject: IB/core: Add per port immutable struct to ib_device As of commit 5eb620c81ce3 "IB/core: Add helpers for uncached GID and P_Key searches"; pkey_tbl_len and gid_tbl_len are immutable data which are stored in the ib_device. The per port core capability flags to be added later are also immutable data to be stored in the ib_device object. In preparation for this create a structure for per port immutable data and place the pkey and gid table lengths within this structure. "get_port_immutable" is added as a mandatory device function to allow the drivers to fill in this data. Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index be4465b5df7b..2d3515edc3fa 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1481,6 +1481,11 @@ struct ib_dma_mapping_ops { struct iw_cm_verbs; +struct ib_port_immutable { + int pkey_tbl_len; + int gid_tbl_len; +}; + struct ib_device { struct device *dma_device; @@ -1494,8 +1499,10 @@ struct ib_device { struct list_head client_data_list; struct ib_cache cache; - int *pkey_tbl_len; - int *gid_tbl_len; + /** + * port_immutable is indexed by port number + */ + struct ib_port_immutable *port_immutable; int num_comp_vectors; @@ -1684,6 +1691,14 @@ struct ib_device { u32 local_dma_lkey; u8 node_type; u8 phys_port_cnt; + + /** + * The following mandatory functions are used only at device + * registration. Keep functions such as these at the end of this + * structure to avoid cache line misses when accessing struct ib_device + * in fast paths. + */ + int (*get_port_immutable)(struct ib_device *, u8, struct ib_port_immutable *); }; struct ib_client { -- cgit v1.2.3 From f9b22e355d38c8dbfa19a2d9d5ef9bf07e7c17e6 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 13 May 2015 20:02:59 -0400 Subject: IB/core: Convert core to use bitfield for caps Remove query_protocol callback Use the new Core Capability bits for: rdma_protocol_* rdma_cap_ib_mad rdma_cap_ib_smi rdma_cap_ib_cm rdma_cap_iw_cm rdma_cap_ib_sa rdma_cap_ib_mcast rdma_cap_af_ib rdma_cap_eth_ah Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 64 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 2d3515edc3fa..73d1b1000785 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -353,6 +353,40 @@ union rdma_protocol_stats { struct iw_protocol_stats iw; }; +/* Define bits for the various functionality this port needs to be supported by + * the core. + */ +/* Management 0x00000FFF */ +#define RDMA_CORE_CAP_IB_MAD 0x00000001 +#define RDMA_CORE_CAP_IB_SMI 0x00000002 +#define RDMA_CORE_CAP_IB_CM 0x00000004 +#define RDMA_CORE_CAP_IW_CM 0x00000008 +#define RDMA_CORE_CAP_IB_SA 0x00000010 + +/* Address format 0x000FF000 */ +#define RDMA_CORE_CAP_AF_IB 0x00001000 +#define RDMA_CORE_CAP_ETH_AH 0x00002000 + +/* Protocol 0xFFF00000 */ +#define RDMA_CORE_CAP_PROT_IB 0x00100000 +#define RDMA_CORE_CAP_PROT_ROCE 0x00200000 +#define RDMA_CORE_CAP_PROT_IWARP 0x00400000 + +#define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \ + | RDMA_CORE_CAP_IB_MAD \ + | RDMA_CORE_CAP_IB_SMI \ + | RDMA_CORE_CAP_IB_CM \ + | RDMA_CORE_CAP_IB_SA \ + | RDMA_CORE_CAP_AF_IB) +#define RDMA_CORE_PORT_IBA_ROCE (RDMA_CORE_CAP_PROT_ROCE \ + | RDMA_CORE_CAP_IB_MAD \ + | RDMA_CORE_CAP_IB_CM \ + | RDMA_CORE_CAP_IB_SA \ + | RDMA_CORE_CAP_AF_IB \ + | RDMA_CORE_CAP_ETH_AH) +#define RDMA_CORE_PORT_IWARP (RDMA_CORE_CAP_PROT_IWARP \ + | RDMA_CORE_CAP_IW_CM) + struct ib_port_attr { enum ib_port_state state; enum ib_mtu max_mtu; @@ -1484,6 +1518,7 @@ struct iw_cm_verbs; struct ib_port_immutable { int pkey_tbl_len; int gid_tbl_len; + u32 core_cap_flags; }; struct ib_device { @@ -1515,8 +1550,6 @@ struct ib_device { int (*query_port)(struct ib_device *device, u8 port_num, struct ib_port_attr *port_attr); - enum rdma_protocol_type (*query_protocol)(struct ib_device *device, - u8 port_num); enum rdma_link_layer (*get_link_layer)(struct ib_device *device, u8 port_num); int (*query_gid)(struct ib_device *device, @@ -1796,24 +1829,23 @@ static inline u8 rdma_end_port(const struct ib_device *device) static inline bool rdma_protocol_ib(struct ib_device *device, u8 port_num) { - return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IB; + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IB; } static inline bool rdma_protocol_iboe(struct ib_device *device, u8 port_num) { - return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IBOE; + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_ROCE; } static inline bool rdma_protocol_iwarp(struct ib_device *device, u8 port_num) { - return device->query_protocol(device, port_num) == RDMA_PROTOCOL_IWARP; + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IWARP; } static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) { - enum rdma_protocol_type pt = device->query_protocol(device, port_num); - - return (pt == RDMA_PROTOCOL_IB || pt == RDMA_PROTOCOL_IBOE); + return device->port_immutable[port_num].core_cap_flags & + (RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_PROT_ROCE); } /** @@ -1830,7 +1862,7 @@ static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) { - return rdma_ib_or_iboe(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_IB_MAD; } /** @@ -1855,7 +1887,7 @@ static inline bool rdma_cap_ib_mad(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) { - return rdma_protocol_ib(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_IB_SMI; } /** @@ -1875,7 +1907,7 @@ static inline bool rdma_cap_ib_smi(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) { - return rdma_ib_or_iboe(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_IB_CM; } /** @@ -1892,7 +1924,7 @@ static inline bool rdma_cap_ib_cm(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) { - return rdma_protocol_iwarp(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_IW_CM; } /** @@ -1912,7 +1944,7 @@ static inline bool rdma_cap_iw_cm(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_ib_sa(struct ib_device *device, u8 port_num) { - return rdma_protocol_ib(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_IB_SA; } /** @@ -1952,7 +1984,7 @@ static inline bool rdma_cap_ib_mcast(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) { - return rdma_ib_or_iboe(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_AF_IB; } /** @@ -1973,7 +2005,7 @@ static inline bool rdma_cap_af_ib(struct ib_device *device, u8 port_num) */ static inline bool rdma_cap_eth_ah(struct ib_device *device, u8 port_num) { - return rdma_protocol_iboe(device, port_num); + return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_ETH_AH; } /** @@ -2001,7 +2033,7 @@ static inline bool rdma_cap_eth_ah(struct ib_device *device, u8 port_num) static inline bool rdma_cap_read_multi_sge(struct ib_device *device, u8 port_num) { - return !rdma_protocol_iwarp(device, port_num); + return !(device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IWARP); } int ib_query_gid(struct ib_device *device, -- cgit v1.2.3 From 5d9fb0440698a8b9e8595353d60cfac7ab30efae Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 14 May 2015 15:01:46 -0400 Subject: IB/core: Change rdma_protocol_iboe to roce After discussion upstream, it was agreed to transition the usage of iboe in the kernel to roce. This keeps our terminology consistent with what was finalized in the IBTA Annex 16 and IBTA Annex 17 publications. Signed-off-by: Ira Weiny Signed-off-by: Doug Ledford --- include/rdma/ib_verbs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 73d1b1000785..3ebf0c019a66 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1832,7 +1832,7 @@ static inline bool rdma_protocol_ib(struct ib_device *device, u8 port_num) return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IB; } -static inline bool rdma_protocol_iboe(struct ib_device *device, u8 port_num) +static inline bool rdma_protocol_roce(struct ib_device *device, u8 port_num) { return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_ROCE; } @@ -1842,7 +1842,7 @@ static inline bool rdma_protocol_iwarp(struct ib_device *device, u8 port_num) return device->port_immutable[port_num].core_cap_flags & RDMA_CORE_CAP_PROT_IWARP; } -static inline bool rdma_ib_or_iboe(struct ib_device *device, u8 port_num) +static inline bool rdma_ib_or_roce(struct ib_device *device, u8 port_num) { return device->port_immutable[port_num].core_cap_flags & (RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_PROT_ROCE); -- cgit v1.2.3