Hi Erez,
my configuration is as follows
- nova compute nodes are equipped with Mellanox ConnectX-3 (MT27500) SR-IOV enabled. They are running Mellanox eSwitchd, Mellanox VIF driver and Mellanox Neutron Agent (for details, see my post Mellanox eSwitchd issue on Openstack Havana nova-compute). Of course no eIPoIB driver here, neither on the hosts (nova compute) nor on the guests.
- the network node is equipped with a Mellanox ConnectX HCA
[root@n02 ~]# ibstat mlx4_0
CA 'mlx4_0'
CA type: MT26428
Number of ports: 1
Firmware version: 2.9.1000
Hardware version: b0
Node GUID: 0x0002c9030050608c
System image GUID: 0x0002c9030050608f
Port 1:
State: Active
Physical state: LinkUp
Rate: 40
Base lid: 26
LMC: 0
SM lid: 15
Capability mask: 0x02510868
Port GUID: 0x0002c9030050608d
Link layer: InfiniBand
It is configured in paravirtualized mode using the eIPoIB driver as described here: Mellanox-Neutron-Havana-Redhat - OpenStack. The eIPoIB interface is eth2 and it's used by Linux Bridge as the physical interface for the IB network (default:1:6)
[root@n02 ~]# grep ^[\[a-z] /etc/neutron/plugin.ini
[vlans]
tenant_network_type = ib
network_vlan_ranges = default:1:6,external
[linux_bridge]
physical_interface_mappings = default:eth2,external:eth1.55
[vxlan]
[agent]
rpc_support_old_agents = True
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
I'm testing with only one tenant IB network so Linux Bridge have created only one tagged eth2 interface on the vlan 1
[root@n02 ~]# ip link show | grep -A1 eth2
7: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 00:02:c9:50:60:8d brd ff:ff:ff:ff:ff:ff
--
15: eth2.1@eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP
link/ether 00:02:c9:50:60:8d brd ff:ff:ff:ff:ff:ff
this interface eth2.1 is bridged with the tap devices for the DHCP and the external router
[root@n02 ~]# brctl show brq570dc494-a8
bridge name bridge id STP enabled interfaces
brq570dc494-a8 8000.0002c950608d no eth2.1
tap03eecdb6-1b
tapbf3135d2-4b
this is the namespace for the external router
[root@n02 ~]# ip netns exec qrouter-6e95aabe-b64a-4139-b12a-1413444071c1 ip a |grep -A2 'qr-'
16: qr-03eecdb6-1b: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether fa:16:3e:52:32:68 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.1/24 brd 10.0.0.255 scope global qr-03eecdb6-1b
inet6 fe80::f816:3eff:fe52:3268/64 scope link
valid_lft forever preferred_lft forever
this is the namespace for the DHCP
[root@n02 ~]# ip netns exec qdhcp-570dc494-a876-401c-a80d-6696e2140a5f ip a |grep -A2 'ns-'
13: ns-bf3135d2-4b: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether fa:16:3e:04:4c:f7 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.3/24 brd 10.0.0.255 scope global ns-bf3135d2-4b
inet6 fe80::f816:3eff:fe04:4cf7/64 scope link
valid_lft forever preferred_lft forever
in order to get it work I have to manually create two ib0 clones with the first pkey (8001) and enslave them to eIPoIB interface (eth2)
[root@n02 ~]# tail /sys/class/net/eth2/eth/{slaves,vifs}
==> /sys/class/net/eth2/eth/slaves <==
ib0.8001.1
ib0.8001.2
==> /sys/class/net/eth2/eth/vifs <==
SLAVE=ib0.8001.1 MAC=fa:16:3e:04:4c:f7 VLAN=1
SLAVE=ib0.8001.2 MAC=fa:16:3e:52:32:68 VLAN=1
- finally, the subnet manager is running on my cloud controller that is also equipped with a Mellanox ConnectX HCA (MT26428).
[root@n01 ~(keystone_admin)]# cat /etc/opensm/partitions.conf
Default=0xffff, ipoib, mtu=4 : ALL=full;
management=0x7fff, ipoib, sl=0, defmember=full : ALL, ALL_SWITCHES=full,SELF=full;
vlan1=0x1, ipoib, sl=0, defmember=full : ALL;
vlan2=0x2, ipoib, sl=0, defmember=full : ALL;
vlan3=0x3, ipoib, sl=0, defmember=full : ALL;
vlan4=0x4, ipoib, sl=0, defmember=full : ALL;
vlan5=0x5, ipoib, sl=0, defmember=full : ALL;
vlan6=0x6, ipoib, sl=0, defmember=full : ALL;
Thank you very much.
Ale