test D30155
Contents
Testbed s1 <==> r1 (d820 nodes with a 10Gbps link) used here are from Emulab.net: Emulab hardware
default `ix` hardware info in FreeBSD 15 CURRENT by commit 6e414739fc
root@s1:~ # ifconfig ix2
ix2: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=4e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS,MEXTPG>
ether a0:36:9f:08:22:b0
inet 10.1.1.2 netmask 0xffffff00 broadcast 10.1.1.255
media: Ethernet autoselect (10Gbase-Twinax <full-duplex,rxpause,txpause>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
root@s1:~ # pciconf -lv | rg "ix2" -A 4
ix2@pci0:66:0:0: class=0x020000 rev=0x01 hdr=0x00 vendor=0x8086 device=0x154d subvendor=0x8086 subdevice=0x7b11
vendor = 'Intel Corporation'
device = 'Ethernet 10G 2P X520 Adapter'
class = network
subclass = ethernet
root@s1:~ #
root@r1:~ # ifconfig ix2
ix2: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=4e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS,MEXTPG>
ether a0:36:9f:07:33:90
inet 10.1.1.3 netmask 0xffffff00 broadcast 10.1.1.255
media: Ethernet autoselect (10Gbase-Twinax <full-duplex,rxpause,txpause>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
root@r1:~ # pciconf -lv | rg "ix2" -A 4
ix2@pci0:66:0:0: class=0x020000 rev=0x01 hdr=0x00 vendor=0x8086 device=0x154d subvendor=0x8086 subdevice=0x7b11
vendor = 'Intel Corporation'
device = 'Ethernet 10G 2P X520 Adapter'
class = network
subclass = ethernet
root@r1:~ #
default `ixgbe` hardware info in Ubuntu-22.04 Linux 5.15.0-112-generic
root@s1:~# ifconfig enp66s0f0
enp66s0f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.1.1.2 netmask 255.255.255.0 broadcast 10.1.1.255
inet6 fe80::a236:9fff:fe08:22b0 prefixlen 64 scopeid 0x20<link>
ether a0:36:9f:08:22:b0 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14 bytes 1076 (1.0 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
root@s1:~# ethtool -i enp66s0f0
driver: ixgbe
version: 5.15.0-112-generic
firmware-version: 0x8000095d, 13.1.10
expansion-rom-version:
bus-info: 0000:42:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
root@s1:~# lspci -v -s 42:00.0
42:00.0 Ethernet controller: Intel Corporation Ethernet 10G 2P X520 Adapter (rev 01)
Subsystem: Intel Corporation 10GbE 2P X520 Adapter
Flags: bus master, fast devsel, latency 0, IRQ 182, NUMA node 1
Memory at d0d00000 (64-bit, non-prefetchable) [size=1M]
I/O ports at bcc0 [size=32]
Memory at d0ff8000 (64-bit, non-prefetchable) [size=16K]
Expansion ROM at d0000000 [disabled] [size=512K]
Capabilities: [40] Power Management version 3
Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
Capabilities: [70] MSI-X: Enable+ Count=64 Masked-
Capabilities: [a0] Express Endpoint, MSI 00
Capabilities: [e0] Vital Product Data
Capabilities: [100] Advanced Error Reporting
Capabilities: [140] Device Serial Number 00-00-00-ff-ff-00-00-00
Capabilities: [150] Alternative Routing-ID Interpretation (ARI)
Capabilities: [160] Single Root I/O Virtualization (SR-IOV)
Kernel driver in use: ixgbe
Kernel modules: ixgbe
root@s1:~#
root@r1:~# ifconfig enp66s0f0
enp66s0f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 10.1.1.3 netmask 255.255.255.0 broadcast 10.1.1.255
inet6 fe80::a236:9fff:fe07:3390 prefixlen 64 scopeid 0x20<link>
ether a0:36:9f:07:33:90 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14 bytes 1076 (1.0 KB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
root@r1:~# ethtool -i enp66s0f0
driver: ixgbe
version: 5.15.0-112-generic
firmware-version: 0x8000095d, 13.1.10
expansion-rom-version:
bus-info: 0000:42:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
root@r1:~# lspci -v -s 42:00.0
42:00.0 Ethernet controller: Intel Corporation Ethernet 10G 2P X520 Adapter (rev 01)
Subsystem: Intel Corporation 10GbE 2P X520 Adapter
Flags: bus master, fast devsel, latency 0, IRQ 183, NUMA node 1
Memory at d1d00000 (64-bit, non-prefetchable) [size=1M]
I/O ports at ccc0 [size=32]
Memory at d1ff8000 (64-bit, non-prefetchable) [size=16K]
Expansion ROM at d1000000 [disabled] [size=512K]
Capabilities: [40] Power Management version 3
Capabilities: [50] MSI: Enable- Count=1/1 Maskable+ 64bit+
Capabilities: [70] MSI-X: Enable+ Count=64 Masked-
Capabilities: [a0] Express Endpoint, MSI 00
Capabilities: [e0] Vital Product Data
Capabilities: [100] Advanced Error Reporting
Capabilities: [140] Device Serial Number 00-00-00-ff-ff-00-00-00
Capabilities: [150] Alternative Routing-ID Interpretation (ARI)
Capabilities: [160] Single Root I/O Virtualization (SR-IOV)
Kernel driver in use: ixgbe
Kernel modules: ixgbe
root@r1:~#
test IPv4 traffic
test ping-pong latency
test result
highlighted ping-pong latency under different enable_aim values
kernel version |
enable_aim value |
min/avg/max ms |
stddev |
delta |
FreeBSD 15.0-CURRENT stock |
0 (default) |
0.198/0.243/0.289 |
0.024 |
base0 |
1 |
0.215/0.237/0.261 |
0.013 |
base1 |
|
2 |
0.204/0.228/0.253 |
0.016 |
base2 |
|
FreeBSD 15.0-CURRENT D30155 |
0 |
0.216/0.227/0.244 |
0.009 |
- |
1 (default) |
0.207/0.240/0.254 |
0.012 |
- |
|
2 |
0.226/0.243/0.264 |
0.011 |
- |
|
Ubuntu Linux 5.15.0-112-generic |
default |
0.215/0.239/0.271 |
0.016 |
- |
test detail
- with enable_aim=0 in D30155 patched kernel
root@s1:~ # sysctl -d dev.ix.2.enable_aim dev.ix.2.enable_aim: Interrupt Moderation root@s1:~ # sysctl dev.ix.2.enable_aim // default value is 1 dev.ix.2.enable_aim: 1 root@s1:~ # sysctl dev.ix.2.enable_aim=0 dev.ix.2.enable_aim: 1 -> 0 root@r1:~ # sysctl -d dev.ix.2.enable_aim dev.ix.2.enable_aim: Interrupt Moderation root@r1:~ # sysctl dev.ix.2.enable_aim // default value is 1 dev.ix.2.enable_aim: 1 root@r1:~ # sysctl dev.ix.2.enable_aim=0 dev.ix.2.enable_aim: 1 -> 0 root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.238 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.217 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.226 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.244 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.224 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.222 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.220 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.234 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.229 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.216 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.216/0.227/0.244/0.009 ms
- with enable_aim=1 in D30155 patched kernel
root@s1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@s1:~ # root@r1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@r1:~ # root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.250 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.233 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.207 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.245 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.254 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.248 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.242 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.238 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.239 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.207/0.240/0.254/0.012 ms
- with enable_aim=2 in D30155 patched kernel
root@s1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@s1:~ # root@r1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.264 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.251 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.248 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.248 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.249 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.239 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.231 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.226 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.230 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.242 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.226/0.243/0.264/0.011 ms
* test ping on a stock kernel without the patch D30155
(1) root@s1:~ # sysctl dev.ix.2.enable_aim dev.ix.2.enable_aim: 0 root@r1:~ # sysctl dev.ix.2.enable_aim dev.ix.2.enable_aim: 0 root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.289 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.256 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.198 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.233 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.237 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.236 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.219 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.268 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.256 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.198/0.243/0.289/0.024 ms (2) root@s1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@r1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.254 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.222 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.261 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.233 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.215 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.231 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.232 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.215/0.237/0.261/0.013 ms (3) root@s1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@r1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@s1:~ # ping -S 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2: 56 data bytes 64 bytes from 10.1.1.3: icmp_seq=0 ttl=64 time=0.253 ms 64 bytes from 10.1.1.3: icmp_seq=1 ttl=64 time=0.244 ms 64 bytes from 10.1.1.3: icmp_seq=2 ttl=64 time=0.232 ms 64 bytes from 10.1.1.3: icmp_seq=3 ttl=64 time=0.204 ms 64 bytes from 10.1.1.3: icmp_seq=4 ttl=64 time=0.225 ms 64 bytes from 10.1.1.3: icmp_seq=5 ttl=64 time=0.206 ms 64 bytes from 10.1.1.3: icmp_seq=6 ttl=64 time=0.241 ms 64 bytes from 10.1.1.3: icmp_seq=7 ttl=64 time=0.237 ms 64 bytes from 10.1.1.3: icmp_seq=8 ttl=64 time=0.224 ms 64 bytes from 10.1.1.3: icmp_seq=9 ttl=64 time=0.211 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 packets received, 0.0% packet loss round-trip min/avg/max/stddev = 0.204/0.228/0.253/0.016 ms root@s1:~ #
* test ping on a stock Ubuntu 22.04 kernel 5.15.0-112-generic
root@s1:~# ping -I 10.1.1.2 -c 10 r1 PING r1-link1 (10.1.1.3) from 10.1.1.2 : 56(84) bytes of data. 64 bytes from r1-link1 (10.1.1.3): icmp_seq=1 ttl=64 time=0.215 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=2 ttl=64 time=0.232 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=3 ttl=64 time=0.248 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=4 ttl=64 time=0.235 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=5 ttl=64 time=0.262 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=6 ttl=64 time=0.271 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=7 ttl=64 time=0.242 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=8 ttl=64 time=0.236 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=9 ttl=64 time=0.226 ms 64 bytes from r1-link1 (10.1.1.3): icmp_seq=10 ttl=64 time=0.229 ms --- r1-link1 ping statistics --- 10 packets transmitted, 10 received, 0% packet loss, time 9218ms rtt min/avg/max/mdev = 0.215/0.239/0.271/0.016 ms
test tso performance under iperf3
The TSO support for IPv4/IPv6 is enabled in default.
test result
- highlighted performance stats with TSO enabled
kernel version |
enable_aim value |
iperf3 throughput |
delta |
FreeBSD 15.0-CURRENT stock |
0 (default) |
3.64 Gbits/sec |
base0 |
1 |
3.57 Gbits/sec |
base1 |
|
2 |
3.54 Gbits/sec |
base2 |
|
FreeBSD 15.0-CURRENT D30155 |
0 |
3.74 Gbits/sec |
+2.7% |
1 (default) |
3.79 Gbits/sec |
+6.2% |
|
2 |
3.78 Gbits/sec |
+6.8% |
|
Ubuntu Linux 5.15.0-112-generic |
default |
9.42 Gbits/sec |
- |
test detail
- with enable_aim=0
root@s1:~ # sysctl dev.ix.2.enable_aim=0 dev.ix.2.enable_aim: 2 -> 0 root@r1:~ # sysctl dev.ix.2.enable_aim=0 dev.ix.2.enable_aim: 2 -> 0 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 55267 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 703 MBytes 2.95 Gbits/sec 129 277 KBytes [ 5] 2.00-4.00 sec 936 MBytes 3.92 Gbits/sec 130 345 KBytes [ 5] 4.00-6.00 sec 943 MBytes 3.96 Gbits/sec 165 215 KBytes [ 5] 6.00-8.00 sec 948 MBytes 3.97 Gbits/sec 120 360 KBytes [ 5] 8.00-10.00 sec 927 MBytes 3.89 Gbits/sec 149 242 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.35 GBytes 3.74 Gbits/sec 693 sender [ 5] 0.00-10.00 sec 4.35 GBytes 3.74 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 5208 dev.ix.2.queue6.interrupt_rate: 31250 dev.ix.2.queue5.interrupt_rate: 31250 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 100000 dev.ix.2.queue2.interrupt_rate: 71428 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 31250
- with enable_aim=1
root@s1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@r1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 43844 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 777 MBytes 3.26 Gbits/sec 511 404 KBytes [ 5] 2.00-4.00 sec 944 MBytes 3.96 Gbits/sec 145 427 KBytes [ 5] 4.00-6.00 sec 907 MBytes 3.80 Gbits/sec 149 295 KBytes [ 5] 6.00-8.00 sec 934 MBytes 3.92 Gbits/sec 93 397 KBytes [ 5] 8.00-10.00 sec 960 MBytes 4.02 Gbits/sec 152 346 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.42 GBytes 3.79 Gbits/sec 1050 sender [ 5] 0.00-10.00 sec 4.42 GBytes 3.79 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 5208 dev.ix.2.queue6.interrupt_rate: 31250 dev.ix.2.queue5.interrupt_rate: 83333 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 7936 dev.ix.2.queue2.interrupt_rate: 71428 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 7936
- with enable_aim=2
root@s1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@r1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 13194 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 713 MBytes 2.99 Gbits/sec 512 360 KBytes [ 5] 2.00-4.00 sec 933 MBytes 3.92 Gbits/sec 85 483 KBytes [ 5] 4.00-6.00 sec 949 MBytes 3.98 Gbits/sec 157 337 KBytes [ 5] 6.00-8.00 sec 953 MBytes 4.00 Gbits/sec 160 437 KBytes [ 5] 8.00-10.00 sec 956 MBytes 4.01 Gbits/sec 151 60.8 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.40 GBytes 3.78 Gbits/sec 1065 sender [ 5] 0.00-10.00 sec 4.40 GBytes 3.78 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 5208 dev.ix.2.queue6.interrupt_rate: 31250 dev.ix.2.queue5.interrupt_rate: 2702 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 100000 dev.ix.2.queue2.interrupt_rate: 71428 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 125000
* test iperf3 performance on a stock kernel without the patch D30155
(1) root@s1:~ # sysctl dev.ix.2.enable_aim dev.ix.2.enable_aim: 0 root@r1:~ # sysctl dev.ix.2.enable_aim dev.ix.2.enable_aim: 0 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 58877 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 782 MBytes 3.28 Gbits/sec 301 349 KBytes [ 5] 2.00-4.00 sec 873 MBytes 3.66 Gbits/sec 178 218 KBytes [ 5] 4.00-6.00 sec 890 MBytes 3.73 Gbits/sec 122 265 KBytes [ 5] 6.00-8.00 sec 883 MBytes 3.70 Gbits/sec 111 356 KBytes [ 5] 8.00-10.00 sec 908 MBytes 3.81 Gbits/sec 108 256 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.23 GBytes 3.64 Gbits/sec 820 sender [ 5] 0.00-10.00 sec 4.23 GBytes 3.64 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 31250 dev.ix.2.queue6.interrupt_rate: 31250 dev.ix.2.queue5.interrupt_rate: 31250 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 31250 dev.ix.2.queue2.interrupt_rate: 31250 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 31250 (2) root@s1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@r1:~ # sysctl dev.ix.2.enable_aim=1 dev.ix.2.enable_aim: 0 -> 1 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 26259 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 677 MBytes 2.84 Gbits/sec 234 301 KBytes [ 5] 2.00-4.00 sec 889 MBytes 3.73 Gbits/sec 153 399 KBytes [ 5] 4.00-6.00 sec 895 MBytes 3.75 Gbits/sec 156 342 KBytes [ 5] 6.00-8.00 sec 894 MBytes 3.75 Gbits/sec 195 296 KBytes [ 5] 8.00-10.00 sec 897 MBytes 3.76 Gbits/sec 177 375 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.15 GBytes 3.57 Gbits/sec 915 sender [ 5] 0.00-10.00 sec 4.15 GBytes 3.56 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 31250 dev.ix.2.queue6.interrupt_rate: 100000 dev.ix.2.queue5.interrupt_rate: 31250 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 31250 dev.ix.2.queue2.interrupt_rate: 100000 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 31250 root@s1:~ # (3) root@s1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@r1:~ # sysctl dev.ix.2.enable_aim=2 dev.ix.2.enable_aim: 1 -> 2 root@s1:~ # iperf3 -c r1 -t 10 -i 2 Connecting to host r1, port 5201 [ 5] local 10.1.1.2 port 41892 connected to 10.1.1.3 port 5201 [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-2.00 sec 694 MBytes 2.91 Gbits/sec 352 294 KBytes [ 5] 2.00-4.00 sec 878 MBytes 3.68 Gbits/sec 175 402 KBytes [ 5] 4.00-6.00 sec 860 MBytes 3.61 Gbits/sec 144 378 KBytes [ 5] 6.00-8.00 sec 891 MBytes 3.74 Gbits/sec 140 326 KBytes [ 5] 8.00-10.00 sec 901 MBytes 3.78 Gbits/sec 131 461 KBytes - - - - - - - - - - - - - - - - - - - - - - - - - [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 4.12 GBytes 3.54 Gbits/sec 942 sender [ 5] 0.00-10.00 sec 4.12 GBytes 3.54 Gbits/sec receiver iperf Done. root@s1:~ # sysctl dev.ix.2 | grep _rate // interrupt rate during iperf3 bulk transfer dev.ix.2.queue7.interrupt_rate: 31250 dev.ix.2.queue6.interrupt_rate: 100000 dev.ix.2.queue5.interrupt_rate: 100000 dev.ix.2.queue4.interrupt_rate: 31250 dev.ix.2.queue3.interrupt_rate: 100000 dev.ix.2.queue2.interrupt_rate: 100000 dev.ix.2.queue1.interrupt_rate: 31250 dev.ix.2.queue0.interrupt_rate: 71428
* test iperf3 on a stock Ubuntu 22.04 kernel 5.15.0-112-generic
root@s1:~# iperf3 -c r1 -t 10 -i 2 -V
iperf 3.9
Linux s1.testd30155.fbsd-transport.emulab.net 5.15.0-112-generic #122-Ubuntu SMP Thu May 23 07:48:21 UTC 2024 x86_64
Control connection MSS 1448
Time: Thu, 17 Oct 2024 13:56:48 GMT
Connecting to host r1, port 5201
Cookie: e2zxxflfogelbora5gobrz7hxxvacaojutqv
TCP MSS: 1448 (default)
[ 5] local 10.1.1.2 port 48078 connected to 10.1.1.3 port 5201
Starting Test: protocol: TCP, 1 streams, 131072 byte blocks, omitting 0 seconds, 10 second test, tos 0
[ ID] Interval Transfer Bitrate Retr Cwnd
[ 5] 0.00-2.00 sec 2.19 GBytes 9.42 Gbits/sec 17 1.38 MBytes
[ 5] 2.00-4.00 sec 2.19 GBytes 9.41 Gbits/sec 46 1.40 MBytes
[ 5] 4.00-6.00 sec 2.19 GBytes 9.42 Gbits/sec 0 1.43 MBytes
[ 5] 6.00-8.00 sec 2.19 GBytes 9.41 Gbits/sec 3 1.43 MBytes
[ 5] 8.00-10.00 sec 2.19 GBytes 9.41 Gbits/sec 0 1.44 MBytes
- - - - - - - - - - - - - - - - - - - - - - - - -
Test Complete. Summary Results:
[ ID] Interval Transfer Bitrate Retr
[ 5] 0.00-10.00 sec 11.0 GBytes 9.42 Gbits/sec 66 sender
[ 5] 0.00-10.04 sec 11.0 GBytes 9.38 Gbits/sec receiver
CPU Utilization: local/sender 54.6% (1.7%u/53.0%s), remote/receiver 23.6% (2.2%u/21.4%s)
snd_tcp_congestion cubic
rcv_tcp_congestion cubic
iperf Done.
root@s1:~#