BU_Tier2 < Admins

r4 - 30 Oct 2007 - 15:37:47 - DantongYuYou are here: TWiki >

NETier2_BU

Tuning on October 30 2007

Tracepath to BNL:

[root@atlas-g01 ~]$ tracepath dct00.usatlas.bnl.gov
 1:  atlas-g01.bu.edu (192.5.207.95)                        0.114ms pmtu 1500
 1:  atlas-net-gw.BU.EDU (192.5.207.1)                      1.422ms
 2:  nox300gw1-Vl-597-NoX-BU.nox.org (192.5.89.141)         0.539ms
 3:  nox300gw1-PEER-NoX-INTERNET2-192-5-89-222.nox.org (192.5.89.222) asymm  4 246.390ms
 4:  aoa-manlan-abilene.es.net (198.124.216.157)          asymm  5 5.225ms
 5:  bnlmr1-aoacr1.es.net (134.55.217.57)                   8.023ms
 6:  bnlsite-bnlmr1.es.net (198.124.216.178)                7.585ms
 7:  dct00.usatlas.bnl.gov (192.12.15.8)                    7.589ms reached
     Resume: pmtu 1500 hops 7 back 7
[root@atlas-g01 ~]$ 


[root@atlas-g01 ~]$ ifconfig eth1
eth1      Link encap:Ethernet  HWaddr 00:1A:64:03:0E:44
          inet addr:192.5.207.95  Bcast:192.5.207.255  Mask:255.255.255.0
          inet6 addr: fe80::21a:64ff:fe03:e44/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:10262 errors:0 dropped:0 overruns:0 frame:0
          TX packets:5728 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:1032233 (1008.0 KiB)  TX bytes:635089 (620.2 KiB)
          Interrupt:185 Memory:e4000000-e4011100 


--------------------------------------------------------------------------------

[root@atlas-g01 ~]$ ethtool eth1
Settings for eth1:
        Supported ports: [ FIBRE ]
        Supported link modes:   1000baseT/Full
        Supports auto-negotiation: Yes
        Advertised link modes:  1000baseT/Full
        Advertised auto-negotiation: Yes
        Speed: 1000Mb/s
        Duplex: Full
        Port: FIBRE
        PHYAD: 1
        Transceiver: internal
        Auto-negotiation: on
        Supports Wake-on: d
        Wake-on: d
        Link detected: yes
-----------------------------------------------------------------------------------

[root@atlas-g01 ~]$ iperf -c dct00.usatlas.bnl.gov -w4M -i2 -t60
------------------------------------------------------------
Client connecting to dct00.usatlas.bnl.gov, TCP port 5001
TCP window size:   264 KByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.95 port 32995 connected with 192.12.15.8 port 5001
[  3]  0.0- 2.0 sec  31.8 MBytes    133 Mbits/sec
[  3]  2.0- 4.0 sec  25.6 MBytes    108 Mbits/sec
[  3]  4.0- 6.0 sec  18.6 MBytes  78.2 Mbits/sec
[  3]  6.0- 8.0 sec  24.7 MBytes    104 Mbits/sec
[  3]  8.0-10.0 sec  14.7 MBytes  61.5 Mbits/sec
[  3] 10.0-12.0 sec  21.4 MBytes  89.7 Mbits/sec
[  3] 12.0-14.0 sec  30.4 MBytes    127 Mbits/sec
[  3] 14.0-16.0 sec  26.3 MBytes    110 Mbits/sec
[  3] 16.0-18.0 sec  32.4 MBytes    136 Mbits/sec
[  3] 18.0-20.0 sec  27.9 MBytes    117 Mbits/sec
[  3] 20.0-22.0 sec  25.9 MBytes    109 Mbits/sec
[  3] 22.0-24.0 sec  22.0 MBytes  92.1 Mbits/sec
[  3] 24.0-26.0 sec  20.1 MBytes  84.2 Mbits/sec
[  3] 26.0-28.0 sec  13.9 MBytes  58.4 Mbits/sec
[  3] 28.0-30.0 sec  22.2 MBytes  93.2 Mbits/sec
[  3] 30.0-32.0 sec  30.6 MBytes    128 Mbits/sec
[  3] 32.0-34.0 sec  25.7 MBytes    108 Mbits/sec
[  3] 34.0-36.0 sec  30.2 MBytes    127 Mbits/sec
[  3] 36.0-38.0 sec  29.3 MBytes    123 Mbits/sec
[  3] 38.0-40.0 sec  23.6 MBytes  98.8 Mbits/sec
[  3] 40.0-42.0 sec  25.7 MBytes    108 Mbits/sec
[  3] 42.0-44.0 sec  25.5 MBytes    107 Mbits/sec
[  3] 44.0-46.0 sec  26.5 MBytes    111 Mbits/sec
[  3] 46.0-48.0 sec  31.5 MBytes    132 Mbits/sec
[  3] 48.0-50.0 sec  25.1 MBytes    105 Mbits/sec
[  3] 50.0-52.0 sec  20.4 MBytes  85.6 Mbits/sec
[  3] 52.0-54.0 sec  21.1 MBytes  88.5 Mbits/sec
[  3] 54.0-56.0 sec  22.9 MBytes  96.0 Mbits/sec
[  3] 56.0-58.0 sec  26.2 MBytes    110 Mbits/sec
[  3] 58.0-60.0 sec  30.3 MBytes    127 Mbits/sec
[  3]  0.0-60.0 sec    753 MBytes    105 Mbits/sec
[root@atlas-g01 ~]$
---------------------------------------------------------------------------------------

[augustin@atlas ~]$ iperf -c dct00.usatlas.bnl.gov -w4M -i2 -t60
------------------------------------------------------------
Client connecting to dct00.usatlas.bnl.gov, TCP port 5001
TCP window size:   256 KByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.10 port 47685 connected with 192.12.15.8 port 5001
[  3]  0.0- 2.0 sec  15.7 MBytes  66.0 Mbits/sec
[  3]  2.0- 4.0 sec  9.96 MBytes  41.8 Mbits/sec
[  3]  4.0- 6.0 sec  16.5 MBytes  69.2 Mbits/sec
[  3]  6.0- 8.0 sec  16.9 MBytes  71.0 Mbits/sec
[  3]  8.0-10.0 sec  14.9 MBytes  62.6 Mbits/sec
[  3] 10.0-12.0 sec  15.2 MBytes  63.9 Mbits/sec
[  3] 12.0-14.0 sec  13.0 MBytes  54.5 Mbits/sec
[  3] 14.0-16.0 sec  8.79 MBytes  36.9 Mbits/sec
[  3] 16.0-18.0 sec  16.0 MBytes  67.2 Mbits/sec
[  3] 18.0-20.0 sec  16.7 MBytes  70.1 Mbits/sec
[  3] 20.0-22.0 sec  19.8 MBytes  83.2 Mbits/sec
[  3] 22.0-24.0 sec  27.4 MBytes    115 Mbits/sec
[  3] 24.0-26.0 sec  22.3 MBytes  93.4 Mbits/sec
[  3] 26.0-28.0 sec  11.4 MBytes  47.9 Mbits/sec
[  3] 28.0-30.0 sec  20.0 MBytes  84.0 Mbits/sec
[  3] 30.0-32.0 sec  18.5 MBytes  77.8 Mbits/sec
[  3] 32.0-34.0 sec  25.6 MBytes    107 Mbits/sec
[  3] 34.0-36.0 sec  15.2 MBytes  63.6 Mbits/sec
[  3] 36.0-38.0 sec  19.1 MBytes  80.3 Mbits/sec
[  3] 38.0-40.0 sec  18.4 MBytes  77.2 Mbits/sec
[  3] 40.0-42.0 sec  15.9 MBytes  66.9 Mbits/sec
[  3] 42.0-44.0 sec  17.1 MBytes  71.6 Mbits/sec
[  3] 44.0-46.0 sec  13.5 MBytes  56.6 Mbits/sec
[  3] 46.0-48.0 sec  16.7 MBytes  70.0 Mbits/sec
[  3] 48.0-50.0 sec  13.8 MBytes  58.0 Mbits/sec
[  3] 50.0-52.0 sec  14.2 MBytes  59.5 Mbits/sec
[  3] 52.0-54.0 sec  11.9 MBytes  49.7 Mbits/sec
[  3] 54.0-56.0 sec  17.3 MBytes  72.6 Mbits/sec
[  3] 56.0-58.0 sec  14.9 MBytes  62.5 Mbits/sec
[  3] 58.0-60.0 sec  16.6 MBytes  69.7 Mbits/sec
[  3]  0.0-60.0 sec    494 MBytes  69.0 Mbits/sec
--------------------------------------------------------------------------------------

[root@atlas-g01 ~]$ diff /tmp/after /tmp/before
2c2
<      rx_bytes: 20375837
---
>      rx_bytes: 12772155
4c4
<      tx_bytes: 1347969676
---
>      tx_bytes: 828216195
6,9c6,9
<      rx_ucast_packets: 274023
<      rx_mcast_packets: 170
<      rx_bcast_packets: 7275
<      tx_ucast_packets: 895239
---
>      rx_ucast_packets: 167232
>      rx_mcast_packets: 169
>      rx_bcast_packets: 7218
>      tx_ucast_packets: 552514
26,27c26,27
<      rx_64_byte_packets: 7132
<      rx_65_to_127_byte_packets: 273475
---
>      rx_64_byte_packets: 7074
>      rx_65_to_127_byte_packets: 166685
29c29
<      rx_256_to_511_byte_packets: 217
---
>      rx_256_to_511_byte_packets: 216
34,38c34,38
<      tx_65_to_127_byte_packets: 4573
<      tx_128_to_255_byte_packets: 1469
<      tx_256_to_511_byte_packets: 1065
<      tx_512_to_1023_byte_packets: 863
<      tx_1024_to_1522_byte_packets: 886998
---
>      tx_65_to_127_byte_packets: 4526
>      tx_128_to_255_byte_packets: 1415
>      tx_256_to_511_byte_packets: 893
>      tx_512_to_1023_byte_packets: 692
>      tx_1024_to_1522_byte_packets: 544717
45c45
<      rx_filtered_packets: 415
---
>      rx_filtered_packets: 412
[root@atlas-g01 ~]$
-------------------------------------------------------------------------------------
[root@atlas-g01 ~]$ sysctl -a |grep tcp
net.ipv4.tcp_rmem = 4096        87380   174760
net.ipv4.tcp_wmem = 4096        16384   131072
net.ipv4.tcp_mem = 786432       1048576 1572864
[root@atlas-g01 ~]$

root@atlas-g01 ~]$ sysctl -a | grep mem net.ipv4.tcp_rmem = 4096 87380 20000000 net.ipv4.tcp_wmem = 4096 87380 20000000 net.ipv4.tcp_mem = 786432 1048576 1572864 net.ipv4.igmp_max_memberships = 20 net.core.optmem_max = 20480 net.core.rmem_default = 135168 net.core.wmem_default = 135168 net.core.rmem_max = 20000000 net.core.wmem_max = 20000000 vm.overcommit_memory = 0

[root@atlas-g01 ~]$ iperf -c dct00.usatlas.bnl.gov -w4M -i2 -t60
------------------------------------------------------------
Client connecting to dct00.usatlas.bnl.gov, TCP port 5001
TCP window size: 8.00 MByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.95 port 33071 connected with 192.12.15.8 port 5001
[  3]  0.0- 2.0 sec  13.2 MBytes  55.2 Mbits/sec
[  3]  2.0- 4.0 sec  5.65 MBytes  23.7 Mbits/sec
[  3]  4.0- 6.0 sec  3.75 MBytes  15.7 Mbits/sec
[  3]  6.0- 8.0 sec  7.52 MBytes  31.6 Mbits/sec
[  3]  8.0-10.0 sec  5.66 MBytes  23.7 Mbits/sec
[  3] 10.0-12.0 sec  7.55 MBytes  31.7 Mbits/sec
[  3] 12.0-14.0 sec  7.55 MBytes  31.7 Mbits/sec
[  3] 14.0-16.0 sec  7.60 MBytes  31.9 Mbits/sec
[  3] 16.0-18.0 sec  7.54 MBytes  31.6 Mbits/sec
[  3] 18.0-20.0 sec  7.53 MBytes  31.6 Mbits/sec
[  3] 20.0-22.0 sec  5.66 MBytes  23.7 Mbits/sec
[  3] 22.0-24.0 sec  9.52 MBytes  39.9 Mbits/sec
[  3] 24.0-26.0 sec  5.67 MBytes  23.8 Mbits/sec
[  3] 26.0-28.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 28.0-30.0 sec  5.65 MBytes  23.7 Mbits/sec
[  3] 30.0-32.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 32.0-34.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 34.0-36.0 sec  7.59 MBytes  31.8 Mbits/sec
[  3] 36.0-38.0 sec  5.66 MBytes  23.8 Mbits/sec
[  3] 38.0-40.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 40.0-42.0 sec  5.66 MBytes  23.7 Mbits/sec
[  3] 42.0-44.0 sec  5.65 MBytes  23.7 Mbits/sec
[  3] 44.0-46.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 46.0-48.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 48.0-50.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 50.0-52.0 sec  7.51 MBytes  31.5 Mbits/sec
[  3] 52.0-54.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 54.0-56.0 sec  5.64 MBytes  23.7 Mbits/sec
[  3] 56.0-58.0 sec  11.4 MBytes  47.7 Mbits/sec
[  3] 58.0-60.0 sec  9.41 MBytes  39.5 Mbits/sec
[  3]  0.0-60.2 sec    204 MBytes  28.4 Mbits/sec


[root@atlas-g01 ~]$ iperf -s -i5 -w4M
------------------------------------------------------------
Server listening on TCP port 5001
TCP window size: 8.00 MByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  4] local 192.5.207.95 port 5001 connected with 192.12.15.8 port 57665
[  4]  0.0- 5.0 sec    555 MBytes    932 Mbits/sec
[  4]  5.0-10.0 sec    561 MBytes    941 Mbits/sec
[  4] 10.0-15.0 sec    561 MBytes    941 Mbits/sec
[  4] 15.0-20.0 sec    561 MBytes    941 Mbits/sec
[  4] 20.0-25.0 sec    561 MBytes    941 Mbits/sec
[  4] 25.0-30.0 sec    516 MBytes    866 Mbits/sec
[  4] 30.0-35.0 sec    561 MBytes    941 Mbits/sec
[  4] 35.0-40.0 sec    561 MBytes    941 Mbits/sec
[  4] 40.0-45.0 sec    561 MBytes    941 Mbits/sec
[  4] 45.0-50.0 sec    561 MBytes    941 Mbits/sec
[  4] 50.0-55.0 sec    520 MBytes    873 Mbits/sec
[  4] 55.0-60.0 sec    561 MBytes    941 Mbits/sec
[  4]  0.0-60.0 sec  6.49 GBytes    929 Mbits/sec

[root@atlas-g01 ~]$ iperf -c dct00.usatlas.bnl.gov -w4M -i5 -t60 -b 950M -u
WARNING: option -b implies udp testing
------------------------------------------------------------
Client connecting to dct00.usatlas.bnl.gov, UDP port 5001
Sending 1470 byte datagrams
UDP buffer size: 8.00 MByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.95 port 32775 connected with 192.12.15.8 port 5001
[  3]  0.0- 5.0 sec    584 MBytes    979 Mbits/sec
[  3]  5.0-10.0 sec    584 MBytes    979 Mbits/sec
[  3] 10.0-15.0 sec    584 MBytes    979 Mbits/sec
[  3] 15.0-20.0 sec    584 MBytes    979 Mbits/sec
[  3] 20.0-25.0 sec    584 MBytes    980 Mbits/sec
[  3] 25.0-30.0 sec    584 MBytes    980 Mbits/sec
[  3] 30.0-35.0 sec    584 MBytes    979 Mbits/sec
[  3] 35.0-40.0 sec    584 MBytes    979 Mbits/sec
[  3] 40.0-45.0 sec    584 MBytes    980 Mbits/sec
[  3] 45.0-50.0 sec    584 MBytes    980 Mbits/sec
[  3] 50.0-55.0 sec    584 MBytes    979 Mbits/sec
[  3] 55.0-60.0 sec    584 MBytes    979 Mbits/sec
[  3]  0.0-60.0 sec  6.84 GBytes    979 Mbits/sec
[  3] Sent 4996905 datagrams
[  3] Server Report:
[  3]  0.0-60.0 sec  6.65 GBytes    952 Mbits/sec  0.018 ms 139219/4996904 (2.8%)
[  3]  0.0-60.0 sec  2 datagrams received out-of-order

[root@atlas-g01 ~]$ iperf -c dct00.usatlas.bnl.gov -w4M -i5 -t60 -b 500M -u
WARNING: option -b implies udp testing
------------------------------------------------------------
Client connecting to dct00.usatlas.bnl.gov, UDP port 5001
Sending 1470 byte datagrams
UDP buffer size: 8.00 MByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.95 port 32776 connected with 192.12.15.8 port 5001
[  3]  0.0- 5.0 sec    305 MBytes    511 Mbits/sec
[  3]  5.0-10.0 sec    305 MBytes    511 Mbits/sec
[  3] 10.0-15.0 sec    305 MBytes    511 Mbits/sec
[  3] 15.0-20.0 sec    305 MBytes    511 Mbits/sec
[  3] 20.0-25.0 sec    305 MBytes    511 Mbits/sec
[  3] 25.0-30.0 sec    305 MBytes    511 Mbits/sec
[  3] 30.0-35.0 sec    305 MBytes    511 Mbits/sec
[  3] 35.0-40.0 sec    305 MBytes    511 Mbits/sec
[  3] 40.0-45.0 sec    305 MBytes    511 Mbits/sec
[  3] 45.0-50.0 sec    305 MBytes    511 Mbits/sec
[  3] 50.0-55.0 sec    305 MBytes    511 Mbits/sec
[  3] 55.0-60.0 sec    305 MBytes    511 Mbits/sec
[  3]  0.0-60.0 sec  3.57 GBytes    511 Mbits/sec
[  3] Sent 2608696 datagrams
[  3] Server Report:
[  3]  0.0-60.0 sec  3.55 GBytes    508 Mbits/sec  0.031 ms 16036/2608695 (0.61%)
[  3]  0.0-60.0 sec  2 datagrams received out-of-order


[root@atlas-g01 ~]$ iperf -s -i5 -w4M -u
------------------------------------------------------------
Server listening on UDP port 5001
Receiving 1470 byte datagrams
UDP buffer size: 8.00 MByte (WARNING: requested 4.00 MByte)
------------------------------------------------------------
[  3] local 192.5.207.95 port 5001 connected with 192.12.15.8 port 41169
[  3]  0.0- 5.0 sec    305 MBytes    511 Mbits/sec  0.003 ms    0/217397 (0%)
[  3]  5.0-10.0 sec    305 MBytes    511 Mbits/sec  0.010 ms    0/217391 (0%)
[  3] 10.0-15.0 sec    305 MBytes    511 Mbits/sec  0.003 ms    0/217391 (0%)
[  3] 15.0-20.0 sec    305 MBytes    511 Mbits/sec  0.010 ms    0/217391 (0%)
[  3] 20.0-25.0 sec    305 MBytes    511 Mbits/sec  0.004 ms    0/217391 (0%)
[  3] 25.0-30.0 sec    305 MBytes    511 Mbits/sec  0.009 ms    0/217391 (0%)
[  3] 30.0-35.0 sec    305 MBytes    511 Mbits/sec  0.010 ms    0/217392 (0%)
[  3] 35.0-40.0 sec    305 MBytes    511 Mbits/sec  0.007 ms    0/217391 (0%)
[  3] 40.0-45.0 sec    305 MBytes    511 Mbits/sec  0.008 ms    0/217391 (0%)
[  3] 45.0-50.0 sec    305 MBytes    511 Mbits/sec  0.007 ms    0/217391 (0%)
[  3] 50.0-55.0 sec    305 MBytes    511 Mbits/sec  0.011 ms    0/217391 (0%)
[  3]  0.0-60.0 sec  3.57 GBytes    511 Mbits/sec  0.010 ms    0/2608695 (0%)
[  3]  0.0-60.0 sec  1 datagrams received out-of-order
[  4] local 192.5.207.95 port 5001 connected with 192.12.15.8 port 41169
[  4]  0.0- 5.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406886 (0%)
[  4]  5.0-10.0 sec    570 MBytes    957 Mbits/sec  0.032 ms    0/406899 (0%)



[  4] 10.0-15.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406892 (0%)
[  4] 15.0-20.0 sec    570 MBytes    957 Mbits/sec  0.010 ms    0/406900 (0%)
[  4] 20.0-25.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406894 (0%)
[  4] 25.0-30.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406899 (0%)
[  4] 30.0-35.0 sec    570 MBytes    957 Mbits/sec  0.016 ms    2/406899 (0.00049%)
[  4] 30.0-35.0 sec  2 datagrams received out-of-order
[  4] 35.0-40.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406897 (0%)
[  4] 40.0-45.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    0/406895 (0%)
[  4] 45.0-50.0 sec    570 MBytes    957 Mbits/sec  0.016 ms    0/406897 (0%)
[  4] 50.0-55.0 sec    570 MBytes    957 Mbits/sec  0.015 ms   10/406905 (0.0025%)
[  4] 55.0-60.0 sec    570 MBytes    957 Mbits/sec  0.015 ms    5/406903 (0.0012%)
[  4]  0.0-60.0 sec  6.69 GBytes    957 Mbits/sec  0.035 ms   14/4882983 (0.00029%)
[  4]  0.0-60.0 sec  3 datagrams received out-of-order

-- DantongYu - 30 Oct 2007

About This Site

Please note that this site is a content mirror of the BNL US ATLAS TWiki. To edit the content of this page, click the Edit this page button at the top of the page and log in with your US ATLAS computing account name and password.

Attachments

Admins

USATLAS Site Admin Topics
Admin Support Home
Site Setup Help
Search

Facility Integration Program
Integration home
Proof-Xrootd

Site Admin External Links
dCache
GUMS

Copyright © by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki? Send feedback