reload sysctl

$ sudo sysctl -p /etc/sysctl.conf

tcp 优化

此配置来自42.3, 注意其中已经修改tcp的timeWait参数

# diff ~/sysctl.conf.bak etc/sysctl.conf 
35a36,48
> net.ipv4.conf.all.accept_redirects = 0
> 
> net.ipv4.ip_default_ttl = 128
> net.ipv4.icmp_echo_ignore_broadcasts = 1
> net.ipv4.tcp_synack_retries = 3
> net.ipv4.tcp_syn_retries = 3
> net.ipv4.tcp_max_syn_backlog = 2048
> net.ipv4.tcp_fin_timeout = 30
> net.ipv4.tcp_keepalive_time = 1800
> net.ipv4.tcp_keepalive_probes = 5
> net.ipv4.tcp_keepalive_intvl = 60
> net.ipv4.tcp_tw_reuse = 0
> net.ipv4.tcp_max_tw_buckets = 60000

for benchmarks

cp /etc/security/limits.conf{,.orig}
cat >> /etc/security/limits.conf << EOF
* soft nofile 200000
* hard nofile 200000
EOF

cp /etc/sysctl.conf /etc/sysctl.conf.orig
cat >> /etc/sysctl.conf << EOF
# "Performance Scalability of a Multi-Core Web Server", Nov 2007
# Bryan Veal and Annie Foong, Intel Corporation, Page 4/10
fs.file-max = 5000000
net.core.netdev_max_backlog = 400000
net.core.optmem_max = 10000000
net.core.rmem_default = 10000000
net.core.rmem_max = 10000000
net.core.somaxconn = 100000
net.core.wmem_default = 10000000
net.core.wmem_max = 10000000
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_congestion_control = bic
net.ipv4.tcp_ecn = 0
net.ipv4.tcp_max_syn_backlog = 12000
net.ipv4.tcp_max_tw_buckets = 2000000
net.ipv4.tcp_mem = 30000000 30000000 30000000
net.ipv4.tcp_rmem = 30000000 30000000 30000000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_wmem = 30000000 30000000 30000000    

# optionally, avoid TIME_WAIT states on localhost no-HTTP Keep-Alive tests:
#    "error: connect() failed: Cannot assign requested address (99)"
# On Linux, the 2MSL time is hardcoded to 60 seconds in /include/net/tcp.h:
# #define TCP_TIMEWAIT_LEN (60*HZ)
# The option below is safe to use:
net.ipv4.tcp_tw_reuse = 1

# The option below lets you reduce TIME_WAITs further
# but this option is for benchmarks, NOT for production (NAT issues)
net.ipv4.tcp_tw_recycle = 1
EOF

今天有建议说调整我fin2的情况，目前如下：

$ sudo /usr/sbin/ss -s
Total: 3154 (kernel 3796)
TCP:   60554 (estab 1838, closed 57603, orphaned 14, synrecv 0, timewait 57602/0), ports 758

Transport Total     IP        IPv6
*     3796      -         -        
RAW   0         0         0        
UDP   21        16        5        
TCP   2951      2642      309      
INET      2972      2658      314      
FRAG      0         0         0     

$ /sbin/sysctl -a | grep net.ipv4.tcp_fin_timeout
net.ipv4.tcp_fin_timeout = 30

$ perl -MData::Dumper -nae '$h{$F[0]}++}{ print Dumper \%h'  ss.out 
$VAR1 = {
          'SYN-SENT' => 10,
          'ESTAB' => 1710,
          'State' => 1,
          'FIN-WAIT-1' => 19,
          'FIN-WAIT-2' => 1441,
          'TIME-WAIT' => 59045,
          'SYN-RECV' => 448,
          'LISTEN' => 23
        };

有建议修改以下项, 参考这里修改

net.ipv4.tcp_fin_timeout = 3
net.ipv4.tcp_keepalive_time = 10
net.ipv4.ip_local_port_range = 1024  65535
net.ipv4.tcp_tw_reuse = 1
net.ipv4.conf.default.rp_filter = 0
net.ipv4.tcp_max_tw_buckets = 100

改善前，TW很大。

$ sudo /usr/sbin/ss -nat | perl -MData::Dumper -nae '$h{$F[0]}++}{ print Dumper \%h'
$VAR1 = {
          'SYN-SENT' => 8,
          'ESTAB' => 2373,
          'State' => 1,
          'FIN-WAIT-1' => 10,
          'LAST-ACK' => 1,
          'FIN-WAIT-2' => 1207,
          'TIME-WAIT' => 59157,
          'SYN-RECV' => 402,
          'LISTEN' => 24
        };``

修改后，改善了fin的值和timewait的值

$ sudo /usr/sbin/ss -nat | perl -MData::Dumper -nae ‘$h{$F[0]}++}{ print Dumper \%h’ $VAR1 = { ‘CLOSING’ => 1, ‘SYN-SENT’ => 7, ‘ESTAB’ => 1463, ‘State’ => 1, ‘FIN-WAIT-1’ => 45, ‘FIN-WAIT-2’ => 895, ‘TIME-WAIT’ => 103, ‘SYN-RECV’ => 386, ‘LISTEN’ => 24 };


但会出现dmesg报错

$ sudo dmesg -c printk: 17661 messages suppressed. TCP: time wait bucket table overflow printk: 17931 messages suppressed. TCP: time wait bucket table overflow


### 一般优化

+ pns `/etc/security/limits.conf`
  • soft core 0
  • hard core 0
  • soft nofile 665535
  • hard nofile 665535
  • soft nproc 65535

  • hard nproc 65535 ```

  • mmsdk /etc/sysctl.conf
net.ipv4.ip_default_ttl = 128
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_syn_retries = 3
net.ipv4.tcp_max_syn_backlog = 2048
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 1800
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 60
net.ipv4.tcp_tw_reuse = 0
net.ipv4.tcp_max_tw_buckets = 60000
  • pns /etc/sysctl.conf
net.ipv4.ip_default_ttl = 128
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_syn_retries = 3
net.ipv4.tcp_max_syn_backlog = 2048
#net.ipv4.netfilter.ip_conntrack_tcp_timeout_established = 1800
#net.ipv4.ip_conntrack_max  =  131072
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 1800
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 60

###pvdriver<begin>
net.ipv4.conf.all.arp_notify = 1
net.ipv4.conf.default.arp_notify = 1
net.ipv4.conf.eth0.arp_notify = 1
net.ipv4.conf.lo.arp_notify = 1
###pvdriver<end>

net.ipv4.conf.eth0.arp_ignore = 1
net.ipv4.conf.eth0.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.core.somaxconn = 4096