With the following settings - e.g. by optimizing and simplifying pf.conf
rules and relayd.conf we were able to push 24400 req/s through with HTTPS.
:) Maybe this helps someone else.
#############################################################################
###
# OpenBSD sysctl.conf
net.inet.carp.preempt=1
kern.bufcachepercent=90
kern.maxfiles=200000
kern.maxproc=50000
kern.maxclusters=32768
net.inet.ip.forwarding=1
net.inet6.ip6.forwarding=1
net.inet.ip.ifq.maxlen=8192
net.inet.ip.mtudisc=0
net.inet.tcp.rfc3390=1
net.inet.tcp.mssdflt=1440
#############################################################################
###
# OpenBSD relayd.conf
ip4_244 = "xx.xx.xx.244"
ip4_245 = "xx.xx.xx.245"
tracker5 = "10.5.3.34"
tracker6 = "10.5.3.42"
tracker7 = "10.5.3.50"
interval 10
table <trackers> { $tracker5, $tracker6, $tracker7 }
prefork 12
http protocol https {
### TCP performance options
tcp { nodelay, sack, socket buffer 65536, backlog 128 }
match request header append "X-Forwarded-For" value "$REMOTE_ADDR"
match request header append "X-Forwarded-By" \
value "$SERVER_ADDR:$SERVER_PORT"
match header set "Keep-Alive" value "$TIMEOUT"
pass
tls { no tlsv1.0, ciphers "HIGH:!aNULL" }
tls session cache disable
}
relay wwwssl {
listen on $ip4_244 port 443 tls
listen on $ip4_245 port 443 tls
protocol "https"
forward to <trackers> port 8083 mode roundrobin check tcp
session timeout 60
}
relay www {
listen on $ip4_244 port 80
listen on $ip4_245 port 80
forward to <trackers> port 8083 mode roundrobin check tcp
}
#############################################################################
###
# OpenBSD: pf.conf
tcp_services = "{ domain }"
udp_services = "{ domain }"
tcp_public_services = "{ www, https }"
pfsync_int = trunk2 # Pfsync interface
int_if = trunk1 # DMZ (internal) interface
ext_if = trunk0 # External CARP interface
# Increase limits
set limit { states 25000, src-nodes 25000, table-entries 300000 }
# Aggressive settings
set optimization aggressive
set timeout { adaptive.end 120000, interval 2, tcp.tsdiff 5, tcp.first 5,
tcp.closing 5, tcp.closed 5, tcp.finwait 5, tcp.established 4200}
# See pf.conf(5) and /etc/examples/pf.conf
anchor "relayd/*"
set block-policy drop
set loginterface $ext_if
set skip on lo
set skip on $int_if
set skip on $pfsync_int
match in all scrub (no-df max-mss 1440)
# Block everything by default
block all
# Allow main service of this host
pass quick proto tcp to port $tcp_public_services keep state
pass out quick proto tcp to port $tcp_services keep state
pass proto udp to port $udp_services keep state
# Pass CARP
pass quick proto carp keep state (no-sync)
# SSH backup channel from Wooga office
pass in on trunk0 inet proto tcp from xx.xx.xx.xx/xx to any port 22 keep
state (no-sync)
# Allow pings for Pingdom status checks
pass on trunk0 inet proto icmp keep state (no-sync)
pass on trunk0 inet6 proto icmp6 keep state (no-sync)
On Tue, Mar 15, 2016 at 11:49 AM, Tobias Feldhaus <[email protected]
> wrote:
> We have 3x Supermicro Intel Dual Xeon E5-2620v3 powered systems with 32GB
> ECC
> memory, 4x 10 Gigabit Ethernet NICs (Intel X520-DA2), and 2x Gigabit
> Ethernet
> onboard NICs connected towards a Virtual Chassis of a Juniper EX 4550
> Ethernet
> Switch, running OpenBSD 5.8 with all (11) patches.
>
> We want to use these 3 systems as loadbalancers, 2x 10GE (trunk0, LACP)
> inbound,
> 2x 10GE (trunk1, LACP) outbound, 2x 1GE (trunk2, LACP) for Pfsync.
>
> LB-1 shares a public IP with LB-2, and LB-2 and LB-3 do the same (via
> CARP). We
> use relayd for Loadbalancing the traffic towards 3 backend servers, all
> they
> currently do is serving a HTTP 200 OK response.
>
> When we load tested one LB's HTTP performance alone with wrk - we get
> about 40k
> req/s when testing with one machine in the same network as a client, and
> more
> than 100k req/s when testing with 3 client machines. Doing the test with
> HTTPS
> brings the performance down to 1400 req/s, and it does not matter if using
> more
> or less clients, the total number of req/s stays almost the same.
>
> The overall load of the systems is low (below 2-3), memory utilization is
> low as well.
>
> As we don't have experience with OpenBSD and relayd we can only compare
> these
> numbers to FreeBSD and HAproxy, which we used in our previous setup. Our
> configuration files are listed below - we would be happy about any comment
> how
> to improve the HTTPS performance.
>
>
>
#############################################################################
###
> # OpenBSD sysctl.conf
>
> net.inet.carp.preempt=1
>
> ### Tried with and without the following settings - with some effect
> kern.bufcachepercent=90
>
> kern.maxfiles=200000
> kern.maxproc=50000
>
> kern.maxclusters=32768
> machdep.allowaperture=2
> net.inet.ip.forwarding=1
> net.inet.ip.ifq.maxlen=8192
> net.inet.ip.mtudisc=0
> net.inet.tcp.rfc3390=1
> net.inet.tcp.mssdflt=1440
>
>
>
>
#############################################################################
###
> # OpenBSD relayd.conf
>
> ip4_244 = "xx.xx.xx.244"
> ip4_245 = "xx.xx.xx.245"
>
> tracker5 = "10.5.3.34"
> tracker6 = "10.5.3.42"
> tracker7 = "10.5.3.50"
>
> interval 10
> table <trackers> { $tracker5, $tracker6, $tracker7 }
>
> prefork 12
>
> http protocol https {
>
> ### TCP performance options
> tcp { nodelay, sack, socket buffer 65536, backlog 128 }
>
> match request header append "X-Forwarded-For" value "$REMOTE_ADDR"
> match request header append "X-Forwarded-By" \
> value "$SERVER_ADDR:$SERVER_PORT"
> match request header set "Connection" value "close"
>
> tls { no tlsv1.0, ciphers HIGH }
> tls session cache disable # tried enabling/disabling -> no effect
> }
>
> relay wwwssl {
> listen on $ip4_244 port 443 tls
> listen on $ip4_245 port 443 tls
> protocol "https"
> forward to <trackers> port 8083 mode loadbalance check tcp
> }
>
> relay www {
> listen on $ip4_244 port 80
> listen on $ip4_245 port 80
> forward to <trackers> port 8083 mode loadbalance check tcp
> }
>
>
>
#############################################################################
###
> # OpenBSD: pf.conf
>
> tcp_services = "{ domain, www, https }"
> udp_services = "{ domain }"
> tcp_public_services = "{ www, https }"
> icmp_types = "{ echorep, echoreq, unreach}"
> icmp6_types = "{ echorep, echoreq, unreach, timex, paramprob, routersol,
> routeradv, neighbrsol, neighbradv, redir }"
>
> pfsync_int = trunk2 # Pfsync interface
> int_if = trunk1 # DMZ (internal) interface
> ext_if = trunk0 # External CARP interface
>
> # Increase limits
> set limit { states 100000, src-nodes 100000, table-entries 2000000 }
>
> # Optimizations
> set optimization aggressive
> set timeout { adaptive.end 120000, interval 2, tcp.tsdiff 5, tcp.first 5,
> tcp.closing 5, tcp.closed 5, tcp.finwait 5, tcp.established 4200} # tried
> with
> # and without - very small effect
>
> # See pf.conf(5) and /etc/examples/pf.conf
> anchor "relayd/*"
>
> set reassemble yes
> set block-policy drop
> set loginterface $ext_if
> set skip on lo
> set skip on $int_if
> set skip on $pfsync_int
>
> # Scrub incoming
> match in all scrub (no-df max-mss 1440)
>
> # Block everything by default
> block all
>
> # Activate spoofing protection
> block in quick from urpf-failed
>
> # Allow main service of this host
> pass out proto tcp to port $tcp_services keep state
> pass in proto tcp to port $tcp_public_services keep state
> pass proto udp to port $udp_services keep state
>
> # Pass CARP and pfsync
> pass proto carp keep state (no-sync)
> pass quick proto pfsync keep state (no-sync)
>
> # SSH backup channel from Wooga office
> pass in on trunk0 inet proto tcp from 185.74.12.0/22 to any port 22 keep
> state (no-sync)
>
> # Allow pings for Pingdom status checks
> pass on trunk0 inet proto icmp icmp-type $icmp_types keep state (no-sync)
> pass on trunk0 inet6 proto icmp6 icmp6-type $icmp6_types keep state
> (no-sync)
>
--
*Tobias Feldhaus | Data Engineer*
Wooga GmbH | Saarbrücker Str. 38 | D-10405 Berlin
Place of business: Berlin
Registered at the local court Berlin-Charlottenburg, HRB 117846 B
Managing Directors: Jens Begemann, Philipp Möser, Jan Miczaika
