On Fri, Jul 09, 2021 at 02:58:50PM +0200, Alexander Bluhm wrote:
> 1. With non parallel forwarding the IPsec traffic stalls after a while.
> esp_input_cb: authentication failed for packet in SA 10.3.45.35/83089fff
Together with tobhe@ we found the issue. The authentication before
decryption uses a different replay counter than after decryption.
As we run crypto operations async, thousands of packets are stored
in the crypto task. While that the replay counter of the tdb can
change. Then the higher 32 bits may increment although the lower
32 bits did not wrap.
checkreplaywindow() must be called twice per packet with the same
replay counter. Store the value in struct tdb_crypto while dangling
in the task queue and doing crypto operations.
> 2. With parallel forwarding the kernel crashes.
This must be a different bug. But with reliable single core IPsec
it should be easier to debug MP.
ok?
bluhm
Index: netinet/ip_ah.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_ah.c,v
retrieving revision 1.150
diff -u -p -r1.150 ip_ah.c
--- netinet/ip_ah.c 8 Jul 2021 21:07:19 -0000 1.150
+++ netinet/ip_ah.c 16 Jul 2021 15:10:00 -0000
@@ -550,7 +550,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
sizeof(u_int32_t), &btsx);
btsx = ntohl(btsx);
- switch (checkreplaywindow(tdb, btsx, &esn, 0)) {
+ switch (checkreplaywindow(tdb, tdb->tdb_rpl, btsx, &esn, 0)) {
case 0: /* All's well. */
break;
case 1:
@@ -697,6 +697,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
tc->tc_proto = tdb->tdb_sproto;
tc->tc_rdomain = tdb->tdb_rdomain;
memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
+ tc->tc_rpl = tdb->tdb_rpl;
KERNEL_LOCK();
error = crypto_dispatch(crp);
@@ -715,6 +716,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
{
const struct auth_hash *ahx = tdb->tdb_authalgxform;
int roff, rplen, skip, protoff;
+ u_int64_t rpl;
u_int32_t btsx, esn;
caddr_t ptr;
unsigned char calc[AH_ALEN_MAX];
@@ -727,6 +729,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
skip = tc->tc_skip;
protoff = tc->tc_protoff;
+ rpl = tc->tc_rpl;
rplen = AH_FLENGTH + sizeof(u_int32_t);
@@ -756,7 +759,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
sizeof(u_int32_t), &btsx);
btsx = ntohl(btsx);
- switch (checkreplaywindow(tdb, btsx, &esn, 1)) {
+ switch (checkreplaywindow(tdb, rpl, btsx, &esn, 1)) {
case 0: /* All's well. */
#if NPFSYNC > 0
pfsync_update_tdb(tdb,0);
Index: netinet/ip_esp.c
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_esp.c,v
retrieving revision 1.168
diff -u -p -r1.168 ip_esp.c
--- netinet/ip_esp.c 16 Jul 2021 15:08:39 -0000 1.168
+++ netinet/ip_esp.c 16 Jul 2021 15:10:00 -0000
@@ -388,7 +388,7 @@ esp_input(struct mbuf *m, struct tdb *td
&btsx);
btsx = ntohl(btsx);
- switch (checkreplaywindow(tdb, btsx, &esn, 0)) {
+ switch (checkreplaywindow(tdb, tdb->tdb_rpl, btsx, &esn, 0)) {
case 0: /* All's well */
break;
case 1:
@@ -511,6 +511,7 @@ esp_input(struct mbuf *m, struct tdb *td
tc->tc_proto = tdb->tdb_sproto;
tc->tc_rdomain = tdb->tdb_rdomain;
tc->tc_dst = tdb->tdb_dst;
+ tc->tc_rpl = tdb->tdb_rpl;
/* Decryption descriptor */
if (espx) {
@@ -549,6 +550,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
int hlen, roff, skip, protoff;
struct mbuf *m1, *mo;
const struct auth_hash *esph;
+ u_int64_t rpl;
u_int32_t btsx, esn;
caddr_t ptr;
#ifdef ENCDEBUG
@@ -557,6 +559,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
skip = tc->tc_skip;
protoff = tc->tc_protoff;
+ rpl = tc->tc_rpl;
NET_ASSERT_LOCKED();
@@ -590,7 +593,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
&btsx);
btsx = ntohl(btsx);
- switch (checkreplaywindow(tdb, btsx, &esn, 1)) {
+ switch (checkreplaywindow(tdb, rpl, btsx, &esn, 1)) {
case 0: /* All's well */
#if NPFSYNC > 0
pfsync_update_tdb(tdb,0);
@@ -1049,14 +1052,15 @@ esp_output_cb(struct tdb *tdb, struct td
* return 3 for packet within current window but already received
*/
int
-checkreplaywindow(struct tdb *tdb, u_int32_t seq, u_int32_t *seqh, int commit)
+checkreplaywindow(struct tdb *tdb, u_int64_t t, u_int32_t seq, u_int32_t *seqh,
+ int commit)
{
u_int32_t tl, th, wl;
u_int32_t packet, window = TDB_REPLAYMAX - TDB_REPLAYWASTE;
int idx, esn = tdb->tdb_flags & TDBF_ESN;
- tl = (u_int32_t)tdb->tdb_rpl;
- th = (u_int32_t)(tdb->tdb_rpl >> 32);
+ tl = (u_int32_t)t;
+ th = (u_int32_t)(t >> 32);
/* Zero SN is not allowed */
if ((esn && seq == 0 && tl <= AH_HMAC_INITIAL_RPL && th == 0) ||
Index: netinet/ip_ipsp.h
===================================================================
RCS file: /data/mirror/openbsd/cvs/src/sys/netinet/ip_ipsp.h,v
retrieving revision 1.201
diff -u -p -r1.201 ip_ipsp.h
--- netinet/ip_ipsp.h 13 Jul 2021 08:16:17 -0000 1.201
+++ netinet/ip_ipsp.h 16 Jul 2021 15:10:00 -0000
@@ -432,12 +432,13 @@ struct tdb_ident {
};
struct tdb_crypto {
- u_int32_t tc_spi;
union sockaddr_union tc_dst;
- u_int8_t tc_proto;
+ u_int64_t tc_rpl;
+ u_int32_t tc_spi;
int tc_protoff;
int tc_skip;
u_int tc_rdomain;
+ u_int8_t tc_proto;
};
struct ipsecinit {
@@ -622,7 +623,7 @@ int tcp_signature_tdb_output(struct mbuf
int, int);
/* Replay window */
-int checkreplaywindow(struct tdb *, u_int32_t, u_int32_t *, int);
+int checkreplaywindow(struct tdb *, u_int64_t, u_int32_t, u_int32_t *, int);
/* Packet processing */
int ipsp_process_packet(struct mbuf *, struct tdb *, int, int);
