Hello Wget developers, I have prepared TCP Fast Open support for HTTP requests in Wget.
I have prepared sample TCP segment dump to show how it works: https://github.com/jy987321/Wget/wiki/TCP-TFO The TFO is disabled by default and can be enabled using --tcp-fast-open option when running Wget. The support for TFO has been added only recently to Linux kernel (3.6 for client and 3.7 for server) and is not supported on Winodws AFAIK. Therefore, it is added as conditional compilation code. In order to force the use of it, you can pass --enable-tcp-tfo (--disable-tcp-tfo) to ./configure script. The default is to include the code if possible (if required constants are defined in <sys/socket.h>). The support for TFO has been added to Python test suite. In order to make it work, you need to enable both client-side and server-side TFO support in your Linux kernel: $ echo 3 > /proc/sys/net/ipv4/tcp_fastopen If there is no server-side support, TCP will transparently fall back to using traditional 3-way handshake. Please let me know what do you think of this feature. Suggestions or comments about patches are more than welcome. The code is available as attachment and on Github. https://github.com/jy987321/Wget/commits/master-hubert PS I was hoping to do some performance tests, but have not found too many servers that support TFO (actually it is only google.com that I have found so far). -- Hubert Tarasiuk
From 467336a84e94195a6832a07def2c2a8a42a659c8 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 13:29:45 +0200 Subject: [PATCH 1/9] Const-qualify buffer when writing to fd. * src/connect.c: add const qualifier in sock_write, fd_write. * src/connect.h: add const qualifier in fd_write declaration and in transport_implementation struct. * src/gnutls.c: add const qualifier in wgnutls_write. --- src/connect.c | 4 ++-- src/connect.h | 4 ++-- src/gnutls.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/connect.c b/src/connect.c index 024b231..45eb57f 100644 --- a/src/connect.c +++ b/src/connect.c @@ -764,7 +764,7 @@ sock_read (int fd, char *buf, int bufsize) } static int -sock_write (int fd, char *buf, int bufsize) +sock_write (int fd, const char *buf, int bufsize) { int res; do @@ -947,7 +947,7 @@ fd_peek (int fd, char *buf, int bufsize, double timeout) TIMEOUT. */ int -fd_write (int fd, char *buf, int bufsize, double timeout) +fd_write (int fd, const char *buf, int bufsize, double timeout) { int res; struct transport_info *info; diff --git a/src/connect.h b/src/connect.h index 9b08848..5ea2616 100644 --- a/src/connect.h +++ b/src/connect.h @@ -65,7 +65,7 @@ bool test_socket_open (int); struct transport_implementation { int (*reader) (int, char *, int, void *); - int (*writer) (int, char *, int, void *); + int (*writer) (int, const char *, int, void *); int (*poller) (int, double, int, void *); int (*peeker) (int, char *, int, void *); const char *(*errstr) (int, void *); @@ -75,7 +75,7 @@ struct transport_implementation { void fd_register_transport (int, struct transport_implementation *, void *); void *fd_transport_context (int); int fd_read (int, char *, int, double); -int fd_write (int, char *, int, double); +int fd_write (int, const char *, int, double); int fd_peek (int, char *, int, double); const char *fd_errstr (int); void fd_close (int); diff --git a/src/gnutls.c b/src/gnutls.c index be04342..250d7fb 100644 --- a/src/gnutls.c +++ b/src/gnutls.c @@ -326,7 +326,7 @@ wgnutls_read (int fd, char *buf, int bufsize, void *arg) } static int -wgnutls_write (int fd _GL_UNUSED, char *buf, int bufsize, void *arg) +wgnutls_write (int fd _GL_UNUSED, const char *buf, int bufsize, void *arg) { int ret; struct wgnutls_transport_context *ctx = arg; -- 2.4.1
From f7f5dda03e06c35f329f9c741f63b3f3c9acd2d1 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 13:38:49 +0200 Subject: [PATCH 2/9] Some refactoring in request_send. * src/http.c (request_send): Move request string length calculation to separate function. * src/http.c (request_str_len): Calculate length of string representation of request struct. --- src/http.c | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/http.c b/src/http.c index 777903b..f8b1d04 100644 --- a/src/http.c +++ b/src/http.c @@ -304,6 +304,29 @@ request_remove_header (struct request *req, const char *name) return false; } +/* Calculate string length needed to represent req. */ +static size_t +request_str_len (const struct request *req) +{ + int i; + size_t len = 0; + + /* METHOD " " ARG " " "HTTP/1.1" "\r\n" */ + len += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2; + + for (i = 0; i < req->hcount; i++) + { + struct request_header *hdr = &req->headers[i]; + /* NAME ": " VALUE "\r\n" */ + len += strlen (hdr->name) + 2 + strlen (hdr->value) + 2; + } + + /* "\r\n\0" */ + len += 3; + + return len; +} + #define APPEND(p, str) do { \ int A_len = strlen (str); \ memcpy (p, str, A_len); \ @@ -318,23 +341,8 @@ static int request_send (const struct request *req, int fd, FILE *warc_tmp) { char *request_string, *p; - int i, size, write_error; - - /* Count the request size. */ - size = 0; - - /* METHOD " " ARG " " "HTTP/1.0" "\r\n" */ - size += strlen (req->method) + 1 + strlen (req->arg) + 1 + 8 + 2; - - for (i = 0; i < req->hcount; i++) - { - struct request_header *hdr = &req->headers[i]; - /* NAME ": " VALUE "\r\n" */ - size += strlen (hdr->name) + 2 + strlen (hdr->value) + 2; - } - - /* "\r\n\0" */ - size += 3; + int i, write_error; + const size_t size = request_str_len (req); p = request_string = alloca_array (char, size); @@ -370,7 +378,7 @@ request_send (const struct request *req, int fd, FILE *warc_tmp) { /* Write a copy of the data to the WARC record. */ int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp); - if (warc_tmp_written != size - 1) + if ((size_t) warc_tmp_written != size - 1) return -2; } return write_error; -- 2.4.1
From 37e0843185f35ba206a785938f9d44ea6aff87f5 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 14:11:24 +0200 Subject: [PATCH 3/9] Split request_send into request creation and actual sending. * src/http.c (request_to_string): Create ready-to-send request string from request structure. * src/http.c (request_to_send): Take string instead of request structure and just send it. --- src/http.c | 59 +++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/http.c b/src/http.c index f8b1d04..65fcc87 100644 --- a/src/http.c +++ b/src/http.c @@ -333,24 +333,23 @@ request_str_len (const struct request *req) p += A_len; \ } while (0) -/* Construct the request and write it to FD using fd_write. - If warc_tmp is set to a file pointer, the request string will - also be written to that file. */ - -static int -request_send (const struct request *req, int fd, FILE *warc_tmp) +/* Create ready-to-send request string from request structure. */ +static char * +request_to_string (const struct request *req) { char *request_string, *p; - int i, write_error; - const size_t size = request_str_len (req); - - p = request_string = alloca_array (char, size); + int i; + const size_t str_size = request_str_len (req); - /* Generate the request. */ + p = request_string = xmalloc (str_size); - APPEND (p, req->method); *p++ = ' '; - APPEND (p, req->arg); *p++ = ' '; - memcpy (p, "HTTP/1.1\r\n", 10); p += 10; + /* Generate the request. */ + APPEND (p, req->method); + *p++ = ' '; + APPEND (p, req->arg); + *p++ = ' '; + memcpy (p, "HTTP/1.1\r\n", 10); + p += 10; for (i = 0; i < req->hcount; i++) { @@ -360,25 +359,35 @@ request_send (const struct request *req, int fd, FILE *warc_tmp) APPEND (p, hdr->value); *p++ = '\r', *p++ = '\n'; } +#undef APPEND *p++ = '\r', *p++ = '\n', *p++ = '\0'; assert (p - request_string == size); -#undef APPEND + return request_string; +} + +/* Write request string to FD using fd_write. + If warc_tmp is set to a file pointer, the request string will + also be written to that file. */ +static int +request_send (const char *req_str, int fd, FILE *warc_tmp) +{ + int write_error; + const size_t size = strlen (req_str); - DEBUGP (("\n---request begin---\n%s---request end---\n", request_string)); + DEBUGP (("\n---request begin---\n%s---request end---\n", req_str)); /* Send the request to the server. */ - - write_error = fd_write (fd, request_string, size - 1, -1); + write_error = fd_write (fd, req_str, size, -1); if (write_error < 0) logprintf (LOG_VERBOSE, _("Failed writing HTTP request: %s.\n"), fd_errstr (fd)); else if (warc_tmp != NULL) { /* Write a copy of the data to the WARC record. */ - int warc_tmp_written = fwrite (request_string, 1, size - 1, warc_tmp); - if ((size_t) warc_tmp_written != size - 1) + int warc_tmp_written = fwrite (req_str, 1, size, warc_tmp); + if ((size_t) warc_tmp_written != size) return -2; } return write_error; @@ -2006,6 +2015,7 @@ establish_connection (struct url *u, struct url **conn_ref, CONNECT method to request passthrough. */ struct request *connreq = request_new ("CONNECT", aprintf ("%s:%d", u->host, u->port)); + char *connreq_str; SET_USER_AGENT (connreq); if (proxyauth) { @@ -2020,7 +2030,9 @@ establish_connection (struct url *u, struct url **conn_ref, aprintf ("%s:%d", u->host, u->port), rel_value); - write_error = request_send (connreq, sock, 0); + connreq_str = request_to_string (connreq); + write_error = request_send (connreq_str, sock, 0); + xfree (connreq_str); request_free (&connreq); if (write_error < 0) { @@ -2473,6 +2485,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, struct iri *iri, int count) { struct request *req = NULL; + char *req_str; char *type = NULL; char *user, *passwd; @@ -2641,7 +2654,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, } /* Send the request to server. */ - write_error = request_send (req, sock, warc_tmp); + req_str = request_to_string (req); + write_error = request_send (req_str, sock, warc_tmp); + xfree (req_str); if (write_error >= 0) { -- 2.4.1
From 7fcc510f9c3f633892774d71b00b2037c488a3ed Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Wed, 27 May 2015 22:53:20 +0200 Subject: [PATCH 4/9] Add autoconf test for TCP Fast Open. * src/configure.ac: --enable-tcp-tfo or --disable-tcp-tfo, enable by default if available. --- configure.ac | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/configure.ac b/configure.ac index 01ea237..db8c8db 100644 --- a/configure.ac +++ b/configure.ac @@ -711,6 +711,48 @@ AS_IF([test "X$enable_pcre" != "Xno"],[ ]) ]) +dnl +dnl Check for TCP Fast Open +dnl + +AC_ARG_ENABLE(tcp-tfo, + AC_HELP_STRING([--enable-tcp-tfo],[Enable TCP Fast Open]), + [case "${enable_tcp_tfo}" in + no) + tcp_tfo=no + ;; + yes) + tcp_tfo=yes + force_tcp_tfo=yes + ;; + auto) + tcp_tfo=yes + ;; + *) + AC_MSG_ERROR([Invalid --enable-tcp-tfo argument \`$enable_tcp_tfo']) + ;; + esac + ], [ + tcp_tfo=yes + ] +) + +AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM( + [[#include <sys/socket.h>],[int tfo = MSG_FASTOPEN;]])], + [has_tfo=yes], + [has_tfo=no]) + +TFO_INFO="TCP Fast Open will not be used" + +if test X$tcp_tfo = Xyes; then + if test X$has_tfo = Xyes; then + AC_DEFINE([HAVE_TFO], [1], [Define if TCP Fast Open is available.]) + TFO_INFO="TCP Fast Open will be used" + elif test X$force_tcp_tfo = Xyes; then + AC_MSG_ERROR([TCP Fast Open not supported on this system]) + fi +fi dnl Needed by src/Makefile.am AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"]) @@ -743,4 +785,5 @@ AC_MSG_NOTICE([Summary of build options: Debugging: $ENABLE_DEBUG Assertions: $ENABLE_ASSERTION Valgrind: $VALGRIND_INFO + TCP Fast Open: $TFO_INFO ]) -- 2.4.1
From 2ff91b3bcdd2f8a521c813cb03a2cf4c62df6996 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 15:01:16 +0200 Subject: [PATCH 5/9] Add --tcp-fast-open option. * src/init.c (commands[]): Add command. * src/init.c (defaults): Enable fast-open by default. * src/options.h (options): Add tcp_fast_open option. --- src/init.c | 3 +++ src/main.c | 7 +++++++ src/options.h | 3 +++ 3 files changed, 13 insertions(+) diff --git a/src/init.c b/src/init.c index a436ef2..13e2f31 100644 --- a/src/init.c +++ b/src/init.c @@ -282,6 +282,9 @@ static const struct { { "spider", &opt.spider, cmd_boolean }, { "startpos", &opt.start_pos, cmd_bytes }, { "strictcomments", &opt.strict_comments, cmd_boolean }, +#ifdef HAVE_TFO + { "tcp-fast-open", &opt.tcp_fast_open, cmd_boolean }, +#endif { "timeout", NULL, cmd_spec_timeout }, { "timestamping", &opt.timestamping, cmd_boolean }, { "tries", &opt.ntry, cmd_number_inf }, diff --git a/src/main.c b/src/main.c index a0044d9..513244d 100644 --- a/src/main.c +++ b/src/main.c @@ -299,6 +299,9 @@ static struct cmdline_option option_data[] = { "spider", 0, OPT_BOOLEAN, "spider", -1 }, { "start-pos", 0, OPT_VALUE, "startpos", -1 }, { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, +#ifdef HAVE_TFO + { "tcp-fast-open", 0, OPT_BOOLEAN, "tcp-fast-open", -1 }, +#endif { "timeout", 'T', OPT_VALUE, "timeout", -1 }, { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, { "if-modified-since", 0, OPT_BOOLEAN, "if-modified-since", -1 }, @@ -653,6 +656,10 @@ HTTP options:\n"), --auth-no-challenge send Basic HTTP authentication information\n\ without first waiting for the server's\n\ challenge\n"), +#ifdef HAVE_TFO + N_("\ + --tcp-fast-open use TCP Fast Open for HTTP requests\n"), +#endif "\n", #ifdef HAVE_SSL diff --git a/src/options.h b/src/options.h index bef1f10..b746678 100644 --- a/src/options.h +++ b/src/options.h @@ -126,6 +126,9 @@ struct options char *http_passwd; /* HTTP password. */ char **user_headers; /* User-defined header(s). */ bool http_keep_alive; /* whether we use keep-alive */ +#ifdef HAVE_TFO + bool tcp_fast_open; /* use TCP Fast Open (RFC7413) */ +#endif bool use_proxy; /* Do we use proxy? */ bool allow_cache; /* Do we allow server-side caching? */ -- 2.4.1
From cd590a980773be750909ee203d3dff6d037c9043 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 15:03:55 +0200 Subject: [PATCH 6/9] Enable TCP_FASTOPEN in testenv server. * testenv/server/http/http_server.py (HTTPd): set socket option. --- testenv/server/http/http_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py index 2356f1c..3036e44 100644 --- a/testenv/server/http/http_server.py +++ b/testenv/server/http/http_server.py @@ -20,6 +20,9 @@ class StoppableHTTPServer(HTTPServer): request_headers = list() """ Define methods for configuring the Server. """ + def server_bind(self): + super(StoppableHTTPServer, self).server_bind() + self.socket.setsockopt(socket.SOL_TCP, socket.TCP_FASTOPEN, 5) def server_conf(self, filelist, conf_dict): """ Set Server Rules and File System for this instance. """ -- 2.4.1
From 3e8b3a031180b226854ba493e77d5b3a58d06229 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 15:55:11 +0200 Subject: [PATCH 7/9] Support TCP Fast Open in connect_to_host and connect_to_ip. * src/connect.c (cwt_context): Hold initial data in context. * src/connect.c (connect_with_timeout_callback): Send TCP TFO segment when initial data is given. * src/connect.c (connect_with_timeout): Forward initial data into context. * src/connect.c (connect_to_ip): Forward initial data. * src/connect.c (connect_to_host): Forward initial data. * src/connect.h: Update connect_to_host and connect_to_ip headers with initial data for TCP TFO. * src/ftp.c (getftp): Do not use TFO for now -> pass NULL. * src/http.c (establish_connection): Do not use TFO for now -> pass NULL. --- src/connect.c | 48 ++++++++++++++++++++++++++++++++++++++++++------ src/connect.h | 5 +++-- src/ftp.c | 4 ++-- src/http.c | 2 +- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/connect.c b/src/connect.c index 45eb57f..4f93e2f 100644 --- a/src/connect.c +++ b/src/connect.c @@ -224,13 +224,38 @@ struct cwt_context { const struct sockaddr *addr; socklen_t addrlen; int result; + const char *initial_data; + size_t initial_data_len; }; static void connect_with_timeout_callback (void *arg) { struct cwt_context *ctx = (struct cwt_context *)arg; - ctx->result = connect (ctx->fd, ctx->addr, ctx->addrlen); +#ifdef HAVE_TFO + if (ctx->initial_data != NULL) + { + ssize_t ret = sendto (ctx->fd, ctx->initial_data, ctx->initial_data_len, + MSG_FASTOPEN, ctx->addr, ctx->addrlen); + if (ret <= 0) + ctx->result = ret; + else if (ret == ctx->initial_data_len) + ctx->result = ret; + else + { + /* We did not manage to send everything with sendto. + Transfer rest of data using fd_write. */ + ctx->result = fd_write (ctx->fd, + ctx->initial_data + ret, + ctx->initial_data_len - ret, + -1); + } + } + else +#endif + { + ctx->result = connect (ctx->fd, ctx->addr, ctx->addrlen); + } } /* Like connect, but specifies a timeout. If connecting takes longer @@ -239,12 +264,15 @@ connect_with_timeout_callback (void *arg) static int connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen, + const char *initial_data, size_t initial_data_len, double timeout) { struct cwt_context ctx; ctx.fd = fd; ctx.addr = addr; ctx.addrlen = addrlen; + ctx.initial_data = initial_data; + ctx.initial_data_len = initial_data_len; if (run_with_timeout (timeout, connect_with_timeout_callback, &ctx)) { @@ -262,11 +290,14 @@ connect_with_timeout (int fd, const struct sockaddr *addr, socklen_t addrlen, connecting to. */ int -connect_to_ip (const ip_address *ip, int port, const char *print) +connect_to_ip (const ip_address *ip, int port, const char *print, + const char *initial_data, size_t initial_data_len, + ssize_t *connect_result) { struct sockaddr_storage ss; struct sockaddr *sa = (struct sockaddr *)&ss; int sock; + ssize_t my_connect_result; /* If PRINT is non-NULL, print the "Connecting to..." line, with PRINT being the host name we're connecting to. */ @@ -351,9 +382,12 @@ connect_to_ip (const ip_address *ip, int port, const char *print) } } + if (!connect_result) + connect_result = &my_connect_result; + *connect_result = connect_with_timeout (sock, sa, sockaddr_size (sa), initial_data, + initial_data_len, opt.connect_timeout); /* Connect the socket to the remote endpoint. */ - if (connect_with_timeout (sock, sa, sockaddr_size (sa), - opt.connect_timeout) < 0) + if (*connect_result < 0) goto err; /* Success. */ @@ -384,7 +418,8 @@ connect_to_ip (const ip_address *ip, int port, const char *print) DNS until connecting to one of them succeeds. */ int -connect_to_host (const char *host, int port) +connect_to_host (const char *host, int port, const char *init_data, + size_t init_data_len, ssize_t *connect_result) { int i, start, end; int sock; @@ -404,7 +439,8 @@ connect_to_host (const char *host, int port) for (i = start; i < end; i++) { const ip_address *ip = address_list_address_at (al, i); - sock = connect_to_ip (ip, port, host); + sock = connect_to_ip (ip, port, host, init_data, init_data_len, + connect_result); if (sock >= 0) { /* Success. */ diff --git a/src/connect.h b/src/connect.h index 5ea2616..44fdd74 100644 --- a/src/connect.h +++ b/src/connect.h @@ -40,8 +40,9 @@ as that of the covered work. */ enum { E_HOST = -100 }; -int connect_to_host (const char *, int); -int connect_to_ip (const ip_address *, int, const char *); +int connect_to_host (const char *, int, const char *, size_t, ssize_t *); +int connect_to_ip (const ip_address *, int, const char *, const char *, size_t, + ssize_t *); int bind_local (const ip_address *, int *); int accept_connection (int); diff --git a/src/ftp.c b/src/ftp.c index 68f1a33..36621e8 100644 --- a/src/ftp.c +++ b/src/ftp.c @@ -297,7 +297,7 @@ getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread, /* First: Establish the control connection. */ - csock = connect_to_host (host, port); + csock = connect_to_host (host, port, NULL, 0, NULL); if (csock == E_HOST) return HOSTERR; else if (csock < 0) @@ -870,7 +870,7 @@ Error in server response, closing control connection.\n")); { DEBUGP (("trying to connect to %s port %d\n", print_address (&passive_addr), passive_port)); - dtsock = connect_to_ip (&passive_addr, passive_port, NULL); + dtsock = connect_to_ip (&passive_addr, passive_port, NULL, NULL, 0, NULL); if (dtsock < 0) { int save_errno = errno; diff --git a/src/http.c b/src/http.c index 65fcc87..788f5fa 100644 --- a/src/http.c +++ b/src/http.c @@ -1999,7 +1999,7 @@ establish_connection (struct url *u, struct url **conn_ref, if (sock < 0) { - sock = connect_to_host (conn->host, conn->port); + sock = connect_to_host (conn->host, conn->port, NULL, 0, NULL); if (sock == E_HOST) return HOSTERR; else if (sock < 0) -- 2.4.1
From 3dbbad6754a22affc25cb7c51cb2f116a1cb8d13 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sun, 24 May 2015 16:03:06 +0200 Subject: [PATCH 8/9] Pass pointer by value instead of by reference to establish_connection. * src/http.c (establish_connection): Use pointer from arguments. * src/http.c (gethttp): Pass pointer instead of its address. --- src/http.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/http.c b/src/http.c index 788f5fa..5a9cc8f 100644 --- a/src/http.c +++ b/src/http.c @@ -1932,13 +1932,12 @@ static uerr_t establish_connection (struct url *u, struct url **conn_ref, struct http_stat *hs, struct url *proxy, char **proxyauth, - struct request **req_ref, bool *using_ssl, + struct request *req, bool *using_ssl, bool inhibit_keep_alive, int *sock_ref) { bool host_lookup_failed = false; int sock = *sock_ref; - struct request *req = *req_ref; struct url *conn = *conn_ref; struct response *resp; int write_error; @@ -2021,9 +2020,6 @@ establish_connection (struct url *u, struct url **conn_ref, { request_set_header (connreq, "Proxy-Authorization", *proxyauth, rel_value); - /* Now that PROXYAUTH is part of the CONNECT request, - zero it out so we don't send proxy authorization with - the regular request below. */ *proxyauth = NULL; } request_set_header (connreq, "Host", @@ -2105,7 +2101,6 @@ establish_connection (struct url *u, struct url **conn_ref, #endif /* HAVE_SSL */ } *conn_ref = conn; - *req_ref = req; *sock_ref = sock; return RETROK; } @@ -2626,7 +2621,7 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, keep_alive = false; { - uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, &req, + uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, req, &using_ssl, inhibit_keep_alive, &sock); if (conn_err != RETROK) { -- 2.4.1
From cb5d4538588187dd55c718643321d6dd22ddee62 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Mon, 25 May 2015 11:25:13 +0200 Subject: [PATCH 9/9] Prototype of TCP Fast Open support http.c. * src/http.c: Rename "establish_connection" to "connect_and_send_request". * src/http.c (connect_and_send_request): Send request from here, either by TFO or the usual way. * src/http.c (gethttp): Do not send the request from here. --- src/http.c | 125 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 91 insertions(+), 34 deletions(-) diff --git a/src/http.c b/src/http.c index 5a9cc8f..3d91fcc 100644 --- a/src/http.c +++ b/src/http.c @@ -1929,12 +1929,12 @@ initialize_proxy_configuration (struct url *u, struct request *req, } static uerr_t -establish_connection (struct url *u, struct url **conn_ref, - struct http_stat *hs, struct url *proxy, - char **proxyauth, - struct request *req, bool *using_ssl, - bool inhibit_keep_alive, - int *sock_ref) +connect_and_send_request (struct url *u, struct url **conn_ref, + struct http_stat *hs, struct url *proxy, + char **proxyauth, + struct request *req, bool *using_ssl, + bool inhibit_keep_alive, + int *sock_ref, FILE *warc_tmp, ssize_t *send_result) { bool host_lookup_failed = false; int sock = *sock_ref; @@ -1942,6 +1942,12 @@ establish_connection (struct url *u, struct url **conn_ref, struct response *resp; int write_error; int statcode; + int retval = RETROK; + char *req_str = request_to_string (req); +#ifdef HAVE_TFO + size_t req_len = strlen (req_str); + bool tfo = !!(u->scheme == SCHEME_HTTP && opt.tcp_fast_open); +#endif if (! inhibit_keep_alive) { @@ -1964,6 +1970,10 @@ establish_connection (struct url *u, struct url **conn_ref, &host_lookup_failed)) { int family = socket_family (pconn.socket, ENDPOINT_PEER); + /* No need for TFO if connection is already established. */ +#ifdef HAVE_TFO + tfo = false; +#endif sock = pconn.socket; *using_ssl = pconn.ssl; #if ENABLE_IPV6 @@ -1988,7 +1998,8 @@ establish_connection (struct url *u, struct url **conn_ref, logprintf(LOG_NOTQUIET, _("%s: unable to resolve host address %s\n"), exec_name, quote (relevant->host)); - return HOSTERR; + retval = HOSTERR; + goto cleanup; } else if (sock != -1) { @@ -1998,12 +2009,31 @@ establish_connection (struct url *u, struct url **conn_ref, if (sock < 0) { - sock = connect_to_host (conn->host, conn->port, NULL, 0, NULL); +#ifdef HAVE_TFO + if (tfo) + { + DEBUGP (("TFO connection\n")); + DEBUGP (("\n---request begin---\n%s---request end---\n", req_str)); + sock = connect_to_host (conn->host, conn->port, req_str, + req_len, send_result); + } + else +#endif + { + DEBUGP (("Non-TFO connection\n")); + sock = connect_to_host (conn->host, conn->port, NULL, 0, NULL); + } if (sock == E_HOST) - return HOSTERR; + { + retval = HOSTERR; + goto cleanup; + } else if (sock < 0) - return (retryable_socket_connect_error (errno) - ? CONERROR : CONIMPOSSIBLE); + { + retval = (retryable_socket_connect_error (errno) + ? CONERROR : CONIMPOSSIBLE); + goto cleanup; + } #ifdef HAVE_SSL if (proxy && u->scheme == SCHEME_HTTPS) @@ -2033,7 +2063,8 @@ establish_connection (struct url *u, struct url **conn_ref, if (write_error < 0) { CLOSE_INVALIDATE (sock); - return WRITEFAILED; + retval = WRITEFAILED; + goto cleanup; } head = read_http_response_head (sock); @@ -2042,7 +2073,8 @@ establish_connection (struct url *u, struct url **conn_ref, logprintf (LOG_VERBOSE, _("Failed reading proxy response: %s\n"), fd_errstr (sock)); CLOSE_INVALIDATE (sock); - return HERR; + retval = HERR; + goto cleanup; } message = NULL; if (!*head) @@ -2062,7 +2094,8 @@ establish_connection (struct url *u, struct url **conn_ref, quotearg_style (escape_quoting_style, _("Malformed status line"))); xfree (head); - return HERR; + retval = HERR; + goto cleanup; } xfree(hs->message); hs->message = xstrdup (message); @@ -2074,7 +2107,8 @@ establish_connection (struct url *u, struct url **conn_ref, logprintf (LOG_NOTQUIET, _("Proxy tunneling failed: %s"), message ? quotearg_style (escape_quoting_style, message) : "?"); xfree (message); - return CONSSLERR; + retval = CONSSLERR; + goto cleanup; } xfree (message); @@ -2089,20 +2123,46 @@ establish_connection (struct url *u, struct url **conn_ref, if (!ssl_connect_wget (sock, u->host)) { CLOSE_INVALIDATE (sock); - return CONSSLERR; + retval = CONSSLERR; + goto cleanup; } else if (!ssl_check_certificate (sock, u->host)) { CLOSE_INVALIDATE (sock); - return VERIFCERTERR; + retval = VERIFCERTERR; + goto cleanup; } *using_ssl = true; } #endif /* HAVE_SSL */ } + +#ifdef HAVE_TFO + if (tfo) + { + /* We sent the request using TFO. Now just store it to warc file. */ + if (warc_tmp != NULL) + { + int warc_tmp_written = fwrite (req_str, 1, req_len, warc_tmp); + if ((size_t) warc_tmp_written != req_len) + { + *send_result = -2; + goto cleanup; + } + } + } + else +#endif + { + /* Send the request as usual. */ + DEBUGP (("Sending regular request...\n")); + *send_result = request_send (req_str, sock, NULL); + } +cleanup: *conn_ref = conn; *sock_ref = sock; - return RETROK; + xfree (req_str); + return retval; } static uerr_t @@ -2480,13 +2540,12 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, struct iri *iri, int count) { struct request *req = NULL; - char *req_str; char *type = NULL; char *user, *passwd; char *proxyauth; int statcode; - int write_error; + ssize_t write_error; wgint contlen, contrange; struct url *conn; FILE *fp; @@ -2620,16 +2679,6 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, if (inhibit_keep_alive) keep_alive = false; - { - uerr_t conn_err = establish_connection (u, &conn, hs, proxy, &proxyauth, req, - &using_ssl, inhibit_keep_alive, &sock); - if (conn_err != RETROK) - { - retval = conn_err; - goto cleanup; - } - } - /* Open the temporary file where we will write the request. */ if (warc_enabled) { @@ -2648,10 +2697,18 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, } } - /* Send the request to server. */ - req_str = request_to_string (req); - write_error = request_send (req_str, sock, warc_tmp); - xfree (req_str); + /* This will both establish connection and send the request. Connection error + will be returned, send error will be stored to write_error. */ + { + uerr_t conn_err = connect_and_send_request (u, &conn, hs, proxy, &proxyauth, + req, &using_ssl, inhibit_keep_alive, + &sock, warc_tmp, &write_error); + if (conn_err != RETROK) + { + retval = conn_err; + goto cleanup; + } + } if (write_error >= 0) { -- 2.4.1
signature.asc
Description: OpenPGP digital signature
