Hello, I hope the patch attached to this email does not get mangled.
Gianluca
From b0b2ac599c6ae01ff0db0d8a14be64c3110b61a8 Mon Sep 17 00:00:00 2001 From: Gianluca Cannata <[email protected]> Date: Wed, 11 Nov 2026 11:42:15 +0100 Subject: [PATCH httpfs v3 2/2] open_connection: use libcurl for HTTP transfer --- Makefile.am | 2 +- configure.in | 3 +- http.c | 182 +++++++++++++------------------- httpfs.c | 39 ++++--- httpfs.h | 4 +- netfs.c | 11 +- parsehtml.c | 11 +- tests/Makefile.am | 13 ++- tests/integration_test_httpfs.c | 131 +++++++++++++++++++++++ tests/test_httpfs.c | 81 +++++++++----- 10 files changed, 304 insertions(+), 173 deletions(-) create mode 100644 tests/integration_test_httpfs.c diff --git a/Makefile.am b/Makefile.am index 27d2588..473f60b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS=httpfs -HTTPFS_SHARED_LIBS = -lxml2 -lpthread -lnetfs -lfshelp -lports -liohelp -lshouldbeinlibc -lhurdbugaddr +HTTPFS_SHARED_LIBS = -lcurl -lxml2 -lpthread -lnetfs -lfshelp -lports -liohelp -lshouldbeinlibc -lhurdbugaddr httpfs_LDADD = $(HTTPFS_SHARED_LIBS) $(LIBS) httpfs_SOURCES=httpfs.c \ diff --git a/configure.in b/configure.in index 0643c19..8b86a6a 100644 --- a/configure.in +++ b/configure.in @@ -4,9 +4,10 @@ AM_INIT_AUTOMAKE AC_PROG_CC +PKG_CHECK_MODULES([CURL], [libcurl >= 7.0.0]) PKG_CHECK_MODULES(HTTPFS, libxml-2.0) -HTTPFS_SHARED_LIBS="-lxml2 -lpthread -lnetfs -lfshelp -lports -liohelp -lshouldbeinlibc -lhurdbugaddr" +HTTPFS_SHARED_LIBS="$CURL_LIBS -lxml2 -lpthread -lnetfs -lfshelp -lports -liohelp -lshouldbeinlibc -lhurdbugaddr" AC_SUBST([HTTPFS_SHARED_LIBS]) # Optional test suite integration (auto-enabled if dependencies are found) diff --git a/http.c b/http.c index 50184c0..6bb6c11 100644 --- a/http.c +++ b/http.c @@ -34,140 +34,100 @@ #include <hurd/hurd_types.h> #include <hurd/netfs.h> -/* do a DNS lookup for NAME and store result in *ENT */ -error_t lookup_host (char *url, struct hostent **ent) +struct curl_read_request { + char *buffer; + loff_t offset; +}; + +static size_t curl_write_response_to_buffer(char *data, size_t size, size_t nmemb, void *userdata) { - if ( (*ent = gethostbyname(url)) == NULL ) - { - fprintf(stderr,"wrong host name\n"); - return EINVAL; + struct curl_read_request *request = (struct curl_read_request *) userdata; + if (!request) { + return 0; } - return 0; + + size_t total_size = size * nmemb - request->offset; + if (total_size == 0) { + return 0; + } + + memcpy(request->buffer, data + request->offset, total_size); + + return total_size; } /* open a connection with the remote web server */ -error_t open_connection(struct netnode *node, int *fd,off_t *head_len) +error_t open_connection(struct netnode *node, char *buffer, loff_t offset, off_t *head_len) { /* HTTP GET command returns head and body so we have to prune the * head. *HEAD_LEN indicates the header length, for pruning upto that */ - - error_t err; - struct hostent *hptr; - struct sockaddr_in dest; - ssize_t written; - size_t towrite; - char buffer[4096]; - ssize_t bytes_read; - char *token,*mesg; - int code; - char delimiters0[] = " "; - char delimiters1[] = "\n"; + error_t err = 0; + long http_status_code = 200; + double http_content_length = 0.0f; + CURLcode curl_err = CURLE_OK; + struct curl_read_request *request; - bzero(&dest,sizeof(dest)); - dest.sin_family = AF_INET; - dest.sin_port = htons (port); + request = malloc(sizeof(struct curl_read_request)); + memset(request, 0, sizeof(struct curl_read_request)); + request->buffer = buffer; + request->offset = offset; - if ( !strcmp(ip_addr,"0.0.0.0") ) - { - /* connection is not through a proxy server - * find IP addr. of remote server */ - err = lookup_host (node->url, &hptr); - if (err) - { - fprintf(stderr,"Could not find IP addr %s\n",node->url); - return err; - } - dest.sin_addr = *(struct in_addr *)hptr->h_addr; + CURL *curl = curl_easy_init(); + if (!curl) { + return (err = ENOMEM); } - else - { - /* connection is through the proxy server - * need not find IP of remote server - * find IP of the proxy server */ - if ( inet_aton(ip_addr,&dest.sin_addr) == 0 ) - { - fprintf(stderr,"Invalid IP for proxy\n"); - return -1; - } + + if (!strcmp(ip_addr, "0.0.0.0")) { + /* Connection is not through a proxy server. Do nothing. */ + } else { + /* Connection is through a proxy server. */ + curl_easy_setopt(curl, CURLOPT_PROXY, ip_addr); } - if (debug_flag) - fprintf (stderr, "trying to open %s:%d/%s\n", node->url, - port, node->conn_req); + curl_easy_setopt(curl, CURLOPT_URL, node->url); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_response_to_buffer); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, request); + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); - *fd = socket (AF_INET, SOCK_STREAM, 0); - if (*fd == -1) - { - fprintf(stderr,"Socket creation error\n"); - return errno; - } + if (debug_flag) + fprintf (stderr, "trying to open %s:%d\n", node->url, + port); - err = connect (*fd, (struct sockaddr *)&dest, sizeof (dest)); - if (err == -1) - { - fprintf(stderr,"Cannot connect to remote host\n"); - return errno; + curl_err = curl_easy_perform(curl); + if (curl_err != CURLE_OK) { + fprintf (stderr, "curl_easy_perform(): %s\n", curl_easy_strerror(curl_err)); + err = EAGAIN; + goto cleanup; } - /* Send a HEAD request find header length */ - sprintf(buffer,"HEAD %s HTTP/1.0\r\nHost: %s\r\n\r\n",node->conn_req,node->url); - towrite = strlen (buffer); - written = TEMP_FAILURE_RETRY (write (*fd, buffer, towrite)); - if ( written == -1 || written < towrite ) - { - fprintf(stderr,"Could not send an HTTP request to host\n"); - return errno; - } - - bytes_read = read(*fd,buffer,sizeof(buffer)); - if ( bytes_read < 0 ) - { - fprintf(stderr,"Error with HEAD read\n"); - return errno; + curl_err = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_status_code); + if (curl_err != CURLE_OK) { + fprintf(stderr, "Cannot get HTTP status code: %s\n", curl_easy_strerror(curl_err)); + err = EAGAIN; + goto cleanup; + } else if (http_status_code != 200) { + fprintf(stderr, "HTTP status code is: %ld\n", http_status_code); + err = EIO; + goto cleanup; } - *head_len = bytes_read; - token = strtok(buffer,delimiters0); - token = strtok(NULL,delimiters0); - sscanf(token,"%d",&code); - token = strtok(NULL,delimiters1); - mesg = strdup(token); - if ( code != 200 ) - { - /* page does not exist */ - fprintf(stderr,"Error Page not Accesible\n"); - fprintf(stderr,"%d %s\n",code,mesg); - return EBADF; - } - - close(*fd); - - /* Send the GET request for the url */ - *fd = socket (AF_INET, SOCK_STREAM, 0); - if (*fd == -1) - { - fprintf(stderr,"Socket creation error\n"); - return errno; + curl_err = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &http_content_length); + if (curl_err != CURLE_OK) { + fprintf(stderr, "Could not determine HTTP content length: %s\n", curl_easy_strerror(curl_err)); + err = EIO; + goto cleanup; } - err = connect (*fd, (struct sockaddr *)&dest, sizeof (dest)); - if (err == -1) - { - fprintf(stderr,"Cannot connect to remote host\n"); - return errno; - } + *head_len = (off_t) http_content_length; - towrite = strlen (node->comm_buf); + err = 0; + goto cleanup; - /* guard against EINTR failures */ - written = TEMP_FAILURE_RETRY (write (*fd, node->comm_buf, towrite)); - written += TEMP_FAILURE_RETRY (write (*fd, "\n\n", 2)); - if (written == -1 || written < (towrite+2)) - { - fprintf(stderr,"Could not send GET request to remote host\n"); - return errno; - } - return 0; + cleanup: + free(request); + curl_easy_cleanup(curl); + return err; } /* fetch a directory node from the web server diff --git a/httpfs.c b/httpfs.c index 2da7f0d..163d091 100644 --- a/httpfs.c +++ b/httpfs.c @@ -35,6 +35,7 @@ main (int argc, char **argv) error_t err; mach_port_t bootstrap; char *temp_url, *temp, *run; + int is_https = 0; char type; char *comm_buf; /* XXX: Is an http request limited to 200 bytes? */ port = 80; @@ -42,6 +43,12 @@ main (int argc, char **argv) mode = 1; /* means directory */ ip_addr = strdup ("0.0.0.0"); + CURLcode curl_err = CURLE_OK; + curl_err = curl_global_init(CURL_GLOBAL_ALL); + if (curl_err != CURLE_OK) { + error (1, 0, "libcurl cannot be initialized."); + } + if (debug_flag) fprintf (stderr, "pid %d\n", getpid ()); httpfs_parse_args (argc, argv); @@ -65,9 +72,17 @@ main (int argc, char **argv) if (! temp_url) error (1, errno, "Cannot duplicate url."); - if (!strncmp (temp_url, "http://", 7)) + if (!strncmp (temp_url, "http://", 7)) { /* go ahead of http:// if given in url */ temp_url = temp_url + 7; + is_https = 0; + } else if (!strncmp (temp_url, "https://", 8)) { + /* go ahead of https:// if given in url */ + temp_url = temp_url + 8; + is_https = 1; + port = 443; + } + if (strchr (temp_url, '/') == NULL) error (1, 0, "Url must have a /, e.g., www.gnu.org/"); @@ -104,7 +119,12 @@ main (int argc, char **argv) dir_tok[no_of_slashes] = strdup ("index.html"); } - strcpy (conn_req, "http://"); + if (is_https) { + strcpy (conn_req, "https://"); + } else { + strcpy (conn_req, "http://"); + } + if (temp_url[strlen (temp_url) - 1] == '/') { strcat (conn_req, temp_url); @@ -140,20 +160,8 @@ main (int argc, char **argv) /* XXX: why is tmp hardcoded? */ httpfs->root = httpfs_make_node (type, url, conn_req, comm_buf, "tmp"); + httpfs->root->nn->type = HTTP_DIR_NOT_FILLED; netfs_init (); - /* translator set to a directory */ - if (mode) - { - /* fill the directory node with files - * call parser for that - * only the current directory is filled - * subdirectories within them are indicated by type - * HTTP_DIR_UNFILLED, and are filled as on demand when an - * ls request comes for them */ - err = parse (httpfs->root->nn); - if (err) - error (1, err, "Error in Parsing."); - } if (debug_flag) fprintf (stderr, "entering the main loop\n"); @@ -165,5 +173,6 @@ main (int argc, char **argv) /* NOT REACHED */ free (httpfs); + curl_global_cleanup(); return 0; } diff --git a/httpfs.h b/httpfs.h index 3b54e48..1ab0669 100644 --- a/httpfs.h +++ b/httpfs.h @@ -21,6 +21,8 @@ #include <sys/stat.h> #include <netdb.h> +#include <curl/curl.h> + #include <hurd/hurd_types.h> #include "config.h" @@ -92,7 +94,7 @@ error_t lookup_host (char *url, struct hostent **ent); /* store the remote socket in *FD and establish the connection and send the * http GET request */ -error_t open_connection(struct netnode *node, int *fd,off_t *head_len); +error_t open_connection(struct netnode *node, char *buffer, loff_t offset, off_t *head_len); /* make an instance of `struct netnode' with the specified parameters, * return NULL on error */ diff --git a/netfs.c b/netfs.c index 0b49d93..e0e6b4b 100644 --- a/netfs.c +++ b/netfs.c @@ -464,18 +464,16 @@ error_t netfs_attempt_readlink (struct iouser *user, struct node *node, error_t netfs_attempt_read (struct iouser *cred, struct node *node, loff_t offset, size_t *len, void *data) { - error_t err; - static int remote_fd; + error_t err = 0; char buffer[4096]; int bytes_read; off_t head_len; if ( offset==0) { - err = open_connection (node->nn, &remote_fd,&head_len); + err = open_connection (node->nn, buffer, offset, &head_len); if (err) return err; - bytes_read = pread(remote_fd, buffer, sizeof(buffer), offset); - memcpy(data,(void *)(buffer+head_len),sizeof(buffer)-(sizeof(char)*head_len)); + bytes_read = strlen(buffer) + head_len; if (bytes_read < 0) err = errno; else { @@ -486,8 +484,7 @@ error_t netfs_attempt_read (struct iouser *cred, struct node *node, } else { - bzero(buffer,sizeof(buffer)); - bytes_read = pread(remote_fd, data, sizeof(buffer), offset); + bytes_read = strlen(buffer) + head_len; if (bytes_read < 0) err = errno; else { diff --git a/parsehtml.c b/parsehtml.c index 3bd202f..5f3ede0 100644 --- a/parsehtml.c +++ b/parsehtml.c @@ -98,7 +98,6 @@ error_t parse(struct netnode *node) /* SAX HTML Parsing Variables */ htmlSAXHandlerPtr emptysax = ∅ error_t err; - static int remote_fd; off_t head_len=0,offset=0; char buffer[MAXBUF]; int bytes_read=0; @@ -125,20 +124,16 @@ error_t parse(struct netnode *node) /* get the HTML stream corresponding to the directory * send a GET request */ - err = open_connection(node,&remote_fd,&head_len); + err = open_connection(node, buffer, offset, &head_len); if ( err ) return err; - bytes_read = pread(remote_fd,buffer,sizeof(char)*6,offset); + bytes_read = strlen(buffer) + head_len; if ( bytes_read > 0 ) { /* Parser operates in Push mode */ ctxt = htmlCreatePushParserCtxt(emptysax,NULL,buffer,bytes_read,"abracabadra",0); //file name is irrelevant abracabadra - - while ( (bytes_read=pread(remote_fd,buffer,sizeof(buffer),offset)) > 0) { - htmlParseChunk(ctxt,buffer,bytes_read,0); - } - htmlParseChunk(ctxt,buffer,0,1); + htmlParseChunk(ctxt, buffer, bytes_read, 0); doc = ctxt->myDoc; htmlFreeParserCtxt(ctxt); if ( doc != NULL ) diff --git a/tests/Makefile.am b/tests/Makefile.am index 2823d0a..641f70c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,15 @@ -check_PROGRAMS = test_httpfs +check_PROGRAMS = test_httpfs integration_test_httpfs +# Unit tests test_httpfs_SOURCES = test_httpfs.c ../http.c ../parsehtml.c ../node.c ../extract.c ../netfs.c ../args.c ../globals.c -test_httpfs_CFLAGS = -I$(top_srcdir) $(CMOCKA_CFLAGS) -test_httpfs_LDFLAGS = -Wl,--wrap=gethostbyname -Wl,--export-dynamic +test_httpfs_CFLAGS = -I$(top_srcdir) $(CMOCKA_CFLAGS) $(MHD_CFLAGS) $(CURL_FLAGS) +test_httpfs_LDFLAGS = -Wl,--wrap=curl_easy_init -Wl,--wrap=curl_easy_perform -Wl,--wrap=curl_easy_setopt -Wl,--wrap=curl_easy_cleanup -Wl,--export-dynamic test_httpfs_LDADD = $(CMOCKA_LIBS) $(MHD_LIBS) $(HTTPFS_SHARED_LIBS) +# Integration tests +integration_test_httpfs_SOURCES = integration_test_httpfs.c ../http.c ../parsehtml.c ../node.c ../extract.c ../netfs.c ../args.c ../globals.c +integration_test_httpfs_CFLAGS = -I$(top_srcdir) $(CMOCKA_CFLAGS) $(MHD_CFLAGS) $(CURL_FLAGS) +integration_test_httpfs_LDFLAGS = -Wl,--export-dynamic +integration_test_httpfs_LDADD = $(CMOCKA_LIBS) $(MHD_LIBS) $(HTTPFS_SHARED_LIBS) + TESTS = $(check_PROGRAMS) diff --git a/tests/integration_test_httpfs.c b/tests/integration_test_httpfs.c new file mode 100644 index 0000000..20b0bed --- /dev/null +++ b/tests/integration_test_httpfs.c @@ -0,0 +1,131 @@ +/* + * httpfs unit tests + * + * Copyright (C) 2026 Free Software Foundation, Inc. + * This file is part of the GNU Hurd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <netdb.h> +#include <microhttpd.h> +#include <curl/curl.h> +#include <stdarg.h> +#include <stddef.h> +#include <setjmp.h> +#include <cmocka.h> + +#include <netinet/in.h> +#include <arpa/inet.h> + +#include "globals.h" + +/* http.c integration tests */ +static enum MHD_Result +answer_to_connection (void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **req_cls) +{ + const char *page = "<html><body>Hello, browser!</body></html>"; + struct MHD_Response *response; + enum MHD_Result ret; + (void) cls; + (void) url; + (void) method; + (void) version; + (void) upload_data; + (void) upload_data_size; + (void) req_cls; + + response = MHD_create_response_from_buffer_static(strlen(page), page); + ret = MHD_queue_response(connection, MHD_HTTP_OK, response); + MHD_destroy_response(response); + + return ret; +} + +static void test_open_connection_curl_easy_perform_success(void **state) +{ + (void) state; + + /* No proxy */ + ip_addr = "0.0.0.0"; + debug_flag = 1; + port = 8080; + + CURLcode curl_err = curl_global_init(CURL_GLOBAL_ALL); + if (curl_err != CURLE_OK) { + fprintf(stderr, "curl_global_init(): %s\n", curl_easy_strerror(curl_err)); + exit(EXIT_FAILURE); + } + + struct MHD_Daemon *daemon; + + struct sockaddr_in localhost; + memset(&localhost, 0, sizeof(localhost)); + localhost.sin_family = AF_INET; + localhost.sin_port = htons(port); + localhost.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + daemon = MHD_start_daemon(MHD_USE_AUTO | MHD_USE_INTERNAL_POLLING_THREAD, + port, NULL, NULL, + &answer_to_connection, NULL, + MHD_OPTION_SOCK_ADDR, (struct sockaddr *) &localhost, + MHD_OPTION_END); + + if (!daemon) { + fprintf(stderr, "MHD_start_daemon(): ENOMEM\n"); + exit(EXIT_FAILURE); + } + + struct netnode *node = malloc(sizeof(struct netnode)); + if (!node) { + fprintf(stderr, "malloc(): ENOMEM.\n"); + exit(EXIT_FAILURE); + } + + memset(node, 0, sizeof(struct netnode)); + node->url = "http://localhost:8080/"; + node->conn_req = "http://localhost:8080/"; + + char buffer[4096]; + off_t head_len; + + error_t err = open_connection(node, buffer, 0, &head_len); + + assert_int_equal(err, 0); + assert_int_equal(head_len, 0); + + free(node); + MHD_stop_daemon(daemon); + curl_global_cleanup(); +} + +int main(void) { + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_open_connection_curl_easy_perform_success), + }; + + return cmocka_run_group_tests(tests, NULL, NULL); +} diff --git a/tests/test_httpfs.c b/tests/test_httpfs.c index bd2d65b..503f79d 100644 --- a/tests/test_httpfs.c +++ b/tests/test_httpfs.c @@ -37,50 +37,79 @@ #include "globals.h" -struct hostent *__wrap_gethostbyname(const char *name) { - assert_non_null(name); +/* http.c unit tests */ +void __wrap_curl_easy_cleanup(CURL *curl) { + (void) curl; +} - if (strcmp(name, "www.example.com") == 0) { - return (struct hostent *)mock(); - } +CURLcode __wrap_curl_easy_setopt(CURL *curl, CURLoption option, ...) { + (void) curl; + (void) option; - return (struct hostent *)mock(); + return CURLE_OK; } -/* http.c unit tests */ -static void test_lookup_host_1(void **state) { - (void) state; +CURL *__wrap_curl_easy_init() { + return (CURL *)mock(); +} + +CURLcode __wrap_curl_easy_perform(CURL *curl) { + return (CURLcode) mock(); +} - struct hostent mock_hostent; - memset(&mock_hostent, 0, sizeof(mock_hostent)); - mock_hostent.h_name = "www.example.com"; +static void test_open_connection_curl_easy_init_fails(void **state) +{ + (void) state; - will_return(__wrap_gethostbyname, &mock_hostent); + will_return(__wrap_curl_easy_init, NULL); - struct hostent *result = NULL; - int ret = lookup_host("www.example.com", &result); + error_t err = open_connection(NULL, NULL, 0, NULL); - assert_int_equal(ret, 0); - assert_non_null(result); - assert_string_equal(result->h_name, "www.example.com"); + assert_int_equal(err, ENOMEM); } -static void test_lookup_host_2(void **state) { +static void test_open_connection_curl_easy_perform_fails(void **state) +{ (void) state; - will_return(__wrap_gethostbyname, NULL); + CURLcode curl_err = curl_global_init(CURL_GLOBAL_ALL); + if (curl_err != CURLE_OK) { + fprintf(stderr, "curl_global_init(): %s\n", curl_easy_strerror(curl_err)); + assert_true(false); + } + + CURL *mock_curl = (CURL *) 0x123456789; + will_return(__wrap_curl_easy_init, mock_curl); + will_return_count(__wrap_curl_easy_setopt, CURLE_OK, -2); + will_return(__wrap_curl_easy_perform, CURLE_ABORTED_BY_CALLBACK); + + /* No proxy */ + ip_addr = "0.0.0.0"; + debug_flag = 1; + port = 8080; + + struct netnode *node = malloc(sizeof(struct netnode)); + if (!node) { + fprintf(stderr, "malloc(): ENOMEM.\n"); + assert_true(false); + } + + memset(node, 0, sizeof(struct netnode)); + node->url = "http://localhost:8080/"; + node->conn_req = "http://localhost:8080/"; + + error_t err = open_connection(node, NULL, 0, NULL); - struct hostent *result = NULL; - int ret = lookup_host("invalid.url", &result); + assert_int_equal(err, EAGAIN); - assert_int_equal(ret, EINVAL); - assert_null(result); + free(node); + curl_global_cleanup(); } int main(void) { const struct CMUnitTest tests[] = { - cmocka_unit_test(test_lookup_host_1), - cmocka_unit_test(test_lookup_host_2), + cmocka_unit_test(test_open_connection_curl_easy_init_fails), + cmocka_unit_test(test_open_connection_curl_easy_perform_fails), }; return cmocka_run_group_tests(tests, NULL, NULL); -- 2.56.1
