tags 158590 +patch thanks These patches will add a "reget" command to the current sid and sarge versions of sftp. They are not thoroughly tested and could do with some cleaning up, but they appear to work.
The tricky bit, as was explained to me when I raised this issue originally, is not in resuming a download, but in preventing an incompletely downloaded file from having gaps. sftp can send multiple concurrent requests for several blocks of a file and may receive them out of order, so it is necessary for it to keep track of any gaps and truncate the file at the first gap if a download is aborted. (Actually it could hold onto isolated blocks in memory, but I'm not sure that the resulting higher memory consumption would be acceptable.) I believe I have ensured that the truncation will be done even if a download is aborted by a signal or fatal error, but I'm not entirely happy about the way that's done. It's also difficult to test this as I believe that it is uncommon for servers to send blocks out of order. Ben. -- Ben Hutchings The two most common things in the universe are hydrogen and stupidity.
diff -ur openssh-4.2p1-5/progressmeter.c openssh-4.2p1/progressmeter.c
--- openssh-4.2p1-5/progressmeter.c 2005-06-16 04:18:05.000000000 +0100
+++ openssh-4.2p1/progressmeter.c 2005-12-29 19:03:10.000000000 +0000
@@ -240,7 +240,7 @@
start = last_update = time(NULL);
file = f;
end_pos = filesize;
- cur_pos = 0;
+ cur_pos = *ctr;
counter = ctr;
stalled = 0;
bytes_per_second = 0;
diff -ur openssh-4.2p1-5/sftp.c openssh-4.2p1/sftp.c
--- openssh-4.2p1-5/sftp.c 2005-08-22 23:06:56.000000000 +0100
+++ openssh-4.2p1/sftp.c 2005-12-29 19:02:32.000000000 +0000
@@ -103,6 +103,7 @@
#define I_SYMLINK 21
#define I_VERSION 22
#define I_PROGRESS 23
+#define I_REGET 24
struct CMD {
const char *c;
@@ -119,6 +120,7 @@
{ "dir", I_LS },
{ "exit", I_QUIT },
{ "get", I_GET },
+ { "reget", I_REGET },
{ "mget", I_GET },
{ "help", I_HELP },
{ "lcd", I_LCHDIR },
@@ -502,7 +504,8 @@
}
static int
-process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd, int pflag)
+process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd, int pflag,
+ int rflag)
{
char *abs_src = NULL;
char *abs_dst = NULL;
@@ -554,7 +557,8 @@
abs_dst = tmp;
printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
- if (do_download(conn, g.gl_pathv[i], abs_dst, pflag) == -1)
+ if (do_download(conn, g.gl_pathv[i], abs_dst, pflag, rflag)
+ == -1)
err = -1;
xfree(abs_dst);
abs_dst = NULL;
@@ -897,6 +901,7 @@
*path1 = *path2 = NULL;
switch (cmdnum) {
case I_GET:
+ case I_REGET:
case I_PUT:
if (parse_getput_flags(&cp, pflag))
return(-1);
@@ -1030,7 +1035,9 @@
err = -1;
break;
case I_GET:
- err = process_get(conn, path1, path2, *pwd, pflag);
+ case I_REGET:
+ err = process_get(conn, path1, path2, *pwd, pflag,
+ cmdnum == I_REGET);
break;
case I_PUT:
err = process_put(conn, path1, path2, *pwd, pflag);
diff -ur openssh-4.2p1-5/sftp-client.c openssh-4.2p1/sftp-client.c
--- openssh-4.2p1-5/sftp-client.c 2005-08-02 08:07:08.000000000 +0100
+++ openssh-4.2p1/sftp-client.c 2005-12-29 19:02:32.000000000 +0000
@@ -735,9 +735,15 @@
buffer_free(&msg);
}
+/* Variables used for removing isolated fragments of the downloaded file
+ * in case of a signal or fatal error */
+static volatile int truncate_fd = -1;
+static volatile u_int32_t truncate_len_upper;
+static volatile u_int32_t truncate_len_lower;
+
int
do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
- int pflag)
+ int pflag, int rflag)
{
Attrib junk, *a;
Buffer msg;
@@ -753,10 +759,12 @@
u_int64_t offset;
TAILQ_ENTRY(request) tq;
};
- TAILQ_HEAD(reqhead, request) requests;
+ TAILQ_HEAD(reqhead, request) requests, fragments;
+ u_int64_t contig_len;
struct request *req;
TAILQ_INIT(&requests);
+ TAILQ_INIT(&fragments);
a = do_stat(conn, remote_path, 0);
if (a == NULL)
@@ -799,7 +807,8 @@
return(-1);
}
- local_fd = open(local_path, O_WRONLY | O_CREAT | O_TRUNC,
+ local_fd = open(local_path,
+ O_WRONLY | O_CREAT | (rflag ? 0 : O_TRUNC),
mode | S_IWRITE);
if (local_fd == -1) {
error("Couldn't open local file \"%s\" for writing: %s",
@@ -808,11 +817,32 @@
xfree(handle);
return(-1);
}
+ if (rflag) {
+ struct stat stat_buf;
+ if (fstat(local_fd, &stat_buf) == -1) {
+ error("Couldn't stat local file \"%s\": %s",
+ local_path, strerror(errno));
+ close(local_fd);
+ buffer_free(&msg);
+ xfree(handle);
+ return(-1);
+ }
+ offset = stat_buf.st_size;
+ } else {
+ offset = 0;
+ }
/* Read from remote and write to local */
- write_error = read_error = write_errno = num_req = offset = 0;
+ write_error = read_error = write_errno = num_req = 0;
max_req = 1;
- progress_counter = 0;
+ progress_counter = contig_len = offset;
+
+ truncate_len_lower = contig_len;
+ truncate_len_upper = contig_len >> 32;
+ truncate_fd = local_fd;
+ signal(SIGTERM, cleanup_exit);
+ signal(SIGINT, cleanup_exit);
+ signal(SIGHUP, cleanup_exit);
if (showprogress && size != 0)
start_progress_meter(remote_path, size, &progress_counter);
@@ -892,9 +922,52 @@
if (len == req->len) {
TAILQ_REMOVE(&requests, req, tq);
- xfree(req);
num_req--;
+ }
+
+ /* Update contig_len and/or fragments */
+ if (req->offset == contig_len) {
+ struct request *frag, *next_frag;
+ contig_len += len;
+ if (len == req->len)
+ xfree(req);
+ for (frag = TAILQ_FIRST(&fragments);
+ frag && frag->offset == contig_len;
+ frag = next_frag) {
+ next_frag = TAILQ_NEXT(frag, tq);
+ contig_len += frag->len;
+ TAILQ_REMOVE(&fragments, frag, tq);
+ xfree(frag);
+ }
+ truncate_len_lower = contig_len;
+ truncate_len_upper = contig_len >> 32;
} else {
+ struct request *frag, *new_frag;
+ if (len == req->len) {
+ new_frag = req;
+ } else {
+ /* Request will stay in the queue
+ * so we need to duplicate it */
+ new_frag = xmalloc(sizeof(*req));
+ *new_frag = *req;
+ /* Set fragment len to actual len */
+ new_frag->len = len;
+ }
+ /* Insert the fragment at the appropriate
+ * point in the list (ordered by offset) */
+ for (frag = TAILQ_FIRST(&fragments);
+ frag && frag->offset < req->offset;
+ frag = TAILQ_NEXT(frag, tq))
+ ;
+ if (frag)
+ TAILQ_INSERT_BEFORE(frag, new_frag,
+ tq);
+ else
+ TAILQ_INSERT_TAIL(&fragments, new_frag,
+ tq);
+ }
+
+ if (len != req->len) {
/* Resend the request for the missing data */
debug3("Short data block, re-requesting "
"%llu -> %llu (%2d)",
@@ -936,6 +1009,24 @@
if (TAILQ_FIRST(&requests) != NULL)
fatal("Transfer complete, but requests still in queue");
+ if (contig_len != progress_counter) {
+ struct request *frag, *next_frag;
+ for (frag = TAILQ_FIRST(&fragments);
+ frag != NULL;
+ frag = next_frag) {
+ next_frag = TAILQ_NEXT(frag, tq);
+ TAILQ_REMOVE(&fragments, frag, tq);
+ xfree(frag);
+ }
+ if (ftruncate(local_fd, contig_len) == -1) {
+ error("Couldn't truncate \"%s\": %s"
+ "; there may be gaps in it",
+ local_path, strerror(errno));
+ }
+ }
+
+ truncate_fd = -1;
+
if (read_error) {
error("Couldn't read from remote file \"%s\" : %s",
remote_path, fx2txt(status));
@@ -1155,3 +1246,13 @@
buffer_free(&msg);
return(status);
}
+
+void
+cleanup_exit(int i)
+{
+ // XXX: error reporting?
+ ftruncate(truncate_fd,
+ ((u_int64_t)truncate_len_upper << 32) | truncate_len_lower);
+
+ _exit(i);
+}
diff -ur openssh-4.2p1-5/sftp-client.h openssh-4.2p1/sftp-client.h
--- openssh-4.2p1-5/sftp-client.h 2005-05-26 03:05:49.000000000 +0100
+++ openssh-4.2p1/sftp-client.h 2005-12-29 19:02:32.000000000 +0000
@@ -86,9 +86,10 @@
/*
* Download 'remote_path' to 'local_path'. Preserve permissions and times
- * if 'pflag' is set
+ * if 'pflag' is set. If 'rflag' is set, assume the contents of the local
+ * file (if it exists) are a prefix of the remote file.
*/
-int do_download(struct sftp_conn *, char *, char *, int);
+int do_download(struct sftp_conn *, char *, char *, int, int);
/*
* Upload 'local_path' to 'remote_path'. Preserve permissions and times
--- openssh-3.8.1p1-8.sarge.4/progressmeter.c 2004-02-06 05:41:37.000000000 +0000
+++ openssh-3.8.1p1/progressmeter.c 2005-12-29 18:18:08.000000000 +0000
@@ -231,7 +231,7 @@
start = last_update = time(NULL);
file = f;
end_pos = filesize;
- cur_pos = 0;
+ cur_pos = *stat;
counter = stat;
stalled = 0;
bytes_per_second = 0;
diff -ur openssh-3.8.1p1-8.sarge.4/sftp.c openssh-3.8.1p1/sftp.c
--- openssh-3.8.1p1-8.sarge.4/sftp.c 2004-03-08 12:12:19.000000000 +0000
+++ openssh-3.8.1p1/sftp.c 2005-12-29 17:45:43.000000000 +0000
@@ -86,6 +86,7 @@
#define I_SYMLINK 21
#define I_VERSION 22
#define I_PROGRESS 23
+#define I_REGET 24
struct CMD {
const char *c;
@@ -102,6 +103,7 @@
{ "dir", I_LS },
{ "exit", I_QUIT },
{ "get", I_GET },
+ { "reget", I_REGET },
{ "mget", I_GET },
{ "help", I_HELP },
{ "lcd", I_LCHDIR },
@@ -437,7 +439,8 @@
}
static int
-process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd, int pflag)
+process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd, int pflag,
+ int rflag)
{
char *abs_src = NULL;
char *abs_dst = NULL;
@@ -489,7 +492,8 @@
abs_dst = tmp;
printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
- if (do_download(conn, g.gl_pathv[i], abs_dst, pflag) == -1)
+ if (do_download(conn, g.gl_pathv[i], abs_dst, pflag, rflag)
+ == -1)
err = -1;
xfree(abs_dst);
abs_dst = NULL;
@@ -802,6 +806,7 @@
*path1 = *path2 = NULL;
switch (cmdnum) {
case I_GET:
+ case I_REGET:
case I_PUT:
if (parse_getput_flags(&cp, pflag))
return(-1);
@@ -935,7 +940,9 @@
err = -1;
break;
case I_GET:
- err = process_get(conn, path1, path2, *pwd, pflag);
+ case I_REGET:
+ err = process_get(conn, path1, path2, *pwd, pflag,
+ cmdnum == I_REGET);
break;
case I_PUT:
err = process_put(conn, path1, path2, *pwd, pflag);
diff -ur openssh-3.8.1p1-8.sarge.4/sftp-client.c openssh-3.8.1p1/sftp-client.c
--- openssh-3.8.1p1-8.sarge.4/sftp-client.c 2004-03-08 12:12:02.000000000 +0000
+++ openssh-3.8.1p1/sftp-client.c 2005-12-29 18:40:22.000000000 +0000
@@ -726,9 +726,15 @@
buffer_free(&msg);
}
+/* Variables used for removing isolated fragments of the downloaded file
+ * in case of a signal or fatal error */
+static volatile int truncate_fd = -1;
+static volatile u_int32_t truncate_len_upper;
+static volatile u_int32_t truncate_len_lower;
+
int
do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
- int pflag)
+ int pflag, int rflag)
{
Attrib junk, *a;
Buffer msg;
@@ -744,10 +750,12 @@
u_int64_t offset;
TAILQ_ENTRY(request) tq;
};
- TAILQ_HEAD(reqhead, request) requests;
+ TAILQ_HEAD(reqhead, request) requests, fragments;
+ u_int64_t contig_len;
struct request *req;
TAILQ_INIT(&requests);
+ TAILQ_INIT(&fragments);
a = do_stat(conn, remote_path, 0);
if (a == NULL)
@@ -790,7 +798,8 @@
return(-1);
}
- local_fd = open(local_path, O_WRONLY | O_CREAT | O_TRUNC,
+ local_fd = open(local_path,
+ O_WRONLY | O_CREAT | (rflag ? 0 : O_TRUNC),
mode | S_IWRITE);
if (local_fd == -1) {
error("Couldn't open local file \"%s\" for writing: %s",
@@ -799,11 +808,32 @@
xfree(handle);
return(-1);
}
+ if (rflag) {
+ struct stat stat_buf;
+ if (fstat(local_fd, &stat_buf) == -1) {
+ error("Couldn't stat local file \"%s\": %s",
+ local_path, strerror(errno));
+ close(local_fd);
+ buffer_free(&msg);
+ xfree(handle);
+ return(-1);
+ }
+ offset = stat_buf.st_size;
+ } else {
+ offset = 0;
+ }
/* Read from remote and write to local */
- write_error = read_error = write_errno = num_req = offset = 0;
+ write_error = read_error = write_errno = num_req = 0;
max_req = 1;
- progress_counter = 0;
+ progress_counter = contig_len = offset;
+
+ truncate_len_lower = contig_len;
+ truncate_len_upper = contig_len >> 32;
+ truncate_fd = local_fd;
+ signal(SIGTERM, cleanup_exit);
+ signal(SIGINT, cleanup_exit);
+ signal(SIGHUP, cleanup_exit);
if (showprogress && size != 0)
start_progress_meter(remote_path, size, &progress_counter);
@@ -873,9 +903,52 @@
if (len == req->len) {
TAILQ_REMOVE(&requests, req, tq);
- xfree(req);
num_req--;
+ }
+
+ /* Update contig_len and/or fragments */
+ if (req->offset == contig_len) {
+ struct request *frag, *next_frag;
+ contig_len += len;
+ if (len == req->len)
+ xfree(req);
+ for (frag = TAILQ_FIRST(&fragments);
+ frag && frag->offset == contig_len;
+ frag = next_frag) {
+ next_frag = TAILQ_NEXT(frag, tq);
+ contig_len += frag->len;
+ TAILQ_REMOVE(&fragments, frag, tq);
+ xfree(frag);
+ }
+ truncate_len_lower = contig_len;
+ truncate_len_upper = contig_len >> 32;
} else {
+ struct request *frag, *new_frag;
+ if (len == req->len) {
+ new_frag = req;
+ } else {
+ /* Request will stay in the queue
+ * so we need to duplicate it */
+ new_frag = xmalloc(sizeof(*req));
+ *new_frag = *req;
+ /* Set fragment len to actual len */
+ new_frag->len = len;
+ }
+ /* Insert the fragment at the appropriate
+ * point in the list (ordered by offset) */
+ for (frag = TAILQ_FIRST(&fragments);
+ frag && frag->offset < req->offset;
+ frag = TAILQ_NEXT(frag, tq))
+ ;
+ if (frag)
+ TAILQ_INSERT_BEFORE(frag, new_frag,
+ tq);
+ else
+ TAILQ_INSERT_TAIL(&fragments, new_frag,
+ tq);
+ }
+
+ if (len != req->len) {
/* Resend the request for the missing data */
debug3("Short data block, re-requesting "
"%llu -> %llu (%2d)",
@@ -918,6 +991,24 @@
if (TAILQ_FIRST(&requests) != NULL)
fatal("Transfer complete, but requests still in queue");
+ if (contig_len != progress_counter) {
+ struct request *frag, *next_frag;
+ for (frag = TAILQ_FIRST(&fragments);
+ frag != NULL;
+ frag = next_frag) {
+ next_frag = TAILQ_NEXT(frag, tq);
+ TAILQ_REMOVE(&fragments, frag, tq);
+ xfree(frag);
+ }
+ if (ftruncate(local_fd, contig_len) == -1) {
+ error("Couldn't truncate \"%s\": %s"
+ "; there may be gaps in it",
+ local_path, strerror(errno));
+ }
+ }
+
+ truncate_fd = -1;
+
if (read_error) {
error("Couldn't read from remote file \"%s\" : %s",
remote_path, fx2txt(status));
@@ -1133,3 +1224,13 @@
buffer_free(&msg);
return(status);
}
+
+void
+cleanup_exit(int i)
+{
+ // XXX: error reporting?
+ ftruncate(truncate_fd,
+ ((u_int64_t)truncate_len_upper << 32) | truncate_len_lower);
+
+ _exit(i);
+}
diff -ur openssh-3.8.1p1-8.sarge.4/sftp-client.h openssh-3.8.1p1/sftp-client.h
--- openssh-3.8.1p1-8.sarge.4/sftp-client.h 2004-02-17 06:08:00.000000000 +0000
+++ openssh-3.8.1p1/sftp-client.h 2005-12-29 17:45:43.000000000 +0000
@@ -86,9 +86,10 @@
/*
* Download 'remote_path' to 'local_path'. Preserve permissions and times
- * if 'pflag' is set
+ * if 'pflag' is set. If 'rflag' is set, assume the contents of the local
+ * file (if it exists) are a prefix of the remote file.
*/
-int do_download(struct sftp_conn *, char *, char *, int);
+int do_download(struct sftp_conn *, char *, char *, int, int);
/*
* Upload 'local_path' to 'remote_path'. Preserve permissions and times
signature.asc
Description: This is a digitally signed message part
