It turns out that Linux never reads or writes more than 2147479552
bytes in a single syscall. For writes this is not a problem as
libgfortran already contains a loop around write() to handle short
writes. But for reads we cannot do this, since then read will hang if
we have a short read when reading from the terminal. Also, there are
reports that macOS fails I/O's larger than 2 GB. Thus, to work around
these issues do large reads/writes in chunks.
The testcase from the PR
program largewr
integer(kind=1) :: a(2_8**31+1)
a = 0
a(size(a, kind=8)) = 1
open(10, file="largewr.dat", access="stream", form="unformatted")
write (10) a
close(10)
a(size(a, kind=8)) = 2
open(10, file="largewr.dat", access="stream", form="unformatted")
read (10) a
if (a(size(a, kind=8)) == 1) then
print *, "All is well"
else
print *, "Oh no"
end if
end program largewr
fails on trunk but works with the patch.
Regtested on x86_64-pc-linux-gnu, committed to trunk.
libgfortran/ChangeLog:
2018-01-02 Janne Blomqvist <[email protected]>
PR libgfortran/83649
* io/unix.c (MAX_CHUNK): New define.
(raw_read): For reads larger than MAX_CHUNK, loop.
(raw_write): Write no more than MAX_CHUNK bytes per iteration.
---
libgfortran/io/unix.c | 50 ++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 42 insertions(+), 8 deletions(-)
diff --git a/libgfortran/io/unix.c b/libgfortran/io/unix.c
index a07a3c9..7a982b3 100644
--- a/libgfortran/io/unix.c
+++ b/libgfortran/io/unix.c
@@ -292,18 +292,49 @@ raw_flush (unix_stream *s __attribute__ ((unused)))
return 0;
}
+/* Write/read at most 2 GB - 4k chunks at a time. Linux never reads or
+ writes more than this, and there are reports that macOS fails for
+ larger than 2 GB as well. */
+#define MAX_CHUNK 2147479552
+
static ssize_t
raw_read (unix_stream *s, void *buf, ssize_t nbyte)
{
/* For read we can't do I/O in a loop like raw_write does, because
that will break applications that wait for interactive I/O. We
- still can loop around EINTR, though. */
- while (true)
+ still can loop around EINTR, though. This however causes a
+ problem for large reads which must be chunked, see comment above.
+ So assume that if the size is larger than the chunk size, we're
+ reading from a file and not the terminal. */
+ if (nbyte <= MAX_CHUNK)
{
- ssize_t trans = read (s->fd, buf, nbyte);
- if (trans == -1 && errno == EINTR)
- continue;
- return trans;
+ while (true)
+ {
+ ssize_t trans = read (s->fd, buf, nbyte);
+ if (trans == -1 && errno == EINTR)
+ continue;
+ return trans;
+ }
+ }
+ else
+ {
+ ssize_t bytes_left = nbyte;
+ char *buf_st = buf;
+ while (bytes_left > 0)
+ {
+ ssize_t to_read = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+ ssize_t trans = read (s->fd, buf_st, to_read);
+ if (trans == -1)
+ {
+ if (errno == EINTR)
+ continue;
+ else
+ return trans;
+ }
+ buf_st += trans;
+ bytes_left -= trans;
+ }
+ return nbyte - bytes_left;
}
}
@@ -317,10 +348,13 @@ raw_write (unix_stream *s, const void *buf, ssize_t nbyte)
buf_st = (char *) buf;
/* We must write in a loop since some systems don't restart system
- calls in case of a signal. */
+ calls in case of a signal. Also some systems might fail outright
+ if we try to write more than 2 GB in a single syscall, so chunk
+ up large writes. */
while (bytes_left > 0)
{
- trans = write (s->fd, buf_st, bytes_left);
+ ssize_t to_write = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+ trans = write (s->fd, buf_st, to_write);
if (trans == -1)
{
if (errno == EINTR)
--
2.7.4
