This is an automated email from the ASF dual-hosted git repository. desruisseaux pushed a commit to branch geoapi-4.0 in repository https://gitbox.apache.org/repos/asf/sis.git
commit 4c94f8269e60aad551c00bd4e2130b5206422395 Author: Martin Desruisseaux <martin.desruisse...@geomatys.com> AuthorDate: Wed Oct 25 04:51:21 2023 +0200 First draft of GeoTIFF writer accessible from public API. For now we do that with a `GeoTiffStore.appen(…)` method. --- .../org/apache/sis/storage/geotiff/DataCube.java | 2 +- .../org/apache/sis/storage/geotiff/DataSubset.java | 2 - .../org/apache/sis/storage/geotiff/GeoTIFF.java | 4 +- .../apache/sis/storage/geotiff/GeoTiffStore.java | 155 ++++++++++++++++++--- .../apache/sis/storage/geotiff/NativeMetadata.java | 6 +- .../org/apache/sis/storage/geotiff/Reader.java | 76 +++++++--- .../sis/storage/geotiff/ReformattedImage.java | 13 ++ .../sis/storage/geotiff/TileMatrixWriter.java | 16 ++- .../org/apache/sis/storage/geotiff/Writer.java | 116 +++++++++++---- .../org/apache/sis/storage/geotiff/WriterTest.java | 7 +- .../sis/storage/ReadOnlyStorageException.java | 1 + ...ception.java => WriteOnlyStorageException.java} | 30 ++-- .../main/org/apache/sis/util/resources/Errors.java | 5 + .../apache/sis/util/resources/Errors.properties | 1 + .../apache/sis/util/resources/Errors_fr.properties | 1 + 15 files changed, 333 insertions(+), 102 deletions(-) diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataCube.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataCube.java index 007de175b1..eff8a80e3e 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataCube.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataCube.java @@ -246,7 +246,7 @@ abstract class DataCube extends TiledGridResource implements ResourceOnFileSyste coverage = preload(coverage); } } catch (RuntimeException e) { - throw canNotRead(reader.input.filename, domain, e); + throw canNotRead(filename(), domain, e); } logReadOperation(reader.store.path, coverage.getGridGeometry(), startTime); return coverage; diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java index 673946ef07..def938762b 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java @@ -46,7 +46,6 @@ import org.apache.sis.storage.geotiff.internal.Resources; import org.apache.sis.util.resources.Errors; import org.apache.sis.math.Vector; -import static java.lang.Math.addExact; import static java.lang.Math.subtractExact; import static java.lang.Math.multiplyExact; import static java.lang.Math.multiplyFull; @@ -386,7 +385,6 @@ class DataSubset extends TiledGridCoverage implements Localized { boolean isEmpty = true; for (int b=0; b<offsets.length; b++) { isEmpty &= (byteCounts[b] == 0); - offsets[b] = addExact(offsets[b], source.reader.origin); } /* * If the length if zero for all bands, the GDAL "sparse files" convention said diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java index 11d3caba35..493acb1bc5 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java @@ -79,8 +79,10 @@ abstract class GeoTIFF implements Closeable { /** * {@return the options (BigTIFF, COG…) used by this reader or writer}. + * + * @see GeoTiffStore#getOptions() */ - abstract Set<GeoTiffOption> getOptions(); + public abstract Set<GeoTiffOption> getOptions(); /** * Returns the resources to use for formatting error messages. diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java index 4b8fa1f835..e6ef201639 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java @@ -27,6 +27,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.awt.image.RenderedImage; import org.opengis.util.NameSpace; import org.opengis.util.NameFactory; import org.opengis.util.GenericName; @@ -41,23 +42,28 @@ import org.apache.sis.storage.DataStoreProvider; import org.apache.sis.storage.StorageConnector; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.DataStoreClosedException; +import org.apache.sis.storage.ReadOnlyStorageException; +import org.apache.sis.storage.WriteOnlyStorageException; import org.apache.sis.storage.IllegalNameException; +import org.apache.sis.storage.base.MetadataBuilder; +import org.apache.sis.storage.base.StoreUtilities; +import org.apache.sis.storage.base.URIDataStore; import org.apache.sis.storage.event.StoreEvent; import org.apache.sis.storage.event.StoreListener; import org.apache.sis.storage.event.StoreListeners; import org.apache.sis.storage.event.WarningEvent; +import org.apache.sis.storage.geotiff.spi.SchemaModifier; import org.apache.sis.io.stream.ChannelDataInput; import org.apache.sis.io.stream.ChannelDataOutput; import org.apache.sis.io.stream.IOUtilities; -import org.apache.sis.storage.base.MetadataBuilder; -import org.apache.sis.storage.base.StoreUtilities; -import org.apache.sis.storage.base.URIDataStore; -import org.apache.sis.storage.geotiff.spi.SchemaModifier; +import org.apache.sis.metadata.iso.DefaultMetadata; +import org.apache.sis.metadata.sql.MetadataStoreException; +import org.apache.sis.coverage.grid.GridCoverage; +import org.apache.sis.coverage.grid.GridGeometry; +import org.apache.sis.coverage.SubspaceNotSpecifiedException; import org.apache.sis.util.ArgumentChecks; import org.apache.sis.util.internal.Constants; import org.apache.sis.util.internal.ListOfUnknownSize; -import org.apache.sis.metadata.iso.DefaultMetadata; -import org.apache.sis.metadata.sql.MetadataStoreException; import org.apache.sis.util.collection.BackingStoreException; import org.apache.sis.util.collection.TreeTable; import org.apache.sis.util.iso.DefaultNameSpace; @@ -158,7 +164,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { * * @see #components() */ - private List<GridCoverageResource> components; + private Components components; /** * Whether this {@code GeotiffStore} will be hidden. If {@code true}, then some metadata that would @@ -230,9 +236,6 @@ public class GeoTiffStore extends DataStore implements Aggregate { } else { ChannelDataInput input = connector.commit(ChannelDataInput.class, Constants.GEOTIFF); reader = new Reader(this, input); - if (getClass() == GeoTiffStore.class) { - listeners.useReadOnlyEvents(); - } } } catch (IOException e) { throw new DataStoreException(e); @@ -247,8 +250,8 @@ public class GeoTiffStore extends DataStore implements Aggregate { * @since 1.5 */ public Set<GeoTiffOption> getOptions() { - if (writer != null) return writer.getOptions(); - if (reader != null) return reader.getOptions(); + final Writer w = writer; if (w != null) return w.getOptions(); + final Reader r = reader; if (r != null) return r.getOptions(); return Set.of(); } @@ -257,6 +260,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { * This method must be invoked inside a block synchronized on {@code this}. */ final NameSpace namespace() { + @SuppressWarnings("LocalVariableHidesMemberVariable") final Reader reader = this.reader; if (!isNamespaceSet && reader != null) { final NameFactory f = reader.nameFactory; @@ -299,10 +303,13 @@ public class GeoTiffStore extends DataStore implements Aggregate { @Override public Optional<ParameterValueGroup> getOpenParameters() { final ParameterValueGroup param = URIDataStore.parameters(provider, location); - if (param != null && writer != null) { - final Set<GeoTiffOption> options = writer.getOptions(); - if (!options.isEmpty()) { - param.parameter(GeoTiffStoreProvider.OPTIONS).setValue(options.toArray(GeoTiffOption[]::new)); + if (param != null) { + final Writer w = writer; + if (w != null) { + final Set<GeoTiffOption> options = w.getOptions(); + if (!options.isEmpty()) { + param.parameter(GeoTiffStoreProvider.OPTIONS).setValue(options.toArray(GeoTiffOption[]::new)); + } } } return Optional.ofNullable(param); @@ -320,6 +327,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { */ @Override public Optional<GenericName> getIdentifier() throws DataStoreException { + @SuppressWarnings("LocalVariableHidesMemberVariable") final NameSpace namespace; synchronized (this) { namespace = namespace(); @@ -352,6 +360,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { @Override public synchronized Metadata getMetadata() throws DataStoreException { if (metadata == null) { + @SuppressWarnings("LocalVariableHidesMemberVariable") final Reader reader = reader(); final MetadataBuilder builder = new MetadataBuilder(); setFormatInfo(builder); @@ -419,33 +428,66 @@ public class GeoTiffStore extends DataStore implements Aggregate { * This method wraps the exception with a {@literal "Cannot read <filename>"} message. */ final DataStoreException errorIO(final IOException e) { - return new DataStoreException(errors().getString(Errors.Keys.CanNotRead_1, reader.input.filename), e); + return new DataStoreException(errors().getString(Errors.Keys.CanNotRead_1, getDisplayName()), e); + } + + /** + * Returns a localized error message saying that this data store has been opened in read-only or write-only mode. + * + * @param mode 0 for read-only, or 1 for write-only. + * @return localized error message. + */ + final String readOrWriteOnly(final int mode) { + return errors().getString(Errors.Keys.OpenedReadOrWriteOnly_2, mode, getDisplayName()); } /** * Returns the reader if it is not closed, or throws an exception otherwise. * + * @return the reader, potentially created when first needed. + * @throws WriteOnlyStorageException if the channel is write-only. + * * @see #close() */ private Reader reader() throws DataStoreException { assert Thread.holdsLock(this); final Reader r = reader; if (r == null) { + if (writer != null) { + throw new WriteOnlyStorageException(readOrWriteOnly(1)); + } throw new DataStoreClosedException(getLocale(), Constants.GEOTIFF, StandardOpenOption.READ); } return r; } /** - * Returns the writer if it is not closed, or throws an exception otherwise. + * Returns the writer if it can be created and is not closed, or throws an exception otherwise. + * If there is no writer but a reader exists, then a writer is created for writing past the last image. + * After the write operation has been completed, it is caller responsibility to invoke the following code: + * + * {@snippet lang="java": + * writer.synchronize(reader, false); + * // Write the image + * writer.flush(); + * writer.synchronize(reader, true); + * } + * + * @return the writer, potentially created when first needed. + * @throws ReadOnlyStorageException if this data store is read-only. * * @see #close() + * @see Writer#synchronize(Reader, boolean) */ - final Writer writer() throws DataStoreException { + private Writer writer() throws DataStoreException, IOException { assert Thread.holdsLock(this); - final Writer w = writer; + Writer w = writer; if (w == null) { - throw new DataStoreClosedException(getLocale(), Constants.GEOTIFF, StandardOpenOption.WRITE); + final Reader r = reader; + if (r == null) { + throw new DataStoreClosedException(getLocale(), Constants.GEOTIFF, StandardOpenOption.WRITE); + } + writer = w = new Writer(r); } return w; } @@ -480,6 +522,10 @@ public class GeoTiffStore extends DataStore implements Aggregate { /** The collection size, cached when first computed. */ private int size = -1; + /** Creates a new list of components. */ + Components() { + } + /** Returns the size or -1 if not yet known. */ @Override protected int sizeIfKnown() { synchronized (GeoTiffStore.this) { @@ -497,6 +543,15 @@ public class GeoTiffStore extends DataStore implements Aggregate { } } + /** Increments the size by the given amount of images. */ + final void incrementSize(final int n) { + synchronized (GeoTiffStore.this) { + if (size >= 0) { + size += n; + } + } + } + /** Returns whether the given index is valid. */ @Override protected boolean exists(final int index) { return (index >= 0) && getImageFileDirectory(index) != null; @@ -562,6 +617,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { * @throws IllegalNameException if the argument use an invalid namespace or if the tip is not an integer. */ private int parseImageIndex(String sequence) throws IllegalNameException { + @SuppressWarnings("LocalVariableHidesMemberVariable") final NameSpace namespace = namespace(); final String separator = DefaultNameSpace.getSeparator(namespace, false); final int s = sequence.lastIndexOf(separator); @@ -581,6 +637,63 @@ public class GeoTiffStore extends DataStore implements Aggregate { } } + /** + * Encodes the given image in the GeoTIFF file. + * The image is appended after any existing images in the GeoTIFF file. + * This method does not handle pyramids such as Cloud Optimized GeoTIFF (COG). + * + * @param image the image to encode. + * @param grid mapping from pixel coordinates to "real world" coordinates, or {@code null} if none. + * @param metadata title, author and other information, or {@code null} if none. + * @throws ReadOnlyStorageException if this data store is read-only. + * @throws DataStoreException if the given {@code image} has a property which is not supported by this writer, + * or if an error occurred while writing to the output stream. + * + * @since 1.5 + */ + public synchronized void append(final RenderedImage image, final GridGeometry grid, final Metadata metadata) + throws DataStoreException + { + try { + @SuppressWarnings("LocalVariableHidesMemberVariable") final Writer writer = writer(); + @SuppressWarnings("LocalVariableHidesMemberVariable") final Reader reader = this.reader; + writer.synchronize(reader, false); + final long offsetIFD; + try { + offsetIFD = writer.append(image, grid, metadata); + writer.flush(); + } finally { + writer.synchronize(reader, true); + } + if (reader != null) { + reader.offsetOfWrittenIFD(offsetIFD); + } + } catch (IOException e) { + throw new DataStoreException(errors().getString(Errors.Keys.CanNotWriteFile_2, Constants.GEOTIFF, getDisplayName()), e); + } + if (components != null) { + components.incrementSize(1); + } + } + + /** + * Adds a new grid coverage in the GeoTIFF file. + * The coverage is appended after any existing images in the GeoTIFF file. + * This method does not handle pyramids such as Cloud Optimized GeoTIFF (COG). + * + * @param coverage the grid coverage to encode. + * @param metadata title, author and other information, or {@code null} if none. + * @throws SubspaceNotSpecifiedException if the given grid coverage is not a two-dimensional slice. + * @throws ReadOnlyStorageException if this data store is read-only. + * @throws DataStoreException if the given {@code image} has a property which is not supported by this writer, + * or if an error occurred while writing to the output stream. + * + * @since 1.5 + */ + public void append(final GridCoverage coverage, final Metadata metadata) throws DataStoreException { + append(coverage.render(null), coverage.getGridGeometry(), metadata); + } + /** * Registers a listener to notify when the specified kind of event occurs in this data store. * The current implementation of this data store can emit only {@link WarningEvent}s; diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/NativeMetadata.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/NativeMetadata.java index e16858cd3b..8bb2de1fed 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/NativeMetadata.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/NativeMetadata.java @@ -109,7 +109,7 @@ final class NativeMetadata extends GeoKeysLoader { root.setValue(NAME, "TIFF"); input.mark(); try { - input.seek(addExact(reader.origin, isClassic ? 2*Short.BYTES : 4*Short.BYTES)); + input.seek(isClassic ? 2*Short.BYTES : 4*Short.BYTES); final Set<Long> doneIFD = new HashSet<>(); long nextIFD; /* @@ -124,7 +124,7 @@ final class NativeMetadata extends GeoKeysLoader { } final TreeTable.Node image = root.newChild(); image.setValue(NAME, vocabulary.getString(Vocabulary.Keys.Image_1, imageNumber)); - input.seek(Math.addExact(reader.origin, nextIFD)); + input.seek(nextIFD); for (long remaining = readInt(true); --remaining >= 0;) { final short tag = (short) input.readUnsignedShort(); final Type type = Type.valueOf(input.readShort()); // May be null. @@ -147,7 +147,7 @@ final class NativeMetadata extends GeoKeysLoader { if (visible) { if (size > offsetSize) { final long offset = readInt(false); - input.seek(Math.addExact(reader.origin, offset)); + input.seek(offset); } /* * Some tags need to be handle in a special way. The main cases are GeoTIFF keys. diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java index add3fce9dc..7a70c4a1cc 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java @@ -25,10 +25,11 @@ import java.util.Iterator; import java.io.IOException; import java.nio.ByteOrder; import org.opengis.util.NameFactory; +import org.apache.sis.io.stream.ChannelDataInput; import org.apache.sis.storage.GridCoverageResource; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.DataStoreContentException; -import org.apache.sis.io.stream.ChannelDataInput; +import org.apache.sis.storage.InternalDataStoreException; import org.apache.sis.storage.geotiff.internal.Resources; import org.apache.sis.util.iso.DefaultNameFactory; import org.apache.sis.util.resources.Errors; @@ -56,11 +57,6 @@ final class Reader extends GeoTIFF { */ final ChannelDataInput input; - /** - * Stream position of the first byte of the GeoTIFF file. This is usually zero. - */ - final long origin; - /** * A multiplication factor for the size of pointers, expressed as a power of 2. * The pointer size in bytes is given by {@code Integer.BYTES << pointerExpansion}. @@ -86,13 +82,21 @@ final class Reader extends GeoTIFF { private ImageFileDirectory lastIFD; /** - * Offset (relative to the beginning of the TIFF file) of the next Image File Directory (IFD) - * to read, or 0 if we have finished to read all of them. + * Offset (relative to the beginning of the TIFF file) of the next Image File Directory (IFD) to read. + * This is valid only if {@link #endOfFile} is {@code false}, otherwise this value is actually the offset + * where to write a new IFD value if a new image is appended by {@link Writer}. * * @see #readNextImageOffset() */ private long nextIFD; + /** + * Whether all existing Image File Directories (IFD) have been read. If {@code true}, the {@link #nextIFD} + * is not the offset of the next IFD, but rather the offset where to <em>write</em> the next IFD if a new + * image is appended by {@link Writer}. + */ + private boolean endOfFile; + /** * Offsets of all <cite>Image File Directory</cite> (IFD) that have been read so far. * This field is used only as a protection against infinite recursivity, by preventing @@ -141,7 +145,6 @@ final class Reader extends GeoTIFF { Reader(final GeoTiffStore store, final ChannelDataInput input) throws IOException, DataStoreException { super(store); this.input = input; - this.origin = input.getStreamPosition(); this.doneIFD = new HashSet<>(); this.nameFactory = DefaultNameFactory.provider(); /* @@ -149,6 +152,7 @@ final class Reader extends GeoTIFF { * Those characters identify the byte order. Note that we do not need to care * about the byte order for this flag since the two bytes shall have the same value. */ + input.relocateOrigin(); final short order = input.readShort(); final boolean isBigEndian = (order == BIG_ENDIAN); if (isBigEndian || order == LITTLE_ENDIAN) { @@ -193,7 +197,7 @@ final class Reader extends GeoTIFF { * {@return the options (BigTIFF, COG…) used by this reader}. */ @Override - final Set<GeoTiffOption> getOptions() { + public final Set<GeoTiffOption> getOptions() { return (intSizeExpansion != 0) ? Set.of(GeoTiffOption.BIG_TIFF) : Set.of(); } @@ -202,8 +206,12 @@ final class Reader extends GeoTIFF { * and makes sure that it will not cause an infinite loop. */ private void readNextImageOffset() throws IOException, DataStoreException { + final long p = input.getStreamPosition(); nextIFD = readUnsignedInt(); - if (!doneIFD.add(nextIFD)) { + endOfFile = (nextIFD == 0); + if (endOfFile) { + nextIFD = p; + } else if (!doneIFD.add(nextIFD)) { throw new DataStoreContentException(resources().getString( Resources.Keys.CircularImageReference_1, input.filename)); } @@ -244,7 +252,7 @@ final class Reader extends GeoTIFF { } /** - * Returns the next <cite>Image File Directory</cite> (IFD), or {@code null} if we reached the last IFD. + * Returns the next Image File Directory (IFD), or {@code null} if we reached the last IFD. * The IFD consists of a 2 (classical) or 8 (BigTiff)-bytes count of the number of directory entries, * followed by a sequence of 12-byte field entries, followed by a pointer to the next IFD (or 0 if none). * @@ -254,12 +262,12 @@ final class Reader extends GeoTIFF { * * @see #getImage(int) */ - private ImageFileDirectory getImageFileDirectory(final int index) throws IOException, DataStoreException { - if (nextIFD == 0) { + private ImageFileDirectory readNextIFD(final int index) throws IOException, DataStoreException { + if (endOfFile || nextIFD == 0) { return null; } resolveDeferredEntries(null, nextIFD); - input.seek(Math.addExact(origin, nextIFD)); + input.seek(nextIFD); nextIFD = 0; // Prevent trying other IFD if we fail to read this one. /* * Design note: we parse the Image File Directory entry now because even if we were @@ -354,7 +362,7 @@ final class Reader extends GeoTIFF { deferredEntries.sort(null); // Sequential order in input stream. deferredNeedsSort = false; } - final long ignoreBefore = input.getStreamPosition() - origin; // Avoid seeking back, unless we need to. + final long ignoreBefore = input.getStreamPosition(); // Avoid seeking back, unless we need to. DeferredEntry stopAfter = null; // Avoid reading more entries than needed. if (dir != null) { for (final Iterator<DeferredEntry> it = deferredEntries.descendingIterator(); it.hasNext();) { @@ -365,7 +373,7 @@ final class Reader extends GeoTIFF { for (final Iterator<DeferredEntry> it = deferredEntries.iterator(); it.hasNext();) { final DeferredEntry entry = it.next(); if (entry.owner == dir || (entry.offset >= ignoreBefore && entry.offset <= ignoreAfter)) { - input.seek(Math.addExact(origin, entry.offset)); + input.seek(entry.offset); Object error; try { error = entry.owner.addEntry(entry.tag, entry.type, entry.count); @@ -395,12 +403,12 @@ final class Reader extends GeoTIFF { * @return the pyramid if we found it, or {@code null} if there is no more pyramid at the given index. * @throws ArithmeticException if the pointer to a next IFD is too far. */ - final GridCoverageResource getImage(final int index) throws IOException, DataStoreException { + public final GridCoverageResource getImage(final int index) throws IOException, DataStoreException { while (index >= images.size()) { int imageIndex = images.size(); ImageFileDirectory fullResolution = lastIFD; if (fullResolution == null) { - fullResolution = getImageFileDirectory(imageIndex); + fullResolution = readNextIFD(imageIndex); if (fullResolution == null) { return null; } @@ -409,7 +417,7 @@ final class Reader extends GeoTIFF { imageIndex++; // In case next image is full-resolution. ImageFileDirectory image; final List<ImageFileDirectory> overviews = new ArrayList<>(); - while ((image = getImageFileDirectory(imageIndex)) != null) { + while ((image = readNextIFD(imageIndex)) != null) { if (image.isReducedResolution()) { overviews.add(image); } else { @@ -459,6 +467,34 @@ final class Reader extends GeoTIFF { store.listeners().warning(errors().getString(key, args), exception); } + /** + * Returns the stream position where to write the IFD for a new image to append. + * Invoking this method forces this reader to read all existing IFDs. + * The {@link #input} position is undetermined, not necessarily end of file. + * The value at the returned offset is zero. + * + * @return offset where to write a new IFD. + */ + final long offsetOfWritableIFD() throws IOException, DataStoreException { + if (getImage(Integer.MAX_VALUE) == null && endOfFile) { + return nextIFD; + } + throw new InternalDataStoreException(); // Should never happen. + } + + /** + * Sets the offsets of the image that has been written. + * + * @param position offset of the new Image File Directory (IFD). + */ + final void offsetOfWrittenIFD(final long position) throws IOException, DataStoreException { + if (getImage(Integer.MAX_VALUE) == null && endOfFile) { + nextIFD = position; + endOfFile = false; + } + throw new InternalDataStoreException(); // Should never happen. + } + /** * Closes this reader. * This method can be invoked asynchronously for interrupting a long reading process. diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java index 9738f7da1f..9bb0cf275f 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java @@ -20,6 +20,7 @@ import java.awt.color.ColorSpace; import java.awt.image.ColorModel; import java.awt.image.IndexColorModel; import java.awt.image.RenderedImage; +import java.awt.image.SampleModel; import static javax.imageio.plugins.tiff.BaselineTIFFTagSet.*; import org.apache.sis.util.ArraysExt; import org.apache.sis.math.Statistics; @@ -103,6 +104,18 @@ found: if (property instanceof Statistics[]) { return new double[2][]; } + /** + * Returns the TIFF sample format. + * + * @return One of {@code SAMPLE_FORMAT_*} constants. + */ + final int getSampleFormat() { + final SampleModel sm = visibleBands.getSampleModel(); + if (ImageUtilities.isUnsignedType(sm)) return SAMPLE_FORMAT_UNSIGNED_INTEGER; + if (ImageUtilities.isIntegerType(sm)) return SAMPLE_FORMAT_SIGNED_INTEGER; + return SAMPLE_FORMAT_FLOATING_POINT; + } + /** * Returns the TIFF color interpretation. * diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java index 64e3406907..9843019ba6 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java @@ -45,6 +45,11 @@ import org.apache.sis.io.stream.HyperRectangleWriter; * @author Martin Desruisseaux (Geomatys) */ final class TileMatrixWriter { + /** + * Offset in {@link ChannelDataOutput} where the IFD starts. + */ + final long offsetIFD; + /** * The images to write. */ @@ -94,16 +99,19 @@ final class TileMatrixWriter { * Creates a new set of information about tiles to write. * * @param image the image to write. - * @param dataType the type of sample values. * @param numBands the number of bands. * @param bitsPerSample number of bits per sample to write. * @param isPlanar whether the planar configuration is to store bands in separated planes. + * @param offsetIFD offset in {@link ChannelDataOutput} where the IFD starts. */ - TileMatrixWriter(final RenderedImage image, final DataType type, final int numPlanes, final int[] bitsPerSample) { + TileMatrixWriter(final RenderedImage image, final int numPlanes, final int[] bitsPerSample, + final long offsetIFD) + { final int pixelSize, numArrays; + this.offsetIFD = offsetIFD; this.numPlanes = numPlanes; - this.type = type; - this.image = image; + this.image = image; + type = DataType.forBands(image); tileWidth = image.getTileWidth(); tileHeight = image.getTileHeight(); pixelSize = (bitsPerSample != null) ? Numerics.ceilDiv(Arrays.stream(bitsPerSample).sum(), Byte.SIZE) : 1; diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java index 4331f9a5d4..91b733ceeb 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java @@ -35,11 +35,11 @@ import javax.imageio.plugins.tiff.TIFFTag; import javax.measure.IncommensurableException; import org.opengis.util.FactoryException; import org.opengis.metadata.Metadata; -import org.apache.sis.image.DataType; import org.apache.sis.image.ImageProcessor; import org.apache.sis.coverage.grid.GridGeometry; import org.apache.sis.storage.DataStoreException; import org.apache.sis.storage.DataStoreReferencingException; +import org.apache.sis.storage.ReadOnlyStorageException; import org.apache.sis.storage.base.MetadataFetcher; import org.apache.sis.io.stream.ChannelDataOutput; import org.apache.sis.io.stream.UpdatableWrite; @@ -129,8 +129,11 @@ final class Writer extends GeoTIFF implements Flushable { private final boolean isBigTIFF; /** - * Offset where to write the next image, or an offset value of 0 if none. - * Shall be one of the elements in the {@link #deferredWrites} queue. + * Offset where to write the next image, or {@code null} if writing a mandatory image (the first one). + * If null, the IFD offset is assumed already written and the {@linkplain #output} already at that position. + * Otherwise the value at the specified offset should be zero and will be updated if a new image is appended. + * + * @see #seekToNextImage() */ private UpdatableWrite<?> nextIFD; @@ -138,13 +141,13 @@ final class Writer extends GeoTIFF implements Flushable { * All values that couldn't be written immediately. * Values shall be sorted in increasing order of stream position. */ - private final Deque<UpdatableWrite<?>> deferredWrites; + private final Deque<UpdatableWrite<?>> deferredWrites = new ArrayDeque<>(); /** * Write operations for tag having data too large for fitting inside a IFD tag entry. * The writing of those data need to be delayed somewhere after the sequence of entries. */ - private final Queue<TagValueWriter> largeTagData; + private final Queue<TagValueWriter> largeTagData = new ArrayDeque<>(); /** * Number of TIFF tag entries in the image being written. @@ -164,10 +167,8 @@ final class Writer extends GeoTIFF implements Flushable { throws IOException, DataStoreException { super(store); - this.output = output; - isBigTIFF = ArraysExt.contains(options, GeoTiffOption.BIG_TIFF); - deferredWrites = new ArrayDeque<>(); - largeTagData = new ArrayDeque<>(); + this.output = output; + isBigTIFF = ArraysExt.contains(options, GeoTiffOption.BIG_TIFF); /* * Write the TIFF file header before first IFD. Stream position matter and must start at zero. * Note that it does not necessarily mean that the stream has no bytes before current position. @@ -184,11 +185,52 @@ final class Writer extends GeoTIFF implements Flushable { } } + /** + * Creates a new writer which will append images a the end of an existing file. + * It is caller's responsibility to keep reader and writer positions consistent. + * This is done by invoking {@link #synchronize(Reader, boolean)} before and after + * write operations. + * + * @param reader the reader of the existing GeoTIFF file. + * @throws ReadOnlyStorageException if the channel is read-only. + * @throws DataStoreException if the writer cannot be created. + * @throws IOException if an I/O error occurred. + */ + Writer(final Reader reader) throws IOException, DataStoreException { + super(reader.store); + isBigTIFF = (reader.intSizeExpansion != 0); + try { + output = new ChannelDataOutput(reader.input); + } catch (ClassCastException e) { + throw new ReadOnlyStorageException(store.readOrWriteOnly(0), e); + } + Class<? extends Number> type = isBigTIFF ? Long.class : Integer.class; + nextIFD = UpdatableWrite.ofZeroAt(reader.offsetOfWritableIFD(), type); + } + + /** + * Ensures that the reader and writer positions are consistent. It is caller's responsibility to invoke + * {@link #flush()} before to invoke {@code synchronize(reader, true)}, unless the write operation failed. + * In the latter case, the caller should cancel the write operation if possible. + * + * @param reader the reader, or {@code null} if none. + * @param finish {@code false} if invoked before write operations, or {@code true} if invoked after. + */ + final void synchronize(final Reader reader, final boolean finish) throws IOException { + if (reader != null) { + if (finish) { + output.yield(reader.input); + } else { + reader.input.yield(output); + } + } + } + /** * {@return the options (BigTIFF, COG…) used by this writer}. */ @Override - final Set<GeoTiffOption> getOptions() { + public final Set<GeoTiffOption> getOptions() { return isBigTIFF ? Set.of(GeoTiffOption.BIG_TIFF) : Set.of(); } @@ -212,11 +254,12 @@ final class Writer extends GeoTIFF implements Flushable { * @param image the image to encode. * @param grid mapping from pixel coordinates to "real world" coordinates, or {@code null} if none. * @param metadata title, author and other information, or {@code null} if none. + * @return offset if {@link #output} where the Image File Directory (IFD) starts. * @throws IOException if an error occurred while writing to the output. * @throws DataStoreException if the given {@code image} has a property * which is not supported by TIFF specification or by this writer. */ - final void append(final RenderedImage image, final GridGeometry grid, final Metadata metadata) + public final long append(final RenderedImage image, final GridGeometry grid, final Metadata metadata) throws IOException, DataStoreException { final TileMatrixWriter tiles; @@ -228,10 +271,32 @@ final class Writer extends GeoTIFF implements Flushable { tiles.writeRasters(output); wordAlign(output); tiles.writeOffsetsAndLengths(output); + return tiles.offsetIFD; } /** - * Writes the Image File Directory (IFD) of the given image. + * Sets the {@linkplain #output} position to where to write the next image. + * + * @return offset where the image IFD will start. This is the {@link #output} position. + * + * @todo Current version append the new image at the end of file. A future version could perform a more extensive + * search for free space in the middle of the file. It could be useful when images have been deleted. + */ + private long seekToNextImage() throws IOException { + if (nextIFD == null) { + // `output` is already at the right position. + return output.getStreamPosition(); + } + final long position = output.length(); + nextIFD.setAsLong(position); + writeOrQueue(nextIFD); + output.seek(position); + nextIFD = null; + return position; + } + + /** + * Writes the Image File Directory (IFD) of the given image at the current {@link #output} position. * This method does not write the pixel values. Those values must be written by the caller. * This separation makes possible to write directories in any order compared to pixel data. * @@ -259,20 +324,11 @@ final class Writer extends GeoTIFF implements Flushable { if (colorInterpretation == PHOTOMETRIC_INTERPRETATION_PALETTE_COLOR) { numberOfTags++; } - final SampleModel sm = image.visibleBands.getSampleModel(); + final SampleModel sm = image.visibleBands.getSampleModel(); + final int sampleFormat = image.getSampleFormat(); final int[] bitsPerSample = sm.getSampleSize(); - final int numBands = sm.getNumBands(); - final int sampleFormat; - final DataType dataType = DataType.forDataBufferType(sm.getDataType()); - if (dataType.isUnsigned()) { - sampleFormat = SAMPLE_FORMAT_UNSIGNED_INTEGER; - } else if (dataType.isInteger()) { - sampleFormat = SAMPLE_FORMAT_SIGNED_INTEGER; - } else { - sampleFormat = SAMPLE_FORMAT_FLOATING_POINT; - } - final int numPlanes; - final int planarConfiguration; + final int numBands = sm.getNumBands(); + final int numPlanes, planarConfiguration; if (sm instanceof BandedSampleModel) { planarConfiguration = PLANAR_CONFIGURATION_PLANAR; numPlanes = numBands; @@ -311,6 +367,7 @@ final class Writer extends GeoTIFF implements Flushable { */ output.flush(); // Makes room in the buffer for increasing our ability to modify past values. largeTagData.clear(); + final long offsetIFD = seekToNextImage(); final UpdatableWrite<?> tagCountWriter = isBigTIFF ? UpdatableWrite.of(output, (long) numberOfTags) : UpdatableWrite.of(output, (short) numberOfTags); @@ -338,7 +395,7 @@ final class Writer extends GeoTIFF implements Flushable { if (colorInterpretation == PHOTOMETRIC_INTERPRETATION_PALETTE_COLOR) { writeColorPalette((IndexColorModel) image.visibleBands.getColorModel(), 1L << bitsPerSample[0]); } - final var tiling = new TileMatrixWriter(image.visibleBands, dataType, numPlanes, bitsPerSample); + final var tiling = new TileMatrixWriter(image.visibleBands, numPlanes, bitsPerSample, offsetIFD); writeTag((short) TAG_TILE_WIDTH, (short) TIFFTag.TIFF_LONG, tiling.tileWidth); writeTag((short) TAG_TILE_LENGTH, (short) TIFFTag.TIFF_LONG, tiling.tileHeight); tiling.offsetsTag = writeTag((short) TAG_TILE_OFFSETS, tiling.offsets); @@ -438,7 +495,9 @@ final class Writer extends GeoTIFF implements Flushable { */ private int writeTagHeader(final short tag, final short type, final long count) throws IOException { numberOfTags++; - output.ensureBufferAccepts(2*Short.BYTES + 2*Long.BYTES); + output.ensureBufferAccepts(isBigTIFF + ? (2*Short.BYTES + 2*Long.BYTES) + : (2*Short.BYTES + 2*Integer.BYTES)); final ByteBuffer buffer = output.buffer; buffer.putShort(tag); buffer.putShort(type); @@ -734,7 +793,8 @@ final class Writer extends GeoTIFF implements Flushable { * @throws IOException if an error occurred while writing deferred data. */ private void flushDeferredWrites() throws IOException { - for (final UpdatableWrite<?> change : deferredWrites) { + UpdatableWrite<?> change; + while ((change = deferredWrites.pollFirst()) != null) { change.update(output); } } diff --git a/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java b/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java index 3fce1da04a..8481ec8d0e 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java @@ -166,13 +166,8 @@ public final class WriterTest extends TestCase { * @throws IOException should never happen since the tests are writing in memory. * @throws DataStoreException if the image is incompatible with writer capability. */ - @SuppressWarnings("SynchronizeOnNonFinalField") private void writeImage() throws IOException, DataStoreException { - synchronized (store) { - final Writer writer = store.writer(); - writer.append(image, gridGeometry, null); - writer.flush(); - } + store.append(image, gridGeometry, null); data.clear().limit(Math.toIntExact(output.size())); } diff --git a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java index f393665e1f..eda978268c 100644 --- a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java +++ b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java @@ -30,6 +30,7 @@ package org.apache.sis.storage; * @version 0.8 * @since 0.8 * + * @see WriteOnlyStorageException * @see ForwardOnlyStorageException */ public class ReadOnlyStorageException extends DataStoreException { diff --git a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/WriteOnlyStorageException.java similarity index 62% copy from endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java copy to endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/WriteOnlyStorageException.java index f393665e1f..50d0bcbc5a 100644 --- a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/ReadOnlyStorageException.java +++ b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/WriteOnlyStorageException.java @@ -18,30 +18,28 @@ package org.apache.sis.storage; /** - * Thrown when a {@code DataStore} cannot perform a write operation. - * If a data store does not support any write operation, then it should not implement - * {@link WritableAggregate} or {@link WritableFeatureSet} interface. - * But in some situations, a data store may implement a {@code Writable*} interface - * and nevertheless be unable to perform a write operation, for example because the - * underlying {@link java.nio.channels.Channel} is read-only or part of the file is - * locked by another process. + * Thrown when a {@code DataStore} cannot perform a read operation. + * This exception may happen if the {@link java.nio.channels.Channel} used by a data store + * implements {@link java.nio.channels.WritableByteChannel} only, without implementing also + * {@link java.nio.channels.ReadableByteChannel}. * - * @author Johann Sorel (Geomatys) - * @version 0.8 - * @since 0.8 + * @author Martin Desruisseaux (Geomatys) + * @version 1.5 + * @since 1.5 * + * @see ReadOnlyStorageException * @see ForwardOnlyStorageException */ -public class ReadOnlyStorageException extends DataStoreException { +public class WriteOnlyStorageException extends DataStoreException { /** * For cross-version compatibility. */ - private static final long serialVersionUID = 5710116172772560023L; + private static final long serialVersionUID = -5809491968506721317L; /** * Creates an exception with no cause and no details message. */ - public ReadOnlyStorageException() { + public WriteOnlyStorageException() { } /** @@ -49,7 +47,7 @@ public class ReadOnlyStorageException extends DataStoreException { * * @param message the detail message. */ - public ReadOnlyStorageException(final String message) { + public WriteOnlyStorageException(final String message) { super(message); } @@ -58,7 +56,7 @@ public class ReadOnlyStorageException extends DataStoreException { * * @param cause the cause for this exception. */ - public ReadOnlyStorageException(final Throwable cause) { + public WriteOnlyStorageException(final Throwable cause) { super(cause); } @@ -68,7 +66,7 @@ public class ReadOnlyStorageException extends DataStoreException { * @param message the detail message. * @param cause the cause for this exception. */ - public ReadOnlyStorageException(final String message, final Throwable cause) { + public WriteOnlyStorageException(final String message, final Throwable cause) { super(message, cause); } } diff --git a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.java b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.java index 2a8f5fe0aa..a13243d6d1 100644 --- a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.java +++ b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.java @@ -789,6 +789,11 @@ public class Errors extends IndexedResourceBundle { */ public static final short OddArrayLength_1 = 118; + /** + * “{1}” is opened in {0,choice,0#read|1#write}-only mode. + */ + public static final short OpenedReadOrWriteOnly_2 = 203; + /** * Coordinate is outside the domain of validity. */ diff --git a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.properties b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.properties index 031bec00d6..c595f1edd4 100644 --- a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.properties +++ b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors.properties @@ -170,6 +170,7 @@ NullMapKey = Null key is not allowed in this dictionary. NullMapValue = Null values are not allowed in this dictionary. NullValueInTable_3 = Unexpected null value in record \u201c{2}\u201d for the column \u201c{1}\u201d in table \u201c{0}\u201d. OddArrayLength_1 = Array length is {0}, while we expected an even length. +OpenedReadOrWriteOnly_2 = \u201c{1}\u201d is opened in {0,choice,0#read|1#write}-only mode. OutsideDomainOfValidity = Coordinate is outside the domain of validity. PropertyNotFound_2 = No property named \u201c{1}\u201d has been found in \u201c{0}\u201d. RecordAlreadyDefined_2 = Record \u201c{1}\u201d is already defined in schema \u201c{0}\u201d. diff --git a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors_fr.properties b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors_fr.properties index ff8a5066cb..16c030cad5 100644 --- a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors_fr.properties +++ b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/resources/Errors_fr.properties @@ -166,6 +166,7 @@ NullMapKey = La cl\u00e9 nulle n\u2019est pas autoris\u00 NullMapValue = Les valeurs nulles ne sont pas autoris\u00e9es dans ce dictionnaire. NullValueInTable_3 = Dans la table \u00ab\u202f{0}\u202f\u00bb, la colonne \u00ab\u202f{1}\u202f\u00bb de l\u2019enregistrement \u00ab\u202f{2}\u202f\u00bb ne devrait pas contenir de valeur nulle. OddArrayLength_1 = La longueur du tableau est {0}, alors qu\u2019on attendait une longueur paire. +OpenedReadOrWriteOnly_2 = \u00ab\u202f{1}\u202f\u00bb a \u00e9t\u00e9 ouvert en mode {0,choice,0#lecture|1#\u00e9criture} seule. OutsideDomainOfValidity = La coordonn\u00e9e est en dehors du domaine de validit\u00e9. PropertyNotFound_2 = Aucune propri\u00e9t\u00e9 nomm\u00e9e \u00ab\u202f{1}\u202f\u00bb n\u2019a \u00e9t\u00e9 trouv\u00e9e dans \u00ab\u202f{0}\u202f\u00bb. RecordAlreadyDefined_2 = L\u2019enregistrement \u00ab\u202f{1}\u202f\u00bb est d\u00e9j\u00e0 d\u00e9finit dans le sch\u00e9ma \u00ab\u202f{0}\u202f\u00bb.