This is an automated email from the ASF dual-hosted git repository. desruisseaux pushed a commit to branch geoapi-4.0 in repository https://gitbox.apache.org/repos/asf/sis.git
commit 8facc18164725ed86aeba392fc59dc79e6ed6dac Author: Martin Desruisseaux <martin.desruisse...@geomatys.com> AuthorDate: Sat Sep 16 20:04:55 2023 +0200 First version of a TIFF writer (work initiated by Erwan Roussel). This initial version works, but with a limited amount of color models and sample models. https://issues.apache.org/jira/browse/SIS-589 --- .../sis/coverage/grid/j2d/ImageUtilities.java | 18 + .../org/apache/sis/storage/geotiff/GeoTIFF.java | 6 + .../apache/sis/storage/geotiff/GeoTiffOption.java | 60 ++ .../apache/sis/storage/geotiff/GeoTiffStore.java | 78 ++- .../sis/storage/geotiff/GeoTiffStoreProvider.java | 18 +- .../org/apache/sis/storage/geotiff/Reader.java | 10 + .../sis/storage/geotiff/ReformattedImage.java | 143 ++++ .../apache/sis/storage/geotiff/TagValueWriter.java | 69 ++ .../sis/storage/geotiff/TileMatrixWriter.java | 204 ++++++ .../org/apache/sis/storage/geotiff/Writer.java | 740 +++++++++++++++++++++ .../apache/sis/storage/geotiff/package-info.java | 2 +- .../org/apache/sis/storage/geotiff/WriterTest.java | 441 ++++++++++++ .../apache/sis/storage/base/MetadataFetcher.java | 339 ++++++++++ .../org/apache/sis/util/internal/Numerics.java | 6 + 14 files changed, 2121 insertions(+), 13 deletions(-) diff --git a/endorsed/src/org.apache.sis.feature/main/org/apache/sis/coverage/grid/j2d/ImageUtilities.java b/endorsed/src/org.apache.sis.feature/main/org/apache/sis/coverage/grid/j2d/ImageUtilities.java index 69d1a5ee85..f7f96ae301 100644 --- a/endorsed/src/org.apache.sis.feature/main/org/apache/sis/coverage/grid/j2d/ImageUtilities.java +++ b/endorsed/src/org.apache.sis.feature/main/org/apache/sis/coverage/grid/j2d/ImageUtilities.java @@ -115,6 +115,22 @@ public final class ImageUtilities extends Static { ((long) low) + aoi.height) - aoi.y); } + /** + * Returns whether the given image has an alpha channel. + * + * @param image the image or {@code null}. + * @return whether the image has an alpha channel. + * + * @see #getTransparencyDescription(ColorModel) + */ + public static boolean hasAlpha(final RenderedImage image) { + if (image != null) { + final ColorModel cm = image.getColorModel(); + if (cm != null) return cm.hasAlpha(); + } + return false; + } + /** * Returns the number of bands in the given image, or 0 if the image or its sample model is null. * @@ -242,6 +258,8 @@ public final class ImageUtilities extends Static { * * @param cm the color model from which to get the transparency, or {@code null}. * @return a {@link Resources.Keys} value for the transparency, or 0 if unknown. + * + * @see #hasAlpha(RenderedImage) */ public static short getTransparencyDescription(final ColorModel cm) { if (cm != null) { diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java index 9d0ee1e76b..11d3caba35 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTIFF.java @@ -16,6 +16,7 @@ */ package org.apache.sis.storage.geotiff; +import java.util.Set; import java.util.Locale; import java.util.TimeZone; import java.io.Closeable; @@ -76,6 +77,11 @@ abstract class GeoTIFF implements Closeable { this.store = store; } + /** + * {@return the options (BigTIFF, COG…) used by this reader or writer}. + */ + abstract Set<GeoTiffOption> getOptions(); + /** * Returns the resources to use for formatting error messages. */ diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffOption.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffOption.java new file mode 100644 index 0000000000..9d74388a74 --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffOption.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import org.apache.sis.setup.OptionKey; +import org.apache.sis.io.stream.InternalOptionKey; + + +/** + * Characteristics of the GeoTIFF file to write. + * The options can control, for example, the maximal size and number of images that can be stored in a TIFF file. + * See {@link #OPTION_KEY} for an usage example. + * + * @author Martin Desruisseaux (Geomatys) + * @version 1.5 + * + * @see GeoTiffStore#getOptions() + * + * @since 1.5 + */ +public enum GeoTiffOption { + /** + * The Big TIFF extension (non-standard). + * When this option is absent (which is the default), the standard TIFF format as defined by Adobe is used. + * That standard uses the addressable space of 32-bits integers, which allows a maximal file size of about 4 GB. + * When the {@code BIG_TIFF} option is present, the addressable space of 64-bits integers is used. + * The BigTIFF format is non-standard and files written with this option may not be read by all TIFF readers. + */ + BIG_TIFF; + + // TODO: COG, SPARSE. + + /** + * The key for declaring GeoTIFF options at store creation time. + * For writing a BigTIFF file, the following code can be used: + * + * {@snippet lang="java" : + * var file = Path.of("my_output_file.tiff"); + * var connector = new StorageConnector(file); + * var options = new GeoTiffOption[] {GeoTiffOption.BIG_TIFF}; + * connector.setOption(GeoTiffOption.OPTION_KEY, options); + * DataStore ds = DataStores.open(c); + * } + */ + public static final OptionKey<GeoTiffOption[]> OPTION_KEY = new InternalOptionKey<>("TIFF_OPTIONS", GeoTiffOption[].class); +} diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java index 2bcb7c4a73..cd0d520035 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStore.java @@ -16,6 +16,7 @@ */ package org.apache.sis.storage.geotiff; +import java.util.Set; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -46,6 +47,7 @@ import org.apache.sis.storage.event.StoreListener; import org.apache.sis.storage.event.StoreListeners; import org.apache.sis.storage.event.WarningEvent; import org.apache.sis.io.stream.ChannelDataInput; +import org.apache.sis.io.stream.ChannelDataOutput; import org.apache.sis.io.stream.IOUtilities; import org.apache.sis.storage.base.MetadataBuilder; import org.apache.sis.storage.base.StoreUtilities; @@ -69,7 +71,7 @@ import org.apache.sis.util.resources.Errors; * @author Martin Desruisseaux (Geomatys) * @author Thi Phuong Hao Nguyen (VNSC) * @author Alexis Manin (Geomatys) - * @version 1.4 + * @version 1.5 * @since 0.8 */ public class GeoTiffStore extends DataStore implements Aggregate { @@ -86,6 +88,19 @@ public class GeoTiffStore extends DataStore implements Aggregate { */ private volatile Reader reader; + /** + * The GeoTIFF writer implementation, or {@code null} if the store has been closed. + * + * @see #writer() + */ + private volatile Writer writer; + + /** + * The locale to use for formatting metadata. This is not necessarily the same as {@link #getLocale()}, + * which is about formatting error messages. A null value means "unlocalized", which is usually English. + */ + final Locale dataLocale; + /** * The {@link GeoTiffStoreProvider#LOCATION} parameter value, or {@code null} if none. * This is used for information purpose only, not for actual reading operations. @@ -197,23 +212,44 @@ public class GeoTiffStore extends DataStore implements Aggregate { super(parent, provider, connector, hidden); this.hidden = hidden; + @SuppressWarnings("LocalVariableHidesMemberVariable") final SchemaModifier customizer = connector.getOption(SchemaModifier.OPTION); this.customizer = (customizer != null) ? customizer : SchemaModifier.DEFAULT; + @SuppressWarnings("LocalVariableHidesMemberVariable") final Charset encoding = connector.getOption(OptionKey.ENCODING); this.encoding = (encoding != null) ? encoding : StandardCharsets.US_ASCII; - location = connector.getStorageAs(URI.class); - path = connector.getStorageAs(Path.class); - final ChannelDataInput input = connector.commit(ChannelDataInput.class, Constants.GEOTIFF); + dataLocale = connector.getOption(OptionKey.LOCALE); + location = connector.getStorageAs(URI.class); + path = connector.getStorageAs(Path.class); try { - reader = new Reader(this, input); + if (URIDataStore.Provider.isWritable(connector)) { + ChannelDataOutput output = connector.commit(ChannelDataOutput.class, Constants.GEOTIFF); + writer = new Writer(this, output, connector.getOption(GeoTiffOption.OPTION_KEY)); + } else { + ChannelDataInput input = connector.commit(ChannelDataInput.class, Constants.GEOTIFF); + reader = new Reader(this, input); + if (getClass() == GeoTiffStore.class) { + listeners.useReadOnlyEvents(); + } + } } catch (IOException e) { throw new DataStoreException(e); } - if (getClass() == GeoTiffStore.class) { - listeners.useReadOnlyEvents(); - } + } + + /** + * Returns the options (BigTIFF, COG…) of this data store. + * + * @return options of this data store. + * + * @since 1.5 + */ + public Set<GeoTiffOption> getOptions() { + if (writer != null) return writer.getOptions(); + if (reader != null) return reader.getOptions(); + return Set.of(); } /** @@ -262,7 +298,14 @@ public class GeoTiffStore extends DataStore implements Aggregate { */ @Override public Optional<ParameterValueGroup> getOpenParameters() { - return Optional.ofNullable(URIDataStore.parameters(provider, location)); + final ParameterValueGroup param = URIDataStore.parameters(provider, location); + if (param != null && writer != null) { + final Set<GeoTiffOption> options = writer.getOptions(); + if (!options.isEmpty()) { + param.parameter(GeoTiffStoreProvider.OPTIONS).setValue(options.toArray(GeoTiffOption[]::new)); + } + } + return Optional.ofNullable(param); } /** @@ -393,6 +436,20 @@ public class GeoTiffStore extends DataStore implements Aggregate { return r; } + /** + * Returns the writer if it is not closed, or throws an exception otherwise. + * + * @see #close() + */ + final Writer writer() throws DataStoreException { + assert Thread.holdsLock(this); + final Writer w = writer; + if (w == null) { + throw new DataStoreClosedException(getLocale(), Constants.GEOTIFF, StandardOpenOption.WRITE); + } + return w; + } + /** * Returns descriptions of all images in this GeoTIFF file. * Images are not immediately loaded. @@ -548,6 +605,8 @@ public class GeoTiffStore extends DataStore implements Aggregate { try { listeners.close(); // Should never fail. final Reader r = reader; + final Writer w = writer; + if (w != null) w.close(); if (r != null) r.close(); } catch (IOException e) { throw new DataStoreException(e); @@ -558,6 +617,7 @@ public class GeoTiffStore extends DataStore implements Aggregate { metadata = null; nativeMetadata = null; reader = null; + writer = null; } } } diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStoreProvider.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStoreProvider.java index aedecf2bae..1c130998e6 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStoreProvider.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/GeoTiffStoreProvider.java @@ -32,6 +32,8 @@ import org.apache.sis.storage.base.StoreMetadata; import org.apache.sis.storage.base.Capability; import org.apache.sis.storage.base.URIDataStore; import org.apache.sis.util.internal.Constants; +import org.apache.sis.util.resources.Vocabulary; +import org.apache.sis.parameter.ParameterBuilder; /** @@ -43,7 +45,7 @@ import org.apache.sis.util.internal.Constants; * the part of the caller. However, the {@link GeoTiffStore} instances created by this factory are not thread-safe. * * @author Martin Desruisseaux (Geomatys) - * @version 1.4 + * @version 1.5 * * @see GeoTiffStore * @@ -51,7 +53,7 @@ import org.apache.sis.util.internal.Constants; */ @StoreMetadata(formatName = Constants.GEOTIFF, fileSuffixes = {"tiff", "tif"}, - capabilities = Capability.READ, + capabilities = {Capability.READ, Capability.WRITE}, resourceTypes = {Aggregate.class, GridCoverageResource.class}) public class GeoTiffStoreProvider extends DataStoreProvider { /** @@ -71,10 +73,20 @@ public class GeoTiffStoreProvider extends DataStoreProvider { */ private static final Logger LOGGER = Logger.getLogger("org.apache.sis.storage.geotiff"); + /** + * Name of the parameter for specifying the options (BigTIFF, COG…). + */ + static final String OPTIONS = "options"; + /** * The parameter descriptor to be returned by {@link #getOpenParameters()}. */ - private static final ParameterDescriptorGroup OPEN_DESCRIPTOR = URIDataStore.Provider.descriptor(Constants.GEOTIFF); + private static final ParameterDescriptorGroup OPEN_DESCRIPTOR; + static { + final var builder = new ParameterBuilder(); + final var options = builder.addName(OPTIONS).setDescription(Vocabulary.formatInternational(Vocabulary.Keys.Options)).create(GeoTiffOption[].class, null); + OPEN_DESCRIPTOR = builder.addName(Constants.GEOTIFF).createGroup(URIDataStore.Provider.LOCATION_PARAM, options); + } /** * Creates a new provider. diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java index 27e3eaf7e2..add3fce9dc 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Reader.java @@ -74,6 +74,8 @@ final class Reader extends GeoTIFF { * * Those values are defined that way for making easier (like a boolean flag) to test if * the file is a BigTIFF format, with statement like {@code if (intSizeExpansion != 0)}. + * + * @see #getFormat() */ final byte intSizeExpansion; @@ -187,6 +189,14 @@ final class Reader extends GeoTIFF { throw new DataStoreContentException(store.errors().getString(Errors.Keys.UnexpectedFileFormat_2, "TIFF", input.filename)); } + /** + * {@return the options (BigTIFF, COG…) used by this reader}. + */ + @Override + final Set<GeoTiffOption> getOptions() { + return (intSizeExpansion != 0) ? Set.of(GeoTiffOption.BIG_TIFF) : Set.of(); + } + /** * Sets {@link #nextIFD} to the next offset read from the TIFF file * and makes sure that it will not cause an infinite loop. diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java new file mode 100644 index 0000000000..9738f7da1f --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/ReformattedImage.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import java.awt.color.ColorSpace; +import java.awt.image.ColorModel; +import java.awt.image.IndexColorModel; +import java.awt.image.RenderedImage; +import static javax.imageio.plugins.tiff.BaselineTIFFTagSet.*; +import org.apache.sis.util.ArraysExt; +import org.apache.sis.math.Statistics; +import org.apache.sis.image.PlanarImage; +import org.apache.sis.coverage.grid.j2d.ImageUtilities; +import org.apache.sis.storage.IncompatibleResourceException; + + +/** + * An image prepared for writing with bands separated in the way they are stored in a TIFF file. + * The TIFF specification stores visible bands, alpha channel and extra bands separately. + * + * @todo Force tile size to multiple of 16. + * + * @author Martin Desruisseaux (Geomatys) + */ +final class ReformattedImage { + /** + * The main image with visible bands. + */ + final RenderedImage visibleBands; + + /* + * TODO: alpha and extra bands not yet stored. + * This will be handled in a future version. + */ + + /** + * Formats the given image into something that can be written in a GeoTIFF file. + * The visible image will have at most 3 bands and should have no alpha channel. + * If no change is needed, then the given image is used unchanged. + * + * @param writer the writer for which to separate in image. + * @param image the image to separate into visible, alpha and extra bands. + */ + ReformattedImage(final Writer writer, final RenderedImage image) { + final int numBands = ImageUtilities.getNumBands(image); +select: if (numBands > 1) { + final int[] bands; + final int band = ImageUtilities.getVisibleBand(image); + if (band >= 0) { + bands = new int[] {band}; + } else { + int max = 3; // TIFF can store only 3 bands (ignoring extra bands). + if (ImageUtilities.hasAlpha(image)) { + max = Math.min(max, numBands - 1); // The alpha band is always the last one. + } + if (numBands <= max) { + break select; + } + bands = ArraysExt.range(0, max); + } + visibleBands = writer.processor().selectBands(image, bands); + return; + } + visibleBands = image; + } + + /** + * Returns statistics about pixel values in the visible bands. + * This method does not scan pixel values if statistics are not already present. + * + * @param numbands number of bands to retain. + * @return statistics in an array of length 2, with minimums first then maximums. + * Array elements may be {@code null} if there is no statistics. + */ + final double[][] statistics(final int numBands) { + final Object property = visibleBands.getProperty(PlanarImage.STATISTICS_KEY); +found: if (property instanceof Statistics[]) { + final var stats = (Statistics[]) property; + final var min = new double[numBands]; + final var max = new double[numBands]; + for (int i=0; i<numBands; i++) { + final Statistics s = stats[i]; + if (s.count() == 0) break found; + min[i] = s.minimum(); + max[i] = s.maximum(); + } + return new double[][] {min, max}; + } + return new double[2][]; + } + + /** + * Returns the TIFF color interpretation. + * + * @return One of {@code PHOTOMETRIC_INTERPRETATION_*} constants. + * @throws IncompatibleResourceException if the color model is not supported. + */ + final int getColorInterpretation() throws IncompatibleResourceException { + final ColorModel cm = visibleBands.getColorModel(); + if (cm instanceof IndexColorModel) { + final var icm = (IndexColorModel) cm; + final int last = icm.getMapSize() - 1; + final float scale = 255f / last; + boolean white = true; + boolean black = true; + for (int i=0; i <= last; i++) { + final int expected = Math.round(i * scale); + if (black) black = icm.getRGB( i) == expected; + if (white) white = icm.getRGB(last-i) == expected; + if (!(black | white)) break; + } + if (black) return PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO; + if (white) return PHOTOMETRIC_INTERPRETATION_WHITE_IS_ZERO; + return PHOTOMETRIC_INTERPRETATION_PALETTE_COLOR; + } + switch (cm.getColorSpace().getType()) { + case ColorSpace.TYPE_GRAY: { + return PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO; + } + case ColorSpace.TYPE_RGB: { + return PHOTOMETRIC_INTERPRETATION_RGB; + } + default: { + // A future version may add support for more color models. + throw new IncompatibleResourceException("Unsupported color model"); + } + } + } +} diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TagValueWriter.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TagValueWriter.java new file mode 100644 index 0000000000..f46e6fffb1 --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TagValueWriter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import java.io.IOException; +import org.apache.sis.io.stream.UpdatableWrite; +import org.apache.sis.io.stream.ChannelDataOutput; + + +/** + * Writer of a tag value or array of values which are too large for fitting directly in a tag entry. + * The tag entry will contain the position in the stream where those values are written, + * and the values themselves will be written after all tag entries. + * + * @author Martin Desruisseaux (Geomatys) + */ +abstract class TagValueWriter { + /** + * A handler for writing the position of tag values when this position will become known. + * This is initialized by {@link Writer#writeLargeTag(short, short, long, TagValueWriter)}. + */ + UpdatableWrite<?> offset; + + /** + * Creates a new container for the values of a tag. + */ + TagValueWriter() { + } + + /** + * Writes the values of the tag at the current position of the given output stream. + * + * @param output the {@link Writer#output} value, provided for convenience. + * @throws IOException if an error occurred while writing the tag values. + */ + abstract void write(ChannelDataOutput output) throws IOException; + + /** + * Writes again the values at the same offset than previously. + * This is used when those values have changed. + * + * @param output the stream where to write tag values. + * @throws IOException if an error occurred while writing the tag values. + * + * @see TileMatrixWriter#isLengthChanged() + */ + final void rewrite(final ChannelDataOutput output) throws IOException { + /* + * The offset is empty if the image has only one tile, + * because in such case the value fits in the IFD entry. + */ + output.seek(offset.getAsLong().orElse(offset.position)); + write(output); + } +} diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java new file mode 100644 index 0000000000..64e3406907 --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/TileMatrixWriter.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import java.util.Arrays; +import java.io.IOException; +import java.awt.Rectangle; +import java.awt.image.Raster; +import java.awt.image.RenderedImage; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferShort; +import java.awt.image.DataBufferUShort; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferFloat; +import java.awt.image.DataBufferDouble; +import java.awt.image.SampleModel; +import java.awt.image.ComponentSampleModel; +import java.awt.image.MultiPixelPackedSampleModel; +import java.awt.image.SinglePixelPackedSampleModel; +import org.apache.sis.image.DataType; +import org.apache.sis.util.internal.Numerics; +import org.apache.sis.io.stream.ChannelDataOutput; +import org.apache.sis.io.stream.HyperRectangleWriter; + + +/** + * Handler for writing offsets and lengths of tiles. + * Tile size should be multiples of 16 according TIFF specification, but this is not enforced here. + * + * @author Martin Desruisseaux (Geomatys) + */ +final class TileMatrixWriter { + /** + * The images to write. + */ + private final RenderedImage image; + + /** + * The type of sample values. + */ + private final DataType type; + + /** + * Number of planes to write. Different then 1 only for planar images. + */ + private final int numPlanes; + + /** + * Number of tiles along each axis and in total. + */ + private final int numXTiles, numYTiles, numTiles; + + /** + * Size of each tile. + */ + final int tileWidth, tileHeight; + + /** + * Uncompressed size of tiles in number of bytes, as an unsigned integer. + */ + private final int tileSize; + + /** + * Compressed size of each tile in number of bytes, as unsigned integers. + */ + final int[] lengths; + + /** + * Offsets to each tiles. Not necessarily in increasing order (it depends on tile order). + */ + final long[] offsets; + + /** + * Tags where are stored offsets and lengths. + */ + TagValueWriter offsetsTag, lengthsTag; + + /** + * Creates a new set of information about tiles to write. + * + * @param image the image to write. + * @param dataType the type of sample values. + * @param numBands the number of bands. + * @param bitsPerSample number of bits per sample to write. + * @param isPlanar whether the planar configuration is to store bands in separated planes. + */ + TileMatrixWriter(final RenderedImage image, final DataType type, final int numPlanes, final int[] bitsPerSample) { + final int pixelSize, numArrays; + this.numPlanes = numPlanes; + this.type = type; + this.image = image; + tileWidth = image.getTileWidth(); + tileHeight = image.getTileHeight(); + pixelSize = (bitsPerSample != null) ? Numerics.ceilDiv(Arrays.stream(bitsPerSample).sum(), Byte.SIZE) : 1; + tileSize = tileWidth * tileHeight * pixelSize; // Overflow is not really a problem for our usage. + numXTiles = image.getNumXTiles(); + numYTiles = image.getNumYTiles(); + numTiles = Math.multiplyExact(numXTiles, numYTiles); + numArrays = Math.multiplyExact(numPlanes, numTiles); + offsets = new long[numArrays]; + lengths = new int [numArrays]; + Arrays.fill(lengths, tileSize); + } + + /** + * Rewrites the offsets and lengths arrays in the IFD. + * This method shall be invoked after all tiles have been written. + * + * @throws IOException if an error occurred while writing to the output stream. + */ + final void writeOffsetsAndLengths(final ChannelDataOutput output) throws IOException { + offsetsTag.rewrite(output); + for (int value : lengths) { + if (value != tileSize) { + lengthsTag.rewrite(output); + break; + } + } + } + + /** + * Writes all tiles of the image. + * Caller shall invoke {@link #writeOffsetsAndLengths(ChannelDataOutput)} after this method. + * This invocation is not done by this method for allowing the caller to control when to write data. + * + * @param output where to write the tiles data. + * @throws IOException if an error occurred while writing to the given output. + */ + final void writeRasters(final ChannelDataOutput output) throws IOException { + SampleModel sm = null; + int[] bankIndices = null; + HyperRectangleWriter rect = null; + final int minTileX = image.getMinTileX(); + final int minTileY = image.getMinTileY(); + int planeIndex = 0; + for (int i=0; i < offsets.length; i++) { + /* + * In current implementation, we iterate from left to right then top to bottom. + * But a future version could use Hilbert iterator (for example). + */ + int tileX = i % numXTiles; + int tileY = i / numXTiles; + tileX += minTileX; + tileY += minTileY; + final Raster tile = image.getTile(tileX, tileY); + if (sm != (sm = tile.getSampleModel())) { + rect = null; + final var region = new Rectangle(tileWidth, tileHeight); + if (sm instanceof ComponentSampleModel) { + final var csm = (ComponentSampleModel) sm; + rect = HyperRectangleWriter.of(csm, region); + bankIndices = csm.getBankIndices(); + } else if (sm instanceof SinglePixelPackedSampleModel) { + final var csm = (SinglePixelPackedSampleModel) sm; + rect = HyperRectangleWriter.of(csm, region); + bankIndices = new int[1]; + } else if (sm instanceof MultiPixelPackedSampleModel) { + final var csm = (MultiPixelPackedSampleModel) sm; + rect = HyperRectangleWriter.of(csm, region); + bankIndices = new int[1]; + } + } + if (rect == null) { + throw new UnsupportedOperationException(); // TODO: reformat using a recycled Raster. + } + final long position = output.getStreamPosition(); + final DataBuffer buffer = tile.getDataBuffer(); + final int[] bufferOffsets = buffer.getOffsets(); + for (int j=0; j < numPlanes; j++) { + final int b = bankIndices[j]; + final int offset = bufferOffsets[b]; + switch (type) { + case BYTE: rect.write(output, ((DataBufferByte) buffer).getData(b), offset); break; + case USHORT: rect.write(output, ((DataBufferUShort) buffer).getData(b), offset); break; + case SHORT: rect.write(output, ((DataBufferShort) buffer).getData(b), offset); break; + case INT: rect.write(output, ((DataBufferInt) buffer).getData(b), offset); break; + case FLOAT: rect.write(output, ((DataBufferFloat) buffer).getData(b), offset); break; + case DOUBLE: rect.write(output, ((DataBufferDouble) buffer).getData(b), offset); break; + } + } + offsets[planeIndex] = position; + lengths[planeIndex] = Math.toIntExact(Math.subtractExact(output.getStreamPosition(), position)); + planeIndex++; + } + if (planeIndex != offsets.length) { + throw new AssertionError(); // Should never happen. + } + } +} diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java new file mode 100644 index 0000000000..f86f7bfb61 --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/Writer.java @@ -0,0 +1,740 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import java.io.Flushable; +import java.io.IOException; +import java.nio.ByteOrder; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Deque; +import java.util.Queue; +import java.util.Set; +import java.awt.image.RenderedImage; +import java.awt.image.SampleModel; +import java.awt.image.BandedSampleModel; +import java.awt.image.IndexColorModel; +import javax.imageio.plugins.tiff.TIFFTag; +import org.opengis.metadata.Metadata; +import org.apache.sis.image.DataType; +import org.apache.sis.image.ImageProcessor; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.base.MetadataFetcher; +import org.apache.sis.io.stream.ChannelDataOutput; +import org.apache.sis.io.stream.UpdatableWrite; +import org.apache.sis.util.internal.Numerics; +import org.apache.sis.util.resources.Errors; +import org.apache.sis.util.CharSequences; +import org.apache.sis.util.ArraysExt; +import org.apache.sis.math.Fraction; + +import static javax.imageio.plugins.tiff.BaselineTIFFTagSet.*; + + +/** + * An image writer for GeoTIFF files. This writer duplicates the implementations performed by other libraries, + * but we nevertheless provide our own writer in Apache SIS for better control on the internal file structure, + * such as keeping metadata close to each other (for Cloud Optimized GeoTIFF) and tiles order. + * This image writer can also handle <cite>Big TIFF</cite> images. + * + * <p>This writer supports only the tile layout. It does not support the writing of stripped images, + * because they are not useful for geospatial applications. This restriction does not reduce the set + * of Java2D images that this writer can encode.</p> + * + * <p>The TIFF format specification version 6.0 (June 3, 1992) is available + * <a href="https://partners.adobe.com/public/developer/en/tiff/TIFF6.pdf">here</a>.</p> + * + * @author Erwan Roussel (Geomatys) + * @author Martin Desruisseaux (Geomatys) + */ +final class Writer extends GeoTIFF implements Flushable { + /** + * BigTIFF code for unsigned 64-bits integer type. + * + * @see Type#ULONG + */ + static final short TIFF_ULONG = 16; + + /** + * Sizes of a few TIFF tags used in this writer. + * + * @see #writeTag(short, short, int[]) + * @see #writeTag(short, short, double[]) + */ + private static final byte[] TYPE_SIZES = new byte[TIFF_ULONG + 1]; + static { + TYPE_SIZES[TIFFTag.TIFF_ASCII] = // TIFF uses US-ASCII encoding as bytes. + TYPE_SIZES[TIFFTag.TIFF_BYTE] = + TYPE_SIZES[TIFFTag.TIFF_SBYTE] = Byte.BYTES; + TYPE_SIZES[TIFFTag.TIFF_SHORT] = + TYPE_SIZES[TIFFTag.TIFF_SSHORT] = Short.BYTES; + TYPE_SIZES[TIFFTag.TIFF_LONG] = + TYPE_SIZES[TIFFTag.TIFF_SLONG] = Integer.BYTES; // What TIFF calls "long" is Java integer. + TYPE_SIZES[TIFFTag.TIFF_RATIONAL] = + TYPE_SIZES[TIFFTag.TIFF_SRATIONAL] = Integer.BYTES * 2; + TYPE_SIZES[TIFFTag.TIFF_FLOAT] = Float.BYTES; + TYPE_SIZES[TIFFTag.TIFF_DOUBLE] = Double.BYTES; + TYPE_SIZES[TIFFTag.TIFF_IFD_POINTER] = Integer.BYTES; // Assuming standard TIFF (not BigTIFF). + TYPE_SIZES[ TIFF_ULONG] = Long.BYTES; + } + + /** + * Minimal number of tags which will be written. This amount is for grayscale images with no metadata + * and no statistics. For RGB images, there is one more tag. For color maps, there is two more tags. + * This number is only a hint for avoiding the need to update this information if the number appears + * to be right. + */ + static final int MINIMAL_NUMBER_OF_TAGS = 16; + + /** + * The processor to use for transforming the image before to write it. + * Created only if needed. + * + * @see #processor() + */ + private ImageProcessor processor; + + /** + * The stream where to write the data. + */ + private final ChannelDataOutput output; + + /** + * Whether the lengths and offsets shall be written as 64-bits integers instead of 32-bits integers. + * + * @see #getFormat() + */ + private final boolean isBigTIFF; + + /** + * Offset where to write the next image, or an offset value of 0 if none. + * Shall be one of the elements in the {@link #deferredWrites} queue. + */ + private UpdatableWrite<?> nextIFD; + + /** + * All values that couldn't be written immediately. + * Values shall be sorted in increasing order of stream position. + */ + private final Deque<UpdatableWrite<?>> deferredWrites; + + /** + * Write operations for tag having data too large for fitting inside a IFD tag entry. + * The writing of those data need to be delayed somewhere after the sequence of entries. + */ + private final Queue<TagValueWriter> largeTagData; + + /** + * Number of TIFF tag entries in the image being written. + * This is a temporary information used during the writing of an Image File Directory (IFD). + */ + private int numberOfTags; + + /** + * Creates a new GeoTIFF writer which will write data in the given output. + * + * @param store the store writing data. + * @param output where to write the bytes. + * @param options the options (BigTIFF, COG…), or {@code null} if none. + * @throws IOException if an error occurred while writing the first bytes to the stream. + */ + Writer(final GeoTiffStore store, final ChannelDataOutput output, final GeoTiffOption[] options) + throws IOException, DataStoreException + { + super(store); + this.output = output; + isBigTIFF = ArraysExt.contains(options, GeoTiffOption.BIG_TIFF); + deferredWrites = new ArrayDeque<>(); + largeTagData = new ArrayDeque<>(); + /* + * Write the TIFF file header before first IFD. Stream position matter and must start at zero. + * Note that it does not necessarily mean that the stream has no bytes before current position. + */ + output.setStreamPosition(0); // Not a seek, only setting the counter. + output.writeShort(ByteOrder.LITTLE_ENDIAN.equals(output.buffer.order()) ? LITTLE_ENDIAN : BIG_ENDIAN); + output.writeShort(isBigTIFF ? BIG_TIFF : CLASSIC); + if (isBigTIFF) { + output.writeShort((short) Long.BYTES); // Byte size of offsets. + output.writeShort((short) 0); // Constant. + output.writeLong(Long.BYTES + 4*Short.BYTES); // Position of the first IFD. + } else { + output.writeInt(Integer.BYTES + 2*Short.BYTES); + } + } + + /** + * {@return the options (BigTIFF, COG…) used by this writer}. + */ + @Override + final Set<GeoTiffOption> getOptions() { + return isBigTIFF ? Set.of(GeoTiffOption.BIG_TIFF) : Set.of(); + } + + /** + * {@return the processor to use for reformatting the image before to write it}. + * The processor is created only when this method is first invoked. + */ + final ImageProcessor processor() { + if (processor == null) { + processor = new ImageProcessor(); + } + return processor; + } + + /** + * Encodes the given image to the output stream given at construction time. + * The image is appended after any previous images written before the given one. + * This method does not handle pyramids such as Cloud Optimized GeoTIFF (COG). + * It is caller responsibility to append image overviews if a pyramid is wanted. + * + * @param image the image to encode. + * @param metadata title, author and other information, or {@code null} if none. + * @throws IOException if an error occurred while writing to the output. + * @throws DataStoreException if the given {@code image} has a property + * which is not supported by TIFF specification or by this writer. + */ + final void append(final RenderedImage image, final Metadata metadata) + throws IOException, DataStoreException + { + final TileMatrixWriter tiles; + try { + tiles = writeImageFileDirectory(new ReformattedImage(this, image), metadata, false); + } finally { + largeTagData.clear(); // For making sure that there is no memory retention. + } + tiles.writeRasters(output); + wordAlign(output); + tiles.writeOffsetsAndLengths(output); + } + + /** + * Writes the Image File Directory (IFD) of the given image. + * This method does not write the pixel values. Those values must be written by the caller. + * This separation makes possible to write directories in any order compared to pixel data. + * + * @param image the image for which to write the IFD. + * @param metadata title, author and other information, or {@code null} if none. + * @param oveverview whether the image is an overview of another image. + * @return handler for writing offsets and lengths of the tiles to write. + * @throws IOException if an error occurred while writing to the output. + * @throws DataStoreException if the given {@code image} has a property + * which is not supported by TIFF specification or by this writer. + */ + private TileMatrixWriter writeImageFileDirectory(final ReformattedImage image, final Metadata metadata, + final boolean overview) throws IOException, DataStoreException + { + /* + * Extract all image properties and metadata that we will need to encode in the Image File Directory. + * It allows us to know if we will be able to encode the image before we start writing in the stream, + * so that the TIFF file is not corrupted if we cannot write that image. It is also more convenient + * because the tags need to be written in increasing code order, which causes ColorModel-related tags + * (for example) to be interleaved with other aspects. + */ + numberOfTags = MINIMAL_NUMBER_OF_TAGS; // Only a guess at this stage. Real number computed later. + final int colorInterpretation = image.getColorInterpretation(); + if (colorInterpretation == PHOTOMETRIC_INTERPRETATION_PALETTE_COLOR) { + numberOfTags++; + } + final SampleModel sm = image.visibleBands.getSampleModel(); + final int[] bitsPerSample = sm.getSampleSize(); + final int numBands = sm.getNumBands(); + final int sampleFormat; + final DataType dataType = DataType.forDataBufferType(sm.getDataType()); + if (dataType.isUnsigned()) { + sampleFormat = SAMPLE_FORMAT_UNSIGNED_INTEGER; + } else if (dataType.isInteger()) { + sampleFormat = SAMPLE_FORMAT_SIGNED_INTEGER; + } else { + sampleFormat = SAMPLE_FORMAT_FLOATING_POINT; + } + final int numPlanes; + final int planarConfiguration; + if (sm instanceof BandedSampleModel) { + planarConfiguration = PLANAR_CONFIGURATION_PLANAR; + numPlanes = numBands; + } else { + planarConfiguration = PLANAR_CONFIGURATION_CHUNKY; + numPlanes = 1; + } + /* + * Metadata (optional) and GeoTIFF. They are managed by separated classes. + */ + final double[][] statistics = image.statistics(numBands); + final int[][] shortStats = toShorts(statistics, sampleFormat); + final MetadataFetcher<String> mf = new MetadataFetcher<>(store.dataLocale) { + @Override protected String parseDate(final Date date) { + return getDateFormat().format(date); + } + }; + mf.accept(metadata); + /* + * Conversion factor from physical size to pixel size. "Physical size" here should be understood as + * paper size, as suggested by the units of measurement which are restricted to inch or centimeters. + * This is not very useful for geospatial applications, except as aspect ratio. + */ + final Fraction xres = new Fraction(1, 1); // TODO + final Fraction yres = xres; + /* + * If the image has any unsupported feature, the exception should have been thrown before this point. + * Now start writing the entries. The entries in an IFD must be sorted in ascending order by tag code. + */ + output.flush(); // Makes room in the buffer for increasing our ability to modify past values. + largeTagData.clear(); + final UpdatableWrite<?> tagCountWriter = + isBigTIFF ? UpdatableWrite.of(output, (long) numberOfTags) + : UpdatableWrite.of(output, (short) numberOfTags); + numberOfTags = 0; + writeTag((short) TAG_NEW_SUBFILE_TYPE, (short) TIFFTag.TIFF_LONG, overview ? 1 : 0); + writeTag((short) TAG_IMAGE_WIDTH, (short) TIFFTag.TIFF_LONG, image.visibleBands.getWidth()); + writeTag((short) TAG_IMAGE_LENGTH, (short) TIFFTag.TIFF_LONG, image.visibleBands.getHeight()); + writeTag((short) TAG_BITS_PER_SAMPLE, (short) TIFFTag.TIFF_SHORT, bitsPerSample); + writeTag((short) TAG_COMPRESSION, (short) TIFFTag.TIFF_SHORT, COMPRESSION_NONE); + writeTag((short) TAG_PHOTOMETRIC_INTERPRETATION, (short) TIFFTag.TIFF_SHORT, colorInterpretation); + writeTag((short) TAG_DOCUMENT_NAME, /* TIFF_ASCII */ mf.series); + writeTag((short) TAG_IMAGE_DESCRIPTION, /* TIFF_ASCII */ mf.title); + writeTag((short) TAG_MODEL, /* TIFF_ASCII */ mf.instrument); + writeTag((short) TAG_SAMPLES_PER_PIXEL, (short) TIFFTag.TIFF_SHORT, numBands); + writeTag((short) TAG_MIN_SAMPLE_VALUE, (short) TIFFTag.TIFF_SHORT, shortStats[0]); + writeTag((short) TAG_MAX_SAMPLE_VALUE, (short) TIFFTag.TIFF_SHORT, shortStats[1]); + writeTag((short) TAG_X_RESOLUTION, /* TIFF_RATIONAL */ xres); + writeTag((short) TAG_Y_RESOLUTION, /* TIFF_RATIONAL */ yres); + writeTag((short) TAG_PLANAR_CONFIGURATION, (short) TIFFTag.TIFF_SHORT, planarConfiguration); + writeTag((short) TAG_RESOLUTION_UNIT, (short) TIFFTag.TIFF_SHORT, RESOLUTION_UNIT_NONE); + writeTag((short) TAG_SOFTWARE, /* TIFF_ASCII */ mf.software); + writeTag((short) TAG_DATE_TIME, /* TIFF_ASCII */ mf.creationDate); + writeTag((short) TAG_ARTIST, /* TIFF_ASCII */ mf.party); + writeTag((short) TAG_HOST_COMPUTER, /* TIFF_ASCII */ mf.procedure); + if (colorInterpretation == PHOTOMETRIC_INTERPRETATION_PALETTE_COLOR) { + writeColorPalette((IndexColorModel) image.visibleBands.getColorModel(), 1L << bitsPerSample[0]); + } + final var tiling = new TileMatrixWriter(image.visibleBands, dataType, numPlanes, bitsPerSample); + writeTag((short) TAG_TILE_WIDTH, (short) TIFFTag.TIFF_LONG, tiling.tileWidth); + writeTag((short) TAG_TILE_LENGTH, (short) TIFFTag.TIFF_LONG, tiling.tileHeight); + tiling.offsetsTag = writeTag((short) TAG_TILE_OFFSETS, tiling.offsets); + tiling.lengthsTag = writeTag((short) TAG_TILE_BYTE_COUNTS, (short) TIFFTag.TIFF_LONG, tiling.lengths); + writeTag((short) TAG_SAMPLE_FORMAT, (short) TIFFTag.TIFF_SHORT, sampleFormat); + writeTag((short) TAG_S_MIN_SAMPLE_VALUE, (short) TIFFTag.TIFF_FLOAT, statistics[0]); + writeTag((short) TAG_S_MAX_SAMPLE_VALUE, (short) TIFFTag.TIFF_FLOAT, statistics[1]); + /* + * At this point, all tags have been written. Update the number of tags, + * then write all values that couldn't be written directly in the tags. + */ + tagCountWriter.setAsLong(numberOfTags); + writeOrQueue(tagCountWriter); + nextIFD = writeOffset(0); + for (final TagValueWriter tag : largeTagData) { + final UpdatableWrite<?> offset = tag.offset; + offset.setAsLong(output.getStreamPosition()); + writeOrQueue(offset); + tag.write(output); + } + return tiling; + } + + /** + * Writes a 32-bits or 64-bits offset, depending on whether the format is classic TIFF or BigTIFF. + * + * @param offset an initial guess of the offset value. + * @return a handler for updating later the offset with its actual value. + * @throws IOException if an error occurred while writing to the output. + */ + private UpdatableWrite<?> writeOffset(final long offset) throws IOException { + return isBigTIFF ? UpdatableWrite.of(output, offset) + : UpdatableWrite.of(output, (int) offset); + // No need to check the validity of above cast because the value is only a guess. + } + + /** + * Forces 16-bits word alignment. + * The TIFF specification requires that tag values are aligned. + * + * @param channel the channel on which to apply 16-bits word alignment. + * @throws IOException if an error occurred while writing to the output stream. + */ + private static void wordAlign(final ChannelDataOutput output) throws IOException { + if ((output.getStreamPosition() & 1) != 0) { + output.writeByte(0); + } + } + + /** + * If the sample format is integer, cast statistics to integer type and clears the given array. + * Otherwise do nothing. This is used for choosing only one of {@code TAG_MIN_SAMPLE_VALUE} and + * {@code TAG_S_MIN_SAMPLE_VALUE} tags (same for maximum). + * + * @param statistics the statistic to potentially cast and clear. + * @param sampleFormat the sample format. + * @return statistics for the tags restricted to integer types. + */ + private static int[][] toShorts(final double[][] statistics, final int sampleFormat) { + final int[][] c = new int[statistics.length][]; + final long min, max; + switch (sampleFormat) { + case SAMPLE_FORMAT_UNSIGNED_INTEGER: min = 0; max = 0xFFFF; break; + case SAMPLE_FORMAT_SIGNED_INTEGER: min = Short.MIN_VALUE; max = Short.MAX_VALUE; break; + default: return c; + } + for (int j=0; j < c.length; j++) { + final double[] source = statistics[j]; + if (source != null) { + final int[] target = new int[source.length]; + for (int i=0; i < source.length; i++) { + target[i] = (int) Math.max(min, Math.min(max, Math.round(source[i]))); + } + c[j] = target; + } + } + return c; + } + + /** + * Writes a new tag except for the value. This method ensures that the buffer has enough room for a full tag entry, + * so the caller can append an {@code int} (classical TIFF) or a {@code long} (big TIFF) directly in the buffer. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param type one of the {@link TIFFTag} constants such as {@code TIFF_SHORT} or {@code TIFF_LONG}. + * @param count number of values. + * @return number of bytes available for the IFD entry value. + * @throws IOException if an error occurred while writing to the output. + * @throws ArithmeticException if the count is too large for the TIFF format in use. + */ + private int writeTagHeader(final short tag, final short type, final long count) throws IOException { + numberOfTags++; + output.ensureBufferAccepts(2*Short.BYTES + 2*Long.BYTES); + final ByteBuffer buffer = output.buffer; + buffer.putShort(tag); + buffer.putShort(type); + if (isBigTIFF) { + buffer.putLong(count); + return Long.BYTES; + } else if ((count & Numerics.HIGH_BITS_MASK) == 0) { + // Note: unsigned integer may look negative after cast, this is okay. + buffer.putInt((int) count); + return Integer.BYTES; + } else { + throw new ArithmeticException(store.errors().getString(Errors.Keys.IntegerOverflow_1, Integer.SIZE)); + } + } + + /** + * Writes a tag value which is potentially too large for fitting in the IFD entry. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param type one of the {@link TIFFTag} constants such as {@code TIFF_SHORT} or {@code TIFF_LONG}. + * @param count number of values. + * @throws IOException if an error occurred while writing to the output. + * @throws ArithmeticException if the count is too large for the TIFF format in use. + */ + private TagValueWriter writeLargeTag(final short tag, final short type, final long count, final TagValueWriter deferred) throws IOException { + final long r = writeTagHeader(tag, type, count) - TYPE_SIZES[type] * count; + if (r >= 0) { + deferred.offset = UpdatableWrite.of(output); // Record only the position. + deferred.write(output); + output.repeat(r, (byte) 0); + } else { + deferred.offset = writeOffset(0); + largeTagData.add(deferred); + } + return deferred; + } + + /** + * Writes the color map tag. + * + * @param cm color model from which to read color values. + * @param count number of colors to write, <strong>not</strong> multiplied by 3 for the RGB bands. + * @throws IOException if an error occurred while writing to the output. + */ + private void writeColorPalette(final IndexColorModel cm, final long count) throws IOException { + final int numBands = 3; + writeLargeTag((short) TAG_COLOR_MAP, (short) TIFFTag.TIFF_SHORT, count * numBands, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + final int n = (int) Math.min(cm.getMapSize(), count); + for (int band=0; band < numBands; band++) { + for (int i=0; i<n; i++) { + final int c; + switch (band) { + case 0: c = cm.getRed (i); break; + case 1: c = cm.getGreen(i); break; + case 2: c = cm.getBlue (i); break; + default: throw new AssertionError(band); + } + output.writeShort(c | (c << Byte.SIZE)); + } + output.repeat((count - n) * Short.BYTES, (byte) 0); + } + } + }); + } + + /** + * Writes a tag with string values stored as ASCII characters. + * The list of valid tag code is defined by TIFF specification. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param values the values to write, or {@code null} if none. + * @throws IOException if an error occurred while writing to the output. + * @throws ArithmeticException if the combined string length is too large. + */ + private void writeTag(final short tag, final List<String> values) throws IOException { + if (values == null) { + return; + } + long count = 0; + final var chars = new byte[values.size()][]; + for (int i=0; i<chars.length; i++) { + String value = values.get(i).trim(); + if (StandardCharsets.US_ASCII.equals(store.encoding)) { + value = CharSequences.toASCII(value).toString(); + } + final byte[] c = value.getBytes(store.encoding); + if (c.length != 0) { + count += c.length + 1L; // Count shall include the trailing NUL character. + chars[i] = c; + } + } + if (count != 0) { + writeLargeTag(tag, (short) TIFFTag.TIFF_ASCII, count, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + for (final byte[] c : chars) { + if (c != null) { + output.write(c); + output.writeByte(0); + wordAlign(output); + } + } + } + }); + } + } + + /** + * Writes a tag as a rational number. Rational numbers are made of two integers: the numerator and denominator, + * in that order. In BigTIFF format, those two numbers fit in the entry and this method returns {@code null}. + * In classical format, those two numbers do not fit and must be stored in an array after the directory entries. + * In such case, this method saves a handle for performing that deferred write operation later. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param value numerator and denominator of the rational number to store, or {@code null} if none. + * @throws IOException if an error occurred while writing to the output. + */ + private void writeTag(final short tag, final Fraction value) throws IOException { + if (value == null) { + return; + } + writeLargeTag(tag, (short) TIFFTag.TIFF_RATIONAL, 1, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + output.writeInt(value.numerator); + output.writeInt(value.denominator); + } + }); + } + + /** + * Writes a tag with values stored as 32 or 64 bits floating point numbers. + * The list of valid tag codes is defined by TIFF specification. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param type {@code TIFF_FLOAT} or {@code TIFF_DOUBLE}. + * @param values the values to write as floating point values. + * @return a handler for rewriting the data if the array content changes. + * @throws IOException if an error occurred while writing to the output. + */ + private TagValueWriter writeTag(final short tag, final short type, final double[] values) throws IOException { + if (values == null || values.length == 0) { + return null; + } + return writeLargeTag(tag, type, values.length, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + switch (type) { + default: throw new AssertionError(type); + case TIFFTag.TIFF_DOUBLE: output.writeDoubles(values); break; + case TIFFTag.TIFF_FLOAT: { + for (final double value : values) { + output.writeFloat((float) value); + } + break; + } + } + } + }); + } + + /** + * Writes a tag with an arbitrary number of values stored as 64 or 32 bits unsigned integers. + * The number of bits depends on whether this writer is writing BigTIFF or classic TIFF. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param values the values to write as unsigned 64 or 32 bits integers. + * @return a handler for rewriting the data if the array content changes. + * @throws IOException if an error occurred while writing to the output. + */ + private TagValueWriter writeTag(final short tag, final long[] values) throws IOException { + if (values == null || values.length == 0) { + return null; + } + final short type = isBigTIFF ? TIFF_ULONG : TIFFTag.TIFF_LONG; + return writeLargeTag(tag, type, values.length, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + switch (type) { + default: throw new AssertionError(type); + case TIFF_ULONG: output.writeLongs(values); break; + case TIFFTag.TIFF_LONG: { + for (final long value : values) { + output.writeInt(Math.toIntExact(value)); + } + break; + } + } + } + }); + } + + /** + * Writes a tag with an arbitrary number of values stored as 16 bits integers. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param values the values to write as 16 bits integers. + * @return a handler for rewriting the data if the array content changes. + * @throws IOException if an error occurred while writing to the output. + */ + private TagValueWriter writeTag(final short tag, final short[] values) throws IOException { + if (values == null || values.length == 0) { + return null; + } + return writeLargeTag(tag, (short) TIFFTag.TIFF_SHORT, values.length, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + output.writeShorts(values); + } + }); + } + + /** + * Writes a tag with an arbitrary number of values stored as 16 or 32 bits unsigned integers. + * The list of valid tag codes is defined by TIFF specification. + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param type {@code TIFF_SHORT} or {@code TIFF_LONG}. + * @param values the values to write as unsigned integers. + * @return a handler for rewriting the data if the array content changes. + * @throws IOException if an error occurred while writing to the output. + */ + private TagValueWriter writeTag(final short tag, final short type, final int[] values) throws IOException { + if (values == null || values.length == 0) { + return null; + } + return writeLargeTag(tag, type, values.length, new TagValueWriter() { + @Override void write(final ChannelDataOutput output) throws IOException { + switch (type) { + default: throw new AssertionError(type); + case TIFFTag.TIFF_LONG: output.writeInts(values); break; + case TIFFTag.TIFF_SHORT: { + for (final int value : values) { + output.writeShort(value); + } + break; + } + } + } + }); + } + + /** + * Writes a tag with a single value stored as a 16 or 32 bits unsigned integer. + * The list of valid tag codes is defined by TIFF specification. + * + * <p>The {@code TIFF_LONG} type is preferred when TIFF specification leaves the choice between 16 or 32 bits, + * because the TIFF structure is such as encoding those numbers on 16 bits does not provide any performance or + * space benefit. It was maybe a performance advantage when 16 bits processors were common.</p> + * + * @param tag the code of the tag to write, usually a constant defined by the TIFF specification. + * @param type {@code TIFF_SHORT} or {@code TIFF_LONG}. + * @param value the value to write as an unsigned integer. + * @throws IOException if an error occurred while writing to the output. + */ + private void writeTag(final short tag, final short type, final int value) throws IOException { + writeTagHeader(tag, type, 1); + final ByteBuffer buffer = output.buffer; + switch (type) { + case TIFFTag.TIFF_LONG: { // TIFF "long" is Java `int` but unsigned. + buffer.putInt(value); + break; + } + case TIFFTag.TIFF_SHORT: { + assert value >= 0 && value <= 0xFFFF : value; + buffer.putShort((short) value); // Value shall be left-aligned. + buffer.putShort((short) 0); // This space is lost. + break; + } + default: throw new AssertionError(type); + } + if (isBigTIFF) { + buffer.putInt(0); // Make the slot 64 bits large, left-aligned value. + } + } + + /** + * Executes the given deferred write operation immediately if doing so is cheap, + * or queue the operation for later execution otherwise. + * + * @param value the deferred value to write immediately or later. + * @throws IOException if an error occurred while writing the value. + */ + private void writeOrQueue(final UpdatableWrite<?> value) throws IOException { + if (!value.tryUpdateBuffer(output)) { + deferredWrites.add(value); + } + } + + /** + * Writes deferred values immediately to the output stream. + * + * @throws IOException if an error occurred while writing deferred data. + */ + private void flushDeferredWrites() throws IOException { + for (final UpdatableWrite<?> change : deferredWrites) { + change.update(output); + } + } + + /** + * Sends to the writable channel any information that are still in buffers. + * This method does not flush the writable channel itself. + * + * @throws IOException if an error occurred while closing this writer. + */ + @Override + public void flush() throws IOException { + flushDeferredWrites(); + output.flush(); + } + + /** + * Closes this writer and the associated writable channel. + * + * @throws IOException if an error occurred while closing this writer. + */ + @Override + public void close() throws IOException { + try (output.channel) { + flush(); + } + } +} diff --git a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/package-info.java b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/package-info.java index 21a0f43933..5fd11e01d3 100644 --- a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/package-info.java +++ b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/package-info.java @@ -32,7 +32,7 @@ * @author Thi Phuong Hao Nguyen (VNSC) * @author Minh Chinh Vu (VNSC) * @author Martin Desruisseaux (Geomatys) - * @version 1.4 + * @version 1.5 * @since 0.8 */ package org.apache.sis.storage.geotiff; diff --git a/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java b/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java new file mode 100644 index 0000000000..0f4dcaac3b --- /dev/null +++ b/endorsed/src/org.apache.sis.storage.geotiff/test/org/apache/sis/storage/geotiff/WriterTest.java @@ -0,0 +1,441 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.geotiff; + +import java.util.Set; +import java.util.Arrays; +import java.util.HashSet; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.lang.reflect.Array; +import java.io.OutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.awt.image.DataBuffer; +import java.awt.image.SampleModel; +import javax.imageio.plugins.tiff.TIFFTag; +import org.apache.sis.io.stream.ByteArrayChannel; +import org.apache.sis.io.stream.ChannelDataOutput; +import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.StorageConnector; +import org.apache.sis.coverage.grid.j2d.ColorModelFactory; +import org.apache.sis.image.DataType; +import org.apache.sis.image.TiledImageMock; +import org.apache.sis.test.TestUtilities; +import org.apache.sis.test.TestCase; +import org.junit.Test; + +import static javax.imageio.plugins.tiff.BaselineTIFFTagSet.*; +import static org.junit.jupiter.api.Assertions.*; + + +/** + * Tests {@link Writer}. + * + * @author Erwan Roussel (Geomatys) + * @author Martin Desruisseaux (Geomatys) + */ +public final class WriterTest extends TestCase { + /** + * Arbitrary size (in pixels) of tiles in the image to test. The TIFF specification restricts those sizes + * to multiples of 16, but the Apache SIS implementation has no such restriction. Enforcing a size of 16 + * is necessary, but this is not what we want to test in this class. For making this test easier to debug, + * we want small sizes (less than 10) with different values for width and height. + */ + private static final int TILE_WIDTH = 7, TILE_HEIGHT = 5; + + /** + * The image to write. + */ + private TiledImageMock image; + + /** + * The channel where the image is written. + * The data can be obtained by a call to {@link ByteArrayChannel#toBuffer()}. + */ + private ByteArrayChannel output; + + /** + * The store to use for writing GeoTIFF files. This store should be closed at the end of each test, + * but it is not a problem if they are not because the tests do not hold any resources (no files). + */ + private GeoTiffStore store; + + /** + * The data produced by the GeoTIFF writer. This is the sequence of bytes to verify. + */ + private ByteBuffer data; + + /** + * Index in {@link #data} where each tile begins. + */ + private int[] tileOffsets; + + /** + * Creates a new test case. + */ + public WriterTest() { + } + + /** + * Initializes the test with a tiled image and a GeoTIFF writer. + * The image is created with some random properties (pixel and tile coordinates) but verifiable pixel values. + * The writer uses a buffer of random size. + * + * @param dataType sample data type as one of the {@link java.awt.image.DataBuffer} constants. + * @param order whether to use little endian or big endian byte order. + * @param banded whether to use {@link BandedSampleModel} instead of {@link PixelInterleavedSampleModel}. + * @param numBands number of bands in the sample model to create. + * @param numTileX number of tiles in the X direction. + * @param numTileY number of tiles in the Y direction. + * @param options whether to write classic TIFF or BigTIFF. + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException should never happen since we control the output class. + */ + private void initialize(final DataType type, final ByteOrder order, final boolean banded, final int numBands, + final int numTileX, final int numTileY, final GeoTiffOption... options) + throws IOException, DataStoreException + { + final var random = TestUtilities.createRandomNumberGenerator(); + image = new TiledImageMock(type.toDataBufferType(), numBands, + random.nextInt(16) - 8, // minX + random.nextInt(16) - 8, // minY + TILE_WIDTH * numTileX, + TILE_HEIGHT * numTileY, + TILE_WIDTH, + TILE_HEIGHT, + random.nextInt(16) - 8, // minTileX + random.nextInt(16) - 8, // minTileY + banded); + + image.validate(); + image.initializeAllTiles(); + output = new ByteArrayChannel(new byte[image.getWidth() * image.getHeight() * numBands * type.bytes() + 400], false); + var d = new ChannelDataOutput("TIFF", output, ByteBuffer.allocate(random.nextInt(128) + 20).order(order)); + var c = new StorageConnector(d); + c.setOption(GeoTiffOption.OPTION_KEY, options); + store = new GeoTiffStore(null, c); + data = output.toBuffer().order(order); + } + + /** + * Writes a single image and updates the data buffer limit. + * After this method call, the {@linkplain #data} position is 0 + * and its limit is the number of bytes written by the TIFF encoder. + * + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException if the image is incompatible with writer capability. + */ + @SuppressWarnings("SynchronizeOnNonFinalField") + private void writeImage() throws IOException, DataStoreException { + synchronized (store) { + final Writer writer = store.writer(); + writer.append(image, null); + writer.flush(); + } + data.clear().limit(Math.toIntExact(output.size())); + } + + /** + * Tests the writing a gray scale image made of a single tile with pixels on 8 bits. + * This is the simplest type of image. + * + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException if the image is incompatible with writer capability. + */ + @Test + public void testUntiledGrayScale() throws IOException, DataStoreException { + initialize(DataType.BYTE, ByteOrder.BIG_ENDIAN, false, 1, 1, 1); + writeImage(); + verifyHeader(false, GeoTIFF.BIG_ENDIAN); + verifyImageFileDirectory(Writer.MINIMAL_NUMBER_OF_TAGS, PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO, + new short[] {Byte.SIZE}); + verifySampleValues(1); + store.close(); + } + + /** + * Same as {@link #testUntiledGrayScale()} but using BigTIFF format. + * + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException if the image is incompatible with writer capability. + */ + @Test + public void testUntiledBigTIFF() throws IOException, DataStoreException { + initialize(DataType.BYTE, ByteOrder.LITTLE_ENDIAN, false, 1, 1, 1, GeoTiffOption.BIG_TIFF); + writeImage(); + verifyHeader(true, GeoTIFF.LITTLE_ENDIAN); + verifyImageFileDirectory(Writer.MINIMAL_NUMBER_OF_TAGS, PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO, + new short[] {Byte.SIZE}); + verifySampleValues(1); + store.close(); + } + + /** + * Tests the writing a gray scale image made of a multiple tiles with pixels on 8 bits. + * We arbitrarily write 3 tiles in the X direction and 4 tiles in the Y direction. + * + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException if the image is incompatible with writer capability. + */ + @Test + public void testTiledGrayScale() throws IOException, DataStoreException { + initialize(DataType.BYTE, ByteOrder.LITTLE_ENDIAN, false, 1, 3, 4); + writeImage(); + verifyHeader(false, GeoTIFF.LITTLE_ENDIAN); + verifyImageFileDirectory(Writer.MINIMAL_NUMBER_OF_TAGS, PHOTOMETRIC_INTERPRETATION_BLACK_IS_ZERO, + new short[] {Byte.SIZE}); + verifySampleValues(1); + store.close(); + } + + /** + * Tests the writing an RGB image made of a single tile with pixels on (8,8,8) bits. + * + * @throws IOException should never happen since the tests are writing in memory. + * @throws DataStoreException if the image is incompatible with writer capability. + */ + @Test + public void testUntiledRGB() throws IOException, DataStoreException { + initialize(DataType.BYTE, ByteOrder.LITTLE_ENDIAN, false, 3, 1, 1); + image.setColorModel(ColorModelFactory.createRGB(image.getSampleModel())); + writeImage(); + verifyHeader(false, GeoTIFF.LITTLE_ENDIAN); + verifyImageFileDirectory(Writer.MINIMAL_NUMBER_OF_TAGS, PHOTOMETRIC_INTERPRETATION_RGB, + new short[] {Byte.SIZE, Byte.SIZE, Byte.SIZE}); + verifySampleValues(3); + store.close(); + } + + /** + * Verifies the TIFF header, before the first Image File Directory (IFD). + * + * @param isBigTIFF whether the file is BigTIFF. + * @param endianness {@link Writer#BIG_ENDIAN} or {@link Writer#LITTLE_ENDIAN}. + */ + private void verifyHeader(final boolean isBigTIFF, final short endianness) { + @SuppressWarnings("LocalVariableHidesMemberVariable") + final ByteBuffer data = this.data; + if (isBigTIFF) { + assertEquals(Set.of(GeoTiffOption.BIG_TIFF), store.getOptions()); + assertEquals(endianness, data.getShort()); + assertEquals(Writer.BIG_TIFF, data.getShort()); + assertEquals(Long.BYTES, data.getShort()); // Byte size of offsets. + assertEquals(0, data.getShort()); // Constant. + assertEquals(data.position() + Long.BYTES, data.getLong()); // Offset of the first IFD. + } else { + assertEquals(Set.of(), store.getOptions()); + assertEquals(endianness, data.getShort()); + assertEquals(Writer.CLASSIC, data.getShort()); + assertEquals(data.position() + Integer.BYTES, data.getInt()); // Offset of the first IFD. + } + } + + /** + * Verifies the Image File Directory starting at the given position in the given buffer. + * + * <h4>Limitation</h4> + * For making this method simpler, all TIFF data should be encoded with {@link ByteOrder#LITTLE_ENDIAN}. + * This is a limitation of this verification method, not a limitation of {@link Writer} implementation. + * The reason is that little endian makes possible to invoke, for example, {@link ByteBuffer#getLong()} + * even if the data is actually a left-aligned {@code int} value followed by 4 bytes of padding zeros. + * The same argument applies to {@link ByteBuffer#getShort()} versus {@code short} followed by padding. + * This property works only for little endian. + * + * <p>Above restriction can be relaxed if the TIFF file is classic and the caller known that all values + * verified by this method are {@code int} types, no {@code short} types.</p> + * + * @param tagCount expected number of tags. + * @param interpretation one of {@code PHOTOMETRIC_INTERPRETATION_} constants. + * @param bitsPerSample expected number of bits per sample. The array length is the number of bands. + */ + private void verifyImageFileDirectory(int tagCount, final int interpretation, final short[] bitsPerSample) { + @SuppressWarnings("LocalVariableHidesMemberVariable") + final ByteBuffer data = this.data; + final boolean isTiled = true; + final boolean isBigTIFF = store.getOptions().contains(GeoTiffOption.BIG_TIFF); + final boolean isBigEndian = ByteOrder.BIG_ENDIAN.equals(data.order()); + assertEquals(tagCount, isBigTIFF ? data.getLong() : data.getShort()); + /* + * Build a list of tags considered mandatory for all images in this test. + * Tags not in this set are considered optional. + */ + final var expectedTags = new HashSet<Integer>(Arrays.asList( + TAG_NEW_SUBFILE_TYPE, TAG_IMAGE_WIDTH, TAG_IMAGE_LENGTH, + TAG_COMPRESSION, TAG_PHOTOMETRIC_INTERPRETATION, TAG_SAMPLES_PER_PIXEL, + TAG_X_RESOLUTION, TAG_Y_RESOLUTION, TAG_RESOLUTION_UNIT)); + if (isTiled) { + expectedTags.addAll(Arrays.asList(TAG_TILE_WIDTH, TAG_TILE_LENGTH, TAG_TILE_OFFSETS, TAG_TILE_BYTE_COUNTS)); + } else { + expectedTags.addAll(Arrays.asList(TAG_STRIP_OFFSETS, TAG_STRIP_BYTE_COUNTS)); + } + /* + * Iterate over all tags. Verify that they are in increasing order, + * and verify the value of some of them (not necessarily all of them). + */ + short previousTag = 0; + while (--tagCount >= 0) { + short tag = data.getShort(); + short type = data.getShort(); + long count = isBigTIFF ? data.getLong() : data.getInt(); + long value = isBigTIFF ? data.getLong() : data.getInt(); + Object expected; // The Number class will define the expected type. + assertTrue(tag > previousTag, "Tags shall be sorted in increasing order."); + expectedTags.remove(Integer.valueOf(tag)); + previousTag = tag; + switch (tag) { + case TAG_NEW_SUBFILE_TYPE: expected = 0; break; + case TAG_IMAGE_WIDTH: expected = image.getWidth(); break; + case TAG_IMAGE_LENGTH: expected = image.getHeight(); break; + case TAG_BITS_PER_SAMPLE: expected = compact(bitsPerSample); break; + case TAG_COMPRESSION: expected = (short) COMPRESSION_NONE; break; + case TAG_PHOTOMETRIC_INTERPRETATION: expected = (short) interpretation; break; + case TAG_SAMPLES_PER_PIXEL: expected = (short) bitsPerSample.length; break; + case TAG_RESOLUTION_UNIT: expected = (short) RESOLUTION_UNIT_NONE; break; + case TAG_TILE_WIDTH: expected = TILE_WIDTH; break; + case TAG_TILE_LENGTH: expected = TILE_HEIGHT; break; + case TAG_TILE_BYTE_COUNTS: expected = expectedTileByteCounts(); break; + case TAG_TILE_OFFSETS: { + tileOffsets = getIntegers(value, count, isBigTIFF ? Writer.TIFF_ULONG : TIFFTag.TIFF_LONG); + continue; + } + default: continue; + } + boolean isShort = (expected instanceof Short); + if (isShort & isBigEndian) { + value >>>= Short.SIZE; // Because 16-bits values in tag entries are left-aligned on 32 bits. + } + isShort |= (expected instanceof short[]); + /* + * Compare the actual value with the expected value. The expected value may be an instance + * of `Short`, `Integer`, `short[]` or `int[]`. The class determines the expected TIFF type. + * We do not support all TIFF types, but only the ones that we want to verify. + */ + Supplier<String> message = () -> Tags.name(tag); + assertEquals(isShort ? TIFFTag.TIFF_SHORT : TIFFTag.TIFF_LONG, type, message); + if (expected.getClass().isArray()) { + assertEquals(Array.getLength(expected), count, message); + final int[] actual = getIntegers(value, count, type); + assertEquals(count, actual.length, message); + for (int i=0; i<actual.length; i++) { + assertEquals(Array.getInt(expected, i), actual[i], message); + } + } else { + assertEquals(1, count, message); + assertEquals(((Number) expected).longValue(), value, message); + } + } + /* + * Verify that all mandatory tags were found. + */ + assertNotNull(tileOffsets); + assertTrue(expectedTags.isEmpty(), () -> "Missing mandatory TIFF tags: " + + expectedTags.stream().map((tag) -> Tags.name(tag.shortValue())).collect(Collectors.joining(", "))); + } + + /** + * Returns the given array as a {@link Short} if it contains exactly one element. + * This is necessary for allowing the {@code isShort & isBigEndian} test to work. + */ + private static Object compact(final short[] array) { + return (array.length == 1) ? array[0] : array; + } + + /** + * Returns the values stored in the tag, which may potentially be an array. + * + * @param value value stored in the tag. offsets where to read integers. + * @param count number of integers to read. + * @param type {@code TIFF_ULONG}, {@code TIFF_LONG} or {@code TIFF_SHORT}. + * @return the integers at the given offset in an array of the specified length. + */ + private int[] getIntegers(final long value, final long count, final int type) { + @SuppressWarnings("LocalVariableHidesMemberVariable") + final ByteBuffer data = this.data; + final int[] array = new int[Math.toIntExact(count)]; + if (array.length == 1) { // Ignoring the case of TIFF_SHORT with count of 2. + array[0] = Math.toIntExact(value); // Value was small enough for fitting in the TIFF tag. + } else { + data.mark(); + data.position(Math.toIntExact(value)); + for (int i=0; i<array.length; i++) { // Values are stored in a separated array. + switch (type) { + case Writer.TIFF_ULONG: array[i] = Math.toIntExact(data.getLong()); break; + case TIFFTag.TIFF_LONG: array[i] = data.getInt(); break; + case TIFFTag.TIFF_SHORT: array[i] = data.getShort(); break; + default: throw new AssertionError(type); + } + } + data.reset(); + } + return array; + } + + /** + * {@return the uncompressed size in bytes of each tile}. + */ + private int[] expectedTileByteCounts() { + final SampleModel sm = image.getSampleModel(); + final int[] sizes = new int[image.getNumXTiles() * image.getNumYTiles()]; + Arrays.fill(sizes, TILE_WIDTH * TILE_HEIGHT * sm.getNumBands() * DataBuffer.getDataTypeSize(sm.getDataType()) / Byte.SIZE); + return sizes; + } + + /** + * Verifies the sample values. Expected values are of the form "BTYX" where B is the band (starting with 1), + * T is the tile index (starting with 1), and X and Y are pixel coordinates starting with 0. + * + * @param numBands number of bands in each tile. + */ + private void verifySampleValues(final int numBands) { + @SuppressWarnings("LocalVariableHidesMemberVariable") + final ByteBuffer data = this.data; + for (int i=0; i<tileOffsets.length; i++) { + data.position(tileOffsets[i]); + for (int y=0; y<TILE_HEIGHT; y++) { + for (int x=0; x<TILE_WIDTH; x++) { + for (int b=0; b<numBands; b++) { + int expected = 1000*(b+1) + 100*(i+1) + 10*y + x; + int actual; + expected &= 0xFF; + actual = Byte.toUnsignedInt(data.get()); + assertEquals(expected, actual); + } + } + } + } + } + + /** + * Saves in a file the TIFF images created by the last test executed. + * The file is created in the local directory. + * This method can be used for checking the TIFF file externally. + * + * @throws IOException if an error occurred while writing the file. + */ + @SuppressWarnings("UseOfSystemOutOrSystemErr") + public void save() throws IOException { + final Path path = Path.of("WriterTest.tiff"); + System.out.println("Saving test TIFF image to " + path.toAbsolutePath()); + try (OutputStream s = Files.newOutputStream(path)) { + s.write(data.array(), 0, data.limit()); + } + } +} diff --git a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/MetadataFetcher.java b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/MetadataFetcher.java new file mode 100644 index 0000000000..879b76c1e9 --- /dev/null +++ b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/MetadataFetcher.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.storage.base; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.function.BiPredicate; +import java.util.function.Function; +import org.opengis.util.InternationalString; +import org.opengis.metadata.Metadata; +import org.opengis.metadata.citation.Citation; +import org.opengis.metadata.citation.CitationDate; +import org.opengis.metadata.citation.DateType; +import org.opengis.metadata.citation.Party; +import org.opengis.metadata.citation.Responsibility; +import org.opengis.metadata.citation.Series; +import org.opengis.metadata.identification.Identification; +import org.opengis.metadata.lineage.Lineage; +import org.opengis.metadata.lineage.ProcessStep; +import org.opengis.metadata.acquisition.AcquisitionInformation; +import org.opengis.metadata.acquisition.Instrument; +import org.opengis.metadata.acquisition.Platform; + + +/** + * Helper methods for fetching metadata to be written by {@code DataStore} implementations. + * This is not a general-purpose builder suitable for public API, since the methods provided + * in this class are tailored for Apache SIS data store needs. + * API of this class may change in any future SIS versions. + * + * @author Martin Desruisseaux (Geomatys) + * + * @param <T> type of temporal objects. + */ +public abstract class MetadataFetcher<T> { + /** + * Title of the resource, or {@code null} if none. + * + * <p>Path: {@code metadata/identificationInfo/citation/title}</p> + */ + public List<String> title; + + /** + * Name of the series of which the resource is a part, or {@code null} if none. + * + * <p>Path: {@code metadata/identificationInfo/citation/series/name}</p> + */ + public List<String> series; + + /** + * Details on which pages of the publication the resource was published, or {@code null} if none. + * + * <p>Path: {@code metadata/identificationInfo/citation/series/page}</p> + */ + public List<String> page; + + /** + * Names of the authors, or {@code null} if none. + * + * <p>Path: {@code metadata/identificationInfo/citation/party/name}</p> + */ + public List<String> party; + + /** + * Dates when the resource has been created, or {@code null} if none. + * Limited to a singleton by default. + * + * <p>Path: {@code metadata/identificationInfo/citation/date}</p> + */ + public List<T> creationDate; + + /** + * Unique identification of the measuring instrument, or {@code null} if none. + * + * <p>Path: {@code metadata/acquisitionInformation/platform/instrument/identifier}</p> + */ + public List<String> instrument; + + /** + * Reference to document describing processing software, or {@code null} if none. + * + * <p>Path: {@code metadata/resourceLineage/processStep/processingInformation/softwareReference/title}</p> + */ + public List<String> software; + + /** + * Additional details about the processing procedure, or {@code null} if none. + * + * <p>Path: {@code metadata/resourceLineage/processStep/processingInformation/procedureDescription}</p> + */ + public List<String> procedure; + + /** + * The locale to use for converting international strings to {@link String} objects. + * May also be used for date or number formatting. + */ + protected final Locale locale; + + /** + * Creates an initially empty metadata fetcher. + * + * @param locale the locale to use for converting international strings to {@link String} objects. + */ + public MetadataFetcher(final Locale locale) { + this.locale = locale; + } + + /** + * Invokes {@code this.accept(E)} for all elements in the given collection. + * This method ignores null values (this is a paranoiac safety) and stops + * the iteration if an {@code accept(E)} call returns {@code true}. + * + * @param <E> type of elements in the method. + * @param accept the method to invoke for each element. + * @param elements the collection of elements, or {@code null} if none. + */ + private <E> void forEach(final BiPredicate<MetadataFetcher<T>,E> accept, final Iterable<? extends E> elements) { + if (elements != null) { + for (final E info : elements) { + if (info != null && accept.test(this, info)) break; + } + } + } + + /** + * Fetches some properties from the given metadata object. + * The default implementation iterates over the {@link Identification}, {@link Lineage} and + * {@link AcquisitionInformation} objects, filters null values (this is a paranoiac safety), + * then delegate to the corresponding {@code accept(…)} method. + * + * @param info the metadata, or {@code null} if none. + */ + public void accept(final Metadata info) { + if (info != null) { + forEach(MetadataFetcher::accept, info.getIdentificationInfo()); + forEach(MetadataFetcher::accept, info.getResourceLineages()); + forEach(MetadataFetcher::accept, info.getAcquisitionInformation()); + } + } + + /** + * Fetches some properties from the given identification object. + * Subclasses can override if they need to fetch more details. + * + * @param info the identification object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final Identification info) { + final Citation citation = info.getCitation(); + if (citation == null) { + return false; + } + title = addString(title, citation.getTitle()); + final Series e = citation.getSeries(); + if (e != null) { + series = addString(series, e.getName()); + page = addString(page, e.getPage()); + } + forEach(MetadataFetcher::accept, citation.getCitedResponsibleParties()); + forEach(MetadataFetcher::accept, citation.getDates()); + return title != null; + } + + /** + * Fetches some properties from the given responsible party. + * Subclasses can override if they need to fetch more details. + * + * @param info the responsible party (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final Responsibility info) { + party = addStrings(party, info.getParties(), Party::getName); + return false; + } + + /** + * Fetches some properties from the given resource citation date. + * Subclasses can override if they need to fetch more details. + * + * @param info the resource citation date (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final CitationDate info) { + if (creationDate == null) { + if (DateType.CREATION.equals(info.getDateType())) { + creationDate = List.of(parseDate(info.getDate())); + } else { + return false; // Search another date. + } + } + return true; + } + + /** + * Fetches some properties from the given lineage object. + * Subclasses can override if they need to fetch more details. + * + * @param info the lineage object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final Lineage info) { + forEach(MetadataFetcher::accept, info.getProcessSteps()); + return false; + } + + /** + * Fetches some properties from the given process step. + * Subclasses can override if they need to fetch more details. + * + * @param info the process step object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final ProcessStep info) { + final var processing = info.getProcessingInformation(); + if (processing != null) { + software = addStrings(software, processing.getSoftwareReferences(), Citation::getTitle); + procedure = addString (procedure, processing.getProcedureDescription()); + } + return false; + } + + /** + * Fetches some properties from the given acquisition object. + * Subclasses can override if they need to fetch more details. + * + * @param info the acquisition object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final AcquisitionInformation info) { + forEach(MetadataFetcher::accept, info.getPlatforms()); + return false; + } + + /** + * Fetches some properties from the given platform object. + * Subclasses can override if they need to fetch more details. + * + * @param info the platform object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final Platform info) { + forEach(MetadataFetcher::accept, info.getInstruments()); + return false; + } + + /** + * Fetches some properties from the given instrument object. + * Subclasses can override if they need to fetch more details. + * + * @param info the instrument object (not null). + * @return whether to stop iteration after the given object. + */ + protected boolean accept(final Instrument info) { + final var id = info.getIdentifier(); + if (id != null) { + instrument = addString(instrument, id.getCode()); + } + return false; + } + + /** + * Adds all international strings in the given collection. + * + * @param <E> type of elements in the collection. + * @param target where to add strings, or {@code null} if not yet created. + * @param source elements from which to get international string, or {@code null} if none. + * @param getter method to invoke on each element for getting the international string. + * @return the collection where strings were added. + */ + private <E> List<String> addStrings(List<String> target, final Iterable<? extends E> source, + final Function<E,InternationalString> getter) + { + if (source != null) { + for (final E e : source) { + if (e != null) { + target = addString(target, getter.apply(e)); + } + } + } + return target; + } + + /** + * Adds the given international string in the given collection. + * + * @param target where to add the string, or {@code null} if not yet created. + * @param value the international string to add, or {@code null} if none. + * @return the collection where the string was added. + */ + private List<String> addString(List<String> target, final InternationalString value) { + if (value != null) { + target = addString(target, value.toString(locale)); + } + return target; + } + + /** + * Adds the given string in the given collection. + * + * @param target where to add the string, or {@code null} if not yet created. + * @param value the string to add, or {@code null} if none. + * @return the collection where the string was added. + */ + private static List<String> addString(List<String> target, String value) { + if (value != null && !(value = value.trim()).isEmpty()) { + if (target == null) { + target = new ArrayList<>(2); // We will usually have only one element. + } + target.add(value); + } + return target; + } + + /** + * Converts the given date into the object to store. + * The {@code <T>} type may be for example {@code <String>} + * with a string representation specified by the format implemented by the store. + * + * @param date the date to convert. + * @return subclass-dependent object representing the given date. + */ + protected abstract T parseDate(final Date date); +} diff --git a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/internal/Numerics.java b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/internal/Numerics.java index c950e11c20..4b72c74232 100644 --- a/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/internal/Numerics.java +++ b/endorsed/src/org.apache.sis.util/main/org/apache/sis/util/internal/Numerics.java @@ -106,6 +106,12 @@ public final class Numerics extends Static { */ public static final long SIGN_BIT_MASK = Long.MIN_VALUE; + /** + * Mask for the highest 32 bits of a long integers. + * It can be used for checking if a {@code long} can be casted as an unsigned integer. + */ + public static final long HIGH_BITS_MASK = ~((1L << Integer.SIZE) - 1); + /** * Number of bits in the significand (mantissa) part of IEEE 754 {@code double} representation, * <strong>not</strong> including the hidden bit.