This is an automated email from the ASF dual-hosted git repository. desruisseaux pushed a commit to branch geoapi-4.0 in repository https://gitbox.apache.org/repos/asf/sis.git
commit 42473c5cd0864ad54555e4850d558abe7bb80e5b Author: Martin Desruisseaux <martin.desruisse...@geomatys.com> AuthorDate: Sun Nov 7 16:17:25 2021 +0100 More filtering of values on which statistics are computed: - Added a `DoubleUnaryOperator[] sampleFilters` argument as a way to specify "no data" values to ignore. - Finer check of `areaOfInterest` shape for determining which tiles need to be computed (optimization). --- .../org/apache/sis/gui/coverage/RenderingData.java | 3 +- .../apache/sis/internal/gui/ImageConverter.java | 3 +- .../java/org/apache/sis/image/AnnotatedImage.java | 50 ++++++++-- .../java/org/apache/sis/image/ImageProcessor.java | 110 +++++++++++++++----- .../java/org/apache/sis/image/PlanarImage.java | 5 +- .../java/org/apache/sis/image/RecoloredImage.java | 4 +- .../org/apache/sis/image/StatisticsCalculator.java | 111 +++++++++++++++++++-- .../java/org/apache/sis/image/Visualization.java | 3 +- .../sis/internal/coverage/j2d/TileOpExecutor.java | 100 ++++++++++++++++--- .../apache/sis/image/StatisticsCalculatorTest.java | 99 ++++++++++++++++-- .../main/java/org/apache/sis/util/ArraysExt.java | 2 +- 11 files changed, 418 insertions(+), 72 deletions(-) diff --git a/application/sis-javafx/src/main/java/org/apache/sis/gui/coverage/RenderingData.java b/application/sis-javafx/src/main/java/org/apache/sis/gui/coverage/RenderingData.java index 646720b..09699c9 100644 --- a/application/sis-javafx/src/main/java/org/apache/sis/gui/coverage/RenderingData.java +++ b/application/sis-javafx/src/main/java/org/apache/sis/gui/coverage/RenderingData.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.HashMap; import java.util.List; import java.util.concurrent.Future; +import java.util.function.DoubleUnaryOperator; import java.io.IOException; import java.io.UncheckedIOException; import java.awt.Graphics2D; @@ -289,7 +290,7 @@ final class RenderingData implements Cloneable { if (selectedDerivative != Stretching.NONE) { final Map<String,Object> modifiers = new HashMap<>(4); if (statistics == null) { - statistics = processor.valueOfStatistics(image, null); + statistics = processor.valueOfStatistics(image, null, (DoubleUnaryOperator[]) null); } modifiers.put("statistics", statistics); if (selectedDerivative == Stretching.AUTOMATIC) { diff --git a/application/sis-javafx/src/main/java/org/apache/sis/internal/gui/ImageConverter.java b/application/sis-javafx/src/main/java/org/apache/sis/internal/gui/ImageConverter.java index 86a58d6..6868f29 100644 --- a/application/sis-javafx/src/main/java/org/apache/sis/internal/gui/ImageConverter.java +++ b/application/sis-javafx/src/main/java/org/apache/sis/internal/gui/ImageConverter.java @@ -17,6 +17,7 @@ package org.apache.sis.internal.gui; import java.util.Map; +import java.util.function.DoubleUnaryOperator; import java.awt.Color; import java.awt.Graphics2D; import java.awt.Rectangle; @@ -126,7 +127,7 @@ final class ImageConverter extends Task<Statistics[]> { toCanvas.translate(-bounds.x, -bounds.y); final ImageProcessor processor = new ImageProcessor(); - final Statistics[] statistics = processor.valueOfStatistics(source, bounds); + final Statistics[] statistics = processor.valueOfStatistics(source, bounds, (DoubleUnaryOperator[]) null); final RenderedImage image = processor.stretchColorRamp(source, JDK9.mapOf("multStdDev", 3, "statistics", statistics)); final RenderedImage mask = getMask(processor); final BufferedImage buffer = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB_PRE); diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/AnnotatedImage.java b/core/sis-feature/src/main/java/org/apache/sis/image/AnnotatedImage.java index e7e70a0..3c1d7e7 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/AnnotatedImage.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/AnnotatedImage.java @@ -17,6 +17,7 @@ package org.apache.sis.image; import java.util.Locale; +import java.util.Arrays; import java.util.Objects; import java.util.WeakHashMap; import java.util.logging.Level; @@ -56,7 +57,7 @@ import org.apache.sis.internal.util.Strings; * on the fact that it can unwrap this image and still get the same pixel values.</div> * * @author Martin Desruisseaux (Geomatys) - * @version 1.1 + * @version 1.2 * @since 1.1 * @module */ @@ -98,7 +99,7 @@ abstract class AnnotatedImage extends ImageAdapter { * those results are replaced by {@link #NULL}.</p> * * <p>Keys are {@link String} instances containing directly the property name when {@link #areaOfInterest} - * is {@code null}, or {@link CacheKey} instances otherwise.</p> + * and {@link #getExtraParameter()} are {@code null}, or {@link CacheKey} instances otherwise.</p> */ private final Cache<Object,Object> cache; @@ -109,25 +110,34 @@ abstract class AnnotatedImage extends ImageAdapter { /** The property name (never null). */ private final String property; - /** The area of interest (never null). */ + /** The area of interest, or null if none. */ private final Shape areaOfInterest; + /** Parameter specific to subclass, or null if none. */ + private final Object[] extraParameter; + /** Creates a new key for the given property and AOI. */ - CacheKey(final String property, final Shape areaOfInterest) { + CacheKey(final String property, final Shape areaOfInterest, final Object[] extraParameter) { this.property = property; this.areaOfInterest = areaOfInterest; + this.extraParameter = extraParameter; } /** Returns a hash code value for this key. */ @Override public int hashCode() { - return property.hashCode() + 19 * areaOfInterest.hashCode(); + return property.hashCode() + + 19 * Objects.hashCode(areaOfInterest) + + 37 * Arrays.hashCode(extraParameter); + } /** Compares this key with the given object for equality. */ @Override public boolean equals(final Object obj) { if (obj instanceof CacheKey) { final CacheKey other = (CacheKey) obj; - return property.equals(other.property) && areaOfInterest.equals(areaOfInterest); + return property.equals(other.property) + && Objects.equals(areaOfInterest, other.areaOfInterest) + && Arrays.equals(extraParameter, other.extraParameter); } return false; } @@ -244,6 +254,22 @@ abstract class AnnotatedImage extends ImageAdapter { } /** + * Returns an optional parameter specific to subclass. This is used for caching purpose + * and for {@link #equals(Object)} and {@link #hashCode()} method implementations only, + * i.e. for distinguishing between two {@code AnnotatedImage} instances that are identical + * except for subclass-defined parameters. + * + * <div class="note"><b>API note:</b> + * the return value is an array because there is typically one parameter value per band. + * This method will not modify the returned array.</div> + * + * @return subclass specific extra parameter, or {@code null} if none. + */ + Object[] getExtraParameter() { + return null; + } + + /** * If the source image is the same operation for the same area of interest, returns that source. * Otherwise returns {@code this} or a previous instance doing the same operation than {@code this}. * @@ -263,7 +289,9 @@ abstract class AnnotatedImage extends ImageAdapter { * @param property value of {@link #getPropertyNames()}. */ private Object getCacheKey(final String property) { - return (areaOfInterest != null) ? new CacheKey(property, areaOfInterest) : property; + final Object[] extraParameter = getExtraParameter(); + return (areaOfInterest != null || extraParameter != null) + ? new CacheKey(property, areaOfInterest, extraParameter) : property; } /** @@ -421,7 +449,8 @@ abstract class AnnotatedImage extends ImageAdapter { if (!failOnException) { executor.setErrorHandler((e) -> errors = e, AnnotatedImage.class, "getProperty"); } - return executor.executeOnReadable(source, collector()); + executor.setAreaOfInterest(source, areaOfInterest); + return executor.executeOnReadable(source, collector); } } } @@ -533,7 +562,8 @@ abstract class AnnotatedImage extends ImageAdapter { * The {@link #errors} is omitted because it is part of computation results. */ private boolean equalParameters(final AnnotatedImage other) { - return Objects.equals(areaOfInterest, other.areaOfInterest) && - parallel == other.parallel && failOnException == other.failOnException; + return parallel == other.parallel && failOnException == other.failOnException + && Objects.equals(areaOfInterest, other.areaOfInterest) + && Arrays.equals(getExtraParameter(), other.getExtraParameter()); } } diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/ImageProcessor.java b/core/sis-feature/src/main/java/org/apache/sis/image/ImageProcessor.java index 866e754..4585a76 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/ImageProcessor.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/ImageProcessor.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.NavigableMap; import java.util.function.Function; import java.util.logging.LogRecord; +import java.util.function.DoubleUnaryOperator; import java.awt.Color; import java.awt.Shape; import java.awt.Rectangle; @@ -537,19 +538,51 @@ public class ImageProcessor implements Cloneable { } /** + * Builds an operator which can be used for filtering "no data" sample values. + * Calls to the operator {@code applyAsDouble(x)} will return {@link Double#NaN} + * if the <var>x</var> value is equal to one of the given no-data {@code values}, + * and will return <var>x</var> unchanged otherwise. + * + * <h4>Usage</h4> + * This operator can be used as a {@code sampleFilters} argument in calls to + * {@link #statistics statistics(…)} or {@link #valueOfStatistics valueOfStatistics(…)} methods. + * It is redundant with {@linkplain SampleDimension#getTransferFunction() transfer function} work, + * but can be useful for images not managed by a {@link org.apache.sis.coverage.grid.GridCoverage}. + * + * @param values the "no data" values, or {@code null} if none. Null and NaN elements are ignored. + * @return an operator for filtering the given "no data" values, + * or {@code null} if there is no non-NaN value to filter. + * + * @see #statistics(RenderedImage, Shape, DoubleUnaryOperator[]) + * @see SampleDimension#getTransferFunction() + * + * @since 1.2 + */ + public DoubleUnaryOperator filterNodataValues(final Number... values) { + return (values != null) ? StatisticsCalculator.filterNodataValues(values) : null; + } + + /** * Returns statistics (minimum, maximum, mean, standard deviation) on each bands of the given image. - * Invoking this method is equivalent to invoking {@link #statistics(RenderedImage, Shape)} and - * extracting immediately the statistics property value, except that errors are handled by the - * {@linkplain #getErrorHandler() error handler}. + * Invoking this method is equivalent to invoking the {@link #statistics statistics(…)} method and + * extracting immediately the statistics property value, except that custom + * {@linkplain #setErrorHandler error handlers} are supported. * - * <p>If {@code areaOfInterest} is {@code null}, then the default is as below:</p> + * <p>If {@code areaOfInterest} is null and {@code sampleFilters} is {@code null} or empty, + * then the default behavior is as below:</p> * <ul> - * <li>If the {@value StatisticsCalculator#STATISTICS_KEY} property value exists in the given image, + * <li>If the {@value PlanarImage#STATISTICS_KEY} property value exists in the given image, * then that value is returned. Note that they are not necessarily statistics for the whole image. - * They are whatever statistics the property provided considered as representative.</li> + * They are whatever statistics the property provider considered as representative.</li> * <li>Otherwise statistics are computed for the whole image.</li> * </ul> * + * <h4>Sample converters</h4> + * An arbitrary {@link DoubleUnaryOperator} can be applied on sample values before to add them to statistics. + * The main purpose is to replace "no-data values" by {@link Double#NaN} values for instructing + * {@link Statistics#accept(double)} to ignore them. The {@link #filterNodataValues(Number...)} + * convenience method can be used for building an operator filtering "no data" sample values. + * * <h4>Properties used</h4> * This operation uses the following properties in addition to method parameters: * <ul> @@ -563,17 +596,23 @@ public class ImageProcessor implements Cloneable { * * @param source the image for which to compute statistics. * @param areaOfInterest pixel coordinates of the area of interest, or {@code null} for the default. + * @param sampleFilters converters to apply on sample values before to add them to statistics, or + * {@code null} or an empty array if none. The array may have any length and may contain null elements. + * For all {@code i < numBands}, non-null {@code sampleFilters[i]} are applied to band <var>i</var>. * @return the statistics of sample values in each band. * @throws ImagingOpException if an error occurred during calculation * and the error handler is {@link ErrorHandler#THROW}. * - * @see #statistics(RenderedImage, Shape) - * @see StatisticsCalculator#STATISTICS_KEY + * @see #statistics(RenderedImage, Shape, DoubleUnaryOperator...) + * @see #filterNodataValues(Number...) + * @see PlanarImage#STATISTICS_KEY */ - public Statistics[] valueOfStatistics(final RenderedImage source, final Shape areaOfInterest) { + public Statistics[] valueOfStatistics(final RenderedImage source, final Shape areaOfInterest, + final DoubleUnaryOperator... sampleFilters) + { ArgumentChecks.ensureNonNull("source", source); - if (areaOfInterest == null) { - final Object property = source.getProperty(StatisticsCalculator.STATISTICS_KEY); + if (areaOfInterest == null && (sampleFilters == null || ArraysExt.allEquals(sampleFilters, null))) { + final Object property = source.getProperty(PlanarImage.STATISTICS_KEY); if (property instanceof Statistics[]) { return (Statistics[]) property; } @@ -590,8 +629,8 @@ public class ImageProcessor implements Cloneable { * The way AnnotatedImage cache mechanism is implemented, if statistics results already * exist, they will be used. */ - final AnnotatedImage calculator = new StatisticsCalculator(source, areaOfInterest, parallel, failOnException); - final Object property = calculator.getProperty(StatisticsCalculator.STATISTICS_KEY); + final AnnotatedImage calculator = new StatisticsCalculator(source, areaOfInterest, sampleFilters, parallel, failOnException); + final Object property = calculator.getProperty(PlanarImage.STATISTICS_KEY); calculator.logAndClearError(ImageProcessor.class, "valueOfStatistics", errorListener); return (Statistics[]) property; } @@ -600,16 +639,33 @@ public class ImageProcessor implements Cloneable { * Returns an image with statistics (minimum, maximum, mean, standard deviation) on each bands. * The property value will be computed when first requested (it is not computed immediately by this method). * - * <p>If {@code areaOfInterest} is {@code null}, then the default is as below:</p> + * <p>If {@code areaOfInterest} is null and {@code sampleFilters} is {@code null} or empty, + * then the default is as below:</p> * <ul> - * <li>If the {@value StatisticsCalculator#STATISTICS_KEY} property value exists in the given image, + * <li>If the {@value PlanarImage#STATISTICS_KEY} property value exists in the given image, * then that image is returned as-is. Note that the existing property value is not necessarily * statistics for the whole image. * They are whatever statistics the property provider considers as representative.</li> - * <li>Otherwise an image augmented with a {@value StatisticsCalculator#STATISTICS_KEY} property value + * <li>Otherwise an image augmented with a {@value PlanarImage#STATISTICS_KEY} property value * is returned.</li> * </ul> * + * <h4>Sample converters</h4> + * An arbitrary {@link DoubleUnaryOperator} can be applied on sample values before to add them to statistics. + * The main purpose is to replace "no-data values" by {@link Double#NaN} values for instructing + * {@link Statistics#accept(double)} to ignore them. The {@link #filterNodataValues(Number...)} + * convenience method can be used for building an operator filtering "no data" sample values. + * + * <div class="note"><b>API design note:</b> + * the {@code areaOfInterest} and {@code sampleFilters} arguments are complementary. + * Both of them filter the data accepted for statistics. In ISO 19123 terminology, + * the {@code areaOfInterest} argument filters the <cite>coverage domain</cite> while + * the {@code sampleFilters} argument filters the <cite>coverage range</cite>. + * Another connection with OGC/ISO standards is that {@link DoubleUnaryOperator} in this context + * does the same work than {@linkplain SampleDimension#getTransferFunction() transfer function}. + * It can be useful for images not managed by a {@link org.apache.sis.coverage.grid.GridCoverage}. + * </div> + * * <h4>Properties used</h4> * This operation uses the following properties in addition to method parameters: * <ul> @@ -619,15 +675,23 @@ public class ImageProcessor implements Cloneable { * * @param source the image for which to provide statistics. * @param areaOfInterest pixel coordinates of the area of interest, or {@code null} for the default. - * @return an image with an {@value StatisticsCalculator#STATISTICS_KEY} property. + * @param sampleFilters converters to apply on sample values before to add them to statistics, or + * {@code null} or an empty array if none. The array may have any length and may contain null elements. + * For all {@code i < numBands}, non-null {@code sampleFilters[i]} are applied to band <var>i</var>. + * @return an image with an {@value PlanarImage#STATISTICS_KEY} property. * May be {@code image} if the given argument already has a statistics property. * - * @see #valueOfStatistics(RenderedImage, Shape) - * @see StatisticsCalculator#STATISTICS_KEY + * @see #valueOfStatistics(RenderedImage, Shape, DoubleUnaryOperator...) + * @see #filterNodataValues(Number...) + * @see PlanarImage#STATISTICS_KEY */ - public RenderedImage statistics(final RenderedImage source, final Shape areaOfInterest) { + public RenderedImage statistics(final RenderedImage source, final Shape areaOfInterest, + final DoubleUnaryOperator... sampleFilters) + { ArgumentChecks.ensureNonNull("source", source); - if (areaOfInterest == null && ArraysExt.contains(source.getPropertyNames(), StatisticsCalculator.STATISTICS_KEY)) { + if (areaOfInterest == null && (sampleFilters == null || ArraysExt.allEquals(sampleFilters, null)) + && ArraysExt.contains(source.getPropertyNames(), PlanarImage.STATISTICS_KEY)) + { return source; } final boolean parallel, failOnException; @@ -635,7 +699,7 @@ public class ImageProcessor implements Cloneable { parallel = parallel(source); failOnException = failOnException(); } - return new StatisticsCalculator(source, areaOfInterest, parallel, failOnException).unique(); + return new StatisticsCalculator(source, areaOfInterest, sampleFilters, parallel, failOnException).unique(); } /** @@ -647,7 +711,7 @@ public class ImageProcessor implements Cloneable { * mapped to their colors. * * <p>The minimum and maximum value can be either specified explicitly, - * or determined from {@link #valueOfStatistics(RenderedImage, Shape) statistics} on the image. + * or determined from {@link #valueOfStatistics statistics} on the image. * In the later case a range of value is determined first from the {@linkplain Statistics#minimum() minimum} * and {@linkplain Statistics#maximum() maximum} values found in the image, optionally narrowed to an interval * of some {@linkplain Statistics#standardDeviation(boolean) standard deviations} around the mean value.</p> diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/PlanarImage.java b/core/sis-feature/src/main/java/org/apache/sis/image/PlanarImage.java index 0dd5392..56046c1 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/PlanarImage.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/PlanarImage.java @@ -26,6 +26,7 @@ import java.awt.image.WritableRaster; import java.awt.image.WritableRenderedImage; import java.awt.image.RenderedImage; import java.util.Vector; +import java.util.function.DoubleUnaryOperator; import org.apache.sis.util.Classes; import org.apache.sis.util.ArgumentChecks; import org.apache.sis.util.resources.Errors; @@ -162,12 +163,12 @@ public abstract class PlanarImage implements RenderedImage { * * <p>Values should be instances of <code>{@linkplain org.apache.sis.math.Statistics}[]</code>. * The array length should be the number of bands. If this property is not provided, Apache SIS - * may have to {@linkplain ImageProcessor#statistics(RenderedImage, Shape) compute statistics itself} + * may have to {@linkplain ImageProcessor#statistics compute statistics itself} * (by iterating over pixel values) when needed.</p> * * <p>Statistics are only indicative. They may be computed on an image sub-region.</p> * - * @see ImageProcessor#statistics(RenderedImage, Shape) + * @see ImageProcessor#statistics(RenderedImage, Shape, DoubleUnaryOperator...) */ public static final String STATISTICS_KEY = "org.apache.sis.Statistics"; diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/RecoloredImage.java b/core/sis-feature/src/main/java/org/apache/sis/image/RecoloredImage.java index d97aa1b..9a6dafc 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/RecoloredImage.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/RecoloredImage.java @@ -19,6 +19,7 @@ package org.apache.sis.image; import java.util.Map; import java.util.List; import java.util.Arrays; +import java.util.function.DoubleUnaryOperator; import java.awt.Shape; import java.awt.image.ColorModel; import java.awt.image.IndexColorModel; @@ -179,7 +180,8 @@ final class RecoloredImage extends ImageAdapter { if (statsAllBands == null) { final Object areaOfInterest = modifiers.get("areaOfInterest"); statsAllBands = processor.valueOfStatistics(statsSource, - (areaOfInterest instanceof Shape) ? (Shape) areaOfInterest : null); + (areaOfInterest instanceof Shape) ? (Shape) areaOfInterest : null, + (DoubleUnaryOperator[]) null); } if (statsAllBands != null && visibleBand < statsAllBands.length) { statistics = statsAllBands[visibleBand]; diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/StatisticsCalculator.java b/core/sis-feature/src/main/java/org/apache/sis/image/StatisticsCalculator.java index 75bbd36..32ea82a 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/StatisticsCalculator.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/StatisticsCalculator.java @@ -20,9 +20,14 @@ import java.awt.Shape; import java.awt.image.Raster; import java.awt.image.RenderedImage; import java.awt.image.ImagingOpException; +import java.util.Arrays; +import java.util.function.DoubleConsumer; +import java.util.function.DoubleUnaryOperator; import java.util.stream.Collector; import org.apache.sis.math.Statistics; +import org.apache.sis.util.ArraysExt; import org.apache.sis.util.resources.Vocabulary; +import org.apache.sis.internal.coverage.j2d.ImageUtilities; /** @@ -31,23 +36,49 @@ import org.apache.sis.util.resources.Vocabulary; * The statistics can be computed in parallel or sequentially for non thread-safe images. * * @author Martin Desruisseaux (Geomatys) - * @version 1.1 + * @version 1.2 * @since 1.1 * @module */ final class StatisticsCalculator extends AnnotatedImage { /** + * An optional function for converting values before to add them to statistics. + * The main purpose is to exclude "no data" values by replacing them by NaN. + * If non-null, the length of this array must be equal to the number of bands. + * Array may contain null elements if no filter should be applied for a band. + */ + private final DoubleUnaryOperator[] sampleFilters; + + /** * Creates a new calculator. * * @param image the image for which to compute statistics. * @param areaOfInterest pixel coordinates of AOI, or {@code null} for the whole image. + * @param sampleFilters converters to apply on sample values before to add them to statistics, or {@code null}. * @param parallel whether parallel execution is authorized. * @param failOnException whether errors occurring during computation should be propagated. */ - StatisticsCalculator(final RenderedImage image, final Shape areaOfInterest, + StatisticsCalculator(final RenderedImage image, final Shape areaOfInterest, DoubleUnaryOperator[] sampleFilters, final boolean parallel, final boolean failOnException) { super(image, areaOfInterest, parallel, failOnException); + if (sampleFilters != null) { + sampleFilters = Arrays.copyOf(sampleFilters, ImageUtilities.getNumBands(image)); + if (ArraysExt.allEquals(sampleFilters, null)) { + sampleFilters = null; + } + } + this.sampleFilters = sampleFilters; + } + + /** + * Returns the optional filter parameter. This is used by parent class for caching + * and for {@link #equals(Object)} and {@link #hashCode()} implementations. + */ + @Override + @SuppressWarnings("ReturnOfCollectionOrArrayField") // Caller will not modify. + final Object[] getExtraParameter() { + return sampleFilters; } /** @@ -72,6 +103,26 @@ final class StatisticsCalculator extends AnnotatedImage { } /** + * Returns accumulators which will apply the {@link #filter} before to add values to statistics. + * If there is no filter to apply, then this method returns the {@code accumulator} array directly. + * + * @param accumulator where to accumulate the statistics results. + * @return the accumulator optionally filtered. + */ + private final DoubleConsumer[] filtered(final Statistics[] accumulator) { + if (sampleFilters == null) { + return accumulator; + } + final DoubleConsumer[] filtered = new DoubleConsumer[accumulator.length]; + for (int i=0; i<filtered.length; i++) { + final DoubleConsumer c = accumulator[i]; + final DoubleUnaryOperator f = sampleFilters[i]; + filtered[i] = (f == null) ? c : (v) -> c.accept(f.applyAsDouble(v)); + } + return filtered; + } + + /** * Computes statistics using the given iterator and accumulates the result for all bands. * This method is invoked in both sequential and parallel case. In the sequential case it * is invoked for the whole image; in the parallel case it is invoked for only one tile. @@ -82,7 +133,7 @@ final class StatisticsCalculator extends AnnotatedImage { * @param accumulator where to accumulate the statistics results. * @param it the iterator on a raster or on the whole image. */ - private void compute(final Statistics[] accumulator, final PixelIterator it) { + private void compute(final DoubleConsumer[] accumulator, final PixelIterator it) { double[] samples = null; while (it.next()) { if (areaOfInterest == null || areaOfInterest.contains(it.x, it.y)) { @@ -103,7 +154,7 @@ final class StatisticsCalculator extends AnnotatedImage { protected Object computeSequentially() { final PixelIterator it = new PixelIterator.Builder().setRegionOfInterest(boundsOfInterest).create(source); final Statistics[] accumulator = createAccumulator(it.getNumBands()); - compute(accumulator, it); + compute(filtered(accumulator), it); return accumulator; } @@ -134,10 +185,10 @@ final class StatisticsCalculator extends AnnotatedImage { * This method will be invoked for each worker thread before the worker starts its execution. * * @return a thread-local variable holding information computed by a single thread. - * May be {@code null} is such objects are not needed. + * May be {@code null} if such objects are not needed. */ private Statistics[] createAccumulator() { - return createAccumulator(source.getSampleModel().getNumBands()); + return createAccumulator(ImageUtilities.getNumBands(source)); } /** @@ -166,6 +217,52 @@ final class StatisticsCalculator extends AnnotatedImage { * @throws RuntimeException if the calculation failed. */ private void compute(final Statistics[] accumulator, final Raster tile) { - compute(accumulator, new PixelIterator.Builder().setRegionOfInterest(boundsOfInterest).create(tile)); + compute(filtered(accumulator), new PixelIterator.Builder().setRegionOfInterest(boundsOfInterest).create(tile)); + } + + /** + * Builds an operator which can be used for filtering "no data" values. Calls to {@code applyAsDouble(x)} + * on the returned operator will return {@link Double#NaN} if the <var>x</var> value is equal to one of + * the given no-data {@code values}, and will return <var>x</var> unchanged otherwise. + * This operator can be used as a {@code sampleFilters} argument in calls to + * {@link #StatisticsCalculator(RenderedImage, Shape, DoubleUnaryOperator[], boolean, boolean)}. + * + * @param values the "no data" values. Null and NaN elements are ignored. + * @return an operator for filtering the given "no data" values, + * or {@code null} if there is no non-NaN value to filter. + */ + static DoubleUnaryOperator filterNodataValues(final Number[] values) { + final double[] retained = new double[values.length]; + int count = 0; + for (final Number v : values) { + if (v != null) { + retained[count++] = v.doubleValue(); + } + } + Arrays.sort(retained, 0, count); + final int n = count; + count = 0; + for (int i=0; i<n; i++) { + final double v = retained[i]; + if (Double.isNaN(v)) break; // NaN values are sorted last. + if (count != 0) { + if (retained[count-1] == v) continue; // Skip duplicated values. + retained[count] = v; + } + count++; + } + switch (count) { + case 0: { + return null; + } + case 1: { + final double nodata = retained[0]; + return (v) -> (nodata != v) ? v : Double.NaN; + } + default: { + final double[] nodata = ArraysExt.resize(retained, count); + return (v) -> Arrays.binarySearch(nodata, v) < 0 ? v : Double.NaN; + } + } } } diff --git a/core/sis-feature/src/main/java/org/apache/sis/image/Visualization.java b/core/sis-feature/src/main/java/org/apache/sis/image/Visualization.java index 785056c..8ba574b 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/image/Visualization.java +++ b/core/sis-feature/src/main/java/org/apache/sis/image/Visualization.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.List; import java.util.Collection; import java.util.function.Function; +import java.util.function.DoubleUnaryOperator; import java.awt.Color; import java.awt.Dimension; import java.awt.Rectangle; @@ -243,7 +244,7 @@ final class Visualization extends ResampledImage { * If none of above Colorizer configurations worked, use statistics in last resort. We do that * after we reduced the image to a single band, in order to reduce the amount of calculations. */ - final Statistics statistics = processor.valueOfStatistics(source, null)[VISIBLE_BAND]; + final Statistics statistics = processor.valueOfStatistics(source, null, (DoubleUnaryOperator[]) null)[VISIBLE_BAND]; colorizer.initialize(statistics.minimum(), statistics.maximum()); } /* diff --git a/core/sis-feature/src/main/java/org/apache/sis/internal/coverage/j2d/TileOpExecutor.java b/core/sis-feature/src/main/java/org/apache/sis/internal/coverage/j2d/TileOpExecutor.java index 3527e8b..f6f3e7b 100644 --- a/core/sis-feature/src/main/java/org/apache/sis/internal/coverage/j2d/TileOpExecutor.java +++ b/core/sis-feature/src/main/java/org/apache/sis/internal/coverage/j2d/TileOpExecutor.java @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.awt.Point; import java.awt.Rectangle; +import java.awt.Shape; import java.awt.image.Raster; import java.awt.image.RenderedImage; import java.awt.image.WritableRaster; @@ -39,6 +40,14 @@ import org.apache.sis.internal.feature.Resources; import org.apache.sis.internal.system.CommonExecutor; import org.apache.sis.internal.util.Strings; +import static java.lang.Math.addExact; +import static java.lang.Math.subtractExact; +import static java.lang.Math.incrementExact; +import static java.lang.Math.decrementExact; +import static java.lang.Math.multiplyExact; +import static java.lang.Math.toIntExact; +import static java.lang.Math.floorDiv; + /** * A read or write action to execute on each tile of an image. The operation may be executed @@ -69,7 +78,7 @@ import org.apache.sis.internal.util.Strings; * method. Those methods are inspired from {@link java.util.stream.Stream#collect(Collector)} API.</p> * * @author Martin Desruisseaux (Geomatys) - * @version 1.1 + * @version 1.2 * @since 1.1 * @module */ @@ -82,6 +91,13 @@ public class TileOpExecutor { private final int minTileX, minTileY, maxTileX, maxTileY; /** + * If the processing should be restricted to a non-rectangular shape, the region in pixel coordinates. + * Otherwise {@code null}. This shape should not be a rectangle because otherwise it would be redundant + * with minimum/maximum tile X/Y fields. + */ + private Shape areaOfInterest; + + /** * Where to report exceptions, or {@link TileErrorHandler#THROW} for throwing them. * If at least one error occurred, then this handler will receive the {@link Cursor#errors} report * after all computation {@linkplain Cursor#finish finished}. @@ -106,16 +122,48 @@ public class TileOpExecutor { final int tileHeight = image.getTileHeight(); final long tileGridXOffset = image.getTileGridXOffset(); // We want 64 bits arithmetic in operations below. final long tileGridYOffset = image.getTileGridYOffset(); - minTileX = Math.toIntExact(Math.floorDiv(aoi.x - tileGridXOffset, tileWidth )); - minTileY = Math.toIntExact(Math.floorDiv(aoi.y - tileGridYOffset, tileHeight)); - maxTileX = Math.toIntExact(Math.floorDiv(aoi.x + (aoi.width - 1L) - tileGridXOffset, tileWidth )); - maxTileY = Math.toIntExact(Math.floorDiv(aoi.y + (aoi.height - 1L) - tileGridYOffset, tileHeight)); + minTileX = toIntExact(floorDiv(aoi.x - tileGridXOffset, tileWidth )); + minTileY = toIntExact(floorDiv(aoi.y - tileGridYOffset, tileHeight)); + maxTileX = toIntExact(floorDiv(aoi.x + (aoi.width - 1L) - tileGridXOffset, tileWidth )); + maxTileY = toIntExact(floorDiv(aoi.y + (aoi.height - 1L) - tileGridYOffset, tileHeight)); } else { minTileX = image.getMinTileX(); minTileY = image.getMinTileY(); - maxTileX = Math.addExact(minTileX, image.getNumXTiles() - 1); - maxTileY = Math.addExact(minTileY, image.getNumYTiles() - 1); + maxTileX = addExact(minTileX, image.getNumXTiles() - 1); + maxTileY = addExact(minTileY, image.getNumYTiles() - 1); + } + } + + /** + * Sets the area of interest as an irregular shape. + * This executor will skip calculations in all tiles that do not intersect the given AOI. + * There is no benefit if this AOI is the same than the rectangle given to the constructor. + * But if the AOI is non-rectangular, then specifying it may help to skip a few more tiles. + * Skipping tiles saves not only {@code TileOpExecutor} computation time, but can save also + * computation time of source image if the source is itself the result of another computation. + * + * @param image the image for which to set an AOI, or {@code null} if unknown. + * @param aoi the non-rectangular AOI, or {@code null} if none. + * + * @since 1.2 + */ + public final void setAreaOfInterest(final RenderedImage image, Shape aoi) { + if (aoi != null && image != null) { + /* + * Compute the bounds of the region where iteration will happen, but with only one pixel in + * the tiles on the border (left, top, bottom, right). If AOI interior contains entirely + * those bounds, then the AOI does not help to reduce the amount of tiles to compute. + */ + final Rectangle bounds = getTileIndices(); + bounds.x = decrementExact(ImageUtilities.tileToPixelX(image, incrementExact(bounds.x)) - 1); + bounds.y = decrementExact(ImageUtilities.tileToPixelY(image, incrementExact(bounds.y)) - 1); + bounds.width = addExact(multiplyExact(bounds.width, image.getTileWidth() - 2), 2); + bounds.height = addExact(multiplyExact(bounds.height, image.getTileHeight() - 2), 2); + if (aoi.contains(bounds)) { + aoi = null; + } } + areaOfInterest = aoi; } /** @@ -161,8 +209,8 @@ public class TileOpExecutor { */ public final Rectangle getTileIndices() { return new Rectangle(minTileX, minTileY, - Math.incrementExact(Math.subtractExact(maxTileX, minTileX)), - Math.incrementExact(Math.subtractExact(maxTileY, minTileY))); + incrementExact(subtractExact(maxTileX, minTileX)), + incrementExact(subtractExact(maxTileY, minTileY))); } /** @@ -575,7 +623,7 @@ public class TileOpExecutor { Cursor(final RI image, final Collector<?,A,?> collector, final boolean stopOnError) { this.image = image; this.combiner = collector.combiner(); - this.numXTiles = Math.incrementExact(Math.subtractExact(maxTileX, minTileX)); + this.numXTiles = incrementExact(subtractExact(maxTileX, minTileX)); this.stopOnError = stopOnError; this.errors = new ErrorHandler.Report(); } @@ -601,14 +649,34 @@ public class TileOpExecutor { final boolean next(final Worker<RI,?,A> indices) { final int index = getAndIncrement(); if (index >= 0) { - indices.tx = Math.addExact(minTileX, index % numXTiles); - indices.ty = Math.addExact(minTileY, index / numXTiles); + indices.tx = addExact(minTileX, index % numXTiles); + indices.ty = addExact(minTileY, index / numXTiles); return indices.ty <= maxTileY; } return false; } /** + * Returns {@code true} if current tile of given worker intersects the area of interest. + * This is a finer check than the AOI specified at {@link TileOpExecutor} construction time, + * because the AOI tested here can be an irregular shape. + * + * @param indices the worker to test. + * @return whether current worker tile intersect the area of interest. + * + * @see #setAreaOfInterest(RenderedImage, Shape) + */ + final boolean intersectAOI(final Worker<RI,?,A> indices) { + if (areaOfInterest == null) { + return true; + } + final Rectangle bounds = new Rectangle(image.getTileWidth(), image.getTileHeight()); + bounds.x = addExact(multiplyExact(indices.tx, bounds.width), image.getTileGridXOffset()); + bounds.y = addExact(multiplyExact(indices.ty, bounds.height), image.getTileGridYOffset()); + return areaOfInterest.intersects(bounds); + } + + /** * Invoked when a thread finished to process all its tiles for combining its result with the result * of previous threads. This method does nothing if the given result is null. * @@ -711,8 +779,8 @@ public class TileOpExecutor { final int index = get(); String tile = "done"; if (index >= 0) { - final int tx = Math.addExact(minTileX, index % numXTiles); - final int ty = Math.addExact(minTileY, index / numXTiles); + final int tx = addExact(minTileX, index % numXTiles); + final int ty = addExact(minTileY, index / numXTiles); if (ty <= maxTileY) { tile = "(" + tx + ", " + ty + ')'; } @@ -786,7 +854,9 @@ public class TileOpExecutor { @Override public final void run() { while (cursor.next(this)) try { - executeOnCurrentTile(); + if (cursor.intersectAOI(this)) { + executeOnCurrentTile(); + } } catch (Exception ex) { cursor.recordError(new Point(tx, ty), trimImagingWrapper(ex)); } diff --git a/core/sis-feature/src/test/java/org/apache/sis/image/StatisticsCalculatorTest.java b/core/sis-feature/src/test/java/org/apache/sis/image/StatisticsCalculatorTest.java index 135799d..140f580 100644 --- a/core/sis-feature/src/test/java/org/apache/sis/image/StatisticsCalculatorTest.java +++ b/core/sis-feature/src/test/java/org/apache/sis/image/StatisticsCalculatorTest.java @@ -16,12 +16,16 @@ */ package org.apache.sis.image; +import java.awt.Shape; +import java.awt.geom.Ellipse2D; import java.util.Random; import java.awt.image.DataBuffer; import java.awt.image.RenderedImage; import java.awt.image.ImagingOpException; +import java.util.function.DoubleUnaryOperator; import org.apache.sis.internal.system.Modules; import org.apache.sis.math.Statistics; +import org.apache.sis.test.DependsOnMethod; import org.apache.sis.test.LoggingWatcher; import org.apache.sis.util.logging.Logging; import org.apache.sis.test.TestCase; @@ -36,7 +40,7 @@ import static org.junit.Assert.*; * {@link org.apache.sis.internal.coverage.j2d.TileOpExecutor} with multi-threading. * * @author Martin Desruisseaux (Geomatys) - * @version 1.1 + * @version 1.2 * @since 1.1 * @module */ @@ -55,6 +59,22 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { public final LoggingWatcher loggings = new LoggingWatcher(Logging.getLogger(Modules.RASTER)); /** + * The area of interest, or {@code null} if none. + */ + private Shape areaOfInterest; + + /** + * The filter to apply on sample values, or {@code null} if none. + */ + private DoubleUnaryOperator[] sampleFilters; + + /** + * Creates a new test case. + */ + public StatisticsCalculatorTest() { + } + + /** * Creates a dummy image for testing purpose. This image will contain many small tiles * of two bands. The first band has deterministic values and the second band contains * random values. @@ -83,20 +103,24 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { * @param source the image on which to compute statistics. * @return statistics on the given image computed sequentially. */ - private static Statistics[] computeSequentially(final RenderedImage source) { - return (Statistics[]) new StatisticsCalculator(source, null, false, true).computeSequentially(); + private Statistics[] computeSequentially(final RenderedImage source) { + return (Statistics[]) new StatisticsCalculator(source, areaOfInterest, sampleFilters, false, true).computeSequentially(); } /** - * Tests with parallel execution. The result of sequential execution is used as a reference. + * Implementation of {@link #testParallelExecution()} and other tests with various options. + * Values in the first band are determinist and values in the second band are randoms. + * + * @param minimum expected minimal sample value in the first band (the deterministic one). + * @param maximum expected maximal sample value in the first band (the deterministic one). */ - @Test - public void testParallelExecution() { - final ImageProcessor operations = new ImageProcessor(); + private void compareParallelWithSequential(final ImageProcessor operations, + final double minimum, final double maximum) + { operations.setExecutionMode(ImageProcessor.Mode.PARALLEL); final TiledImageMock image = createImage(); final Statistics[] expected = computeSequentially(image); - final Statistics[] actual = operations.valueOfStatistics(image, null); + final Statistics[] actual = operations.valueOfStatistics(image, areaOfInterest, sampleFilters); for (int i=0; i<expected.length; i++) { final Statistics e = expected[i]; final Statistics a = actual [i]; @@ -105,6 +129,42 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { assertEquals("sum", e.sum(), a.sum(), STRICT); } loggings.assertNoUnexpectedLog(); + assertEquals("minimum", minimum, actual[0].minimum(), STRICT); + assertEquals("maximum", maximum, actual[0].maximum(), STRICT); + } + + /** + * Tests with parallel execution. The result of sequential execution is used as a reference. + * The expected minimum sample value is 100 because this is by definition the value written + * in the first pixel of the first tile created by {@link #createImage()}. + */ + @Test + public void testParallelExecution() { + compareParallelWithSequential(new ImageProcessor(), 100, 51324); + } + + /** + * Tests with an arbitrary area of interest. + * The expected minimum and maximum values are determined empirically. + */ + @Test + public void testWithAOI() { + areaOfInterest = new Ellipse2D.Float(70, -50, TILE_WIDTH*11.6f, TILE_HEIGHT*9.2f); + compareParallelWithSequential(new ImageProcessor(), 19723, 44501); + } + + /** + * Tests with sample filters. The filter excludes the first and the few last pixels in the image + * created by {@link #createImage()}, which produces a visible effect on minimum and maximum values. + */ + @Test + @DependsOnMethod("testFilterNodataValues") + public void testWithSampleFilters() { + final ImageProcessor operations = new ImageProcessor(); + sampleFilters = new DoubleUnaryOperator[] { + operations.filterNodataValues(100, 51324, 51323, 201, 310) + }; + compareParallelWithSequential(operations, 101, 51322); } /** @@ -117,7 +177,7 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { final TiledImageMock image = createImage(); image.failRandomly(new Random(-8739538736973900203L), true); try { - operations.valueOfStatistics(image, null); + operations.valueOfStatistics(image, areaOfInterest, sampleFilters); fail("Expected ImagingOpException."); } catch (ImagingOpException e) { final String message = e.getMessage(); @@ -137,7 +197,7 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { operations.setErrorHandler(ErrorHandler.LOG); final TiledImageMock image = createImage(); image.failRandomly(new Random(8004277484984714811L), true); - final Statistics[] stats = operations.valueOfStatistics(image, null); + final Statistics[] stats = operations.valueOfStatistics(image, areaOfInterest, sampleFilters); for (final Statistics a : stats) { assertTrue(a.count() > 0); } @@ -148,4 +208,23 @@ public final strictfp class StatisticsCalculatorTest extends TestCase { loggings.assertNextLogContains(/* no keywords we could rely on. */); loggings.assertNoUnexpectedLog(); } + + /** + * Tests {@link StatisticsCalculator#filterNodataValues(Number[])}. + */ + @Test + public void testFilterNodataValues() { + assertNull(StatisticsCalculator.filterNodataValues(new Number[] {null, Double.NaN})); + DoubleUnaryOperator op = StatisticsCalculator.filterNodataValues(new Number[] {100}); + assertEquals( 10, op.applyAsDouble( 10), STRICT); + assertEquals(202, op.applyAsDouble(202), STRICT); + assertTrue(Double.isNaN(op.applyAsDouble(100))); + + op = StatisticsCalculator.filterNodataValues(new Number[] {201, null, 100, 310, Double.NaN, 201}); + assertEquals( 10, op.applyAsDouble( 10), STRICT); + assertEquals(202, op.applyAsDouble(202), STRICT); + assertTrue(Double.isNaN(op.applyAsDouble(100))); + assertTrue(Double.isNaN(op.applyAsDouble(310))); + assertTrue(Double.isNaN(op.applyAsDouble(201))); + } } diff --git a/core/sis-utility/src/main/java/org/apache/sis/util/ArraysExt.java b/core/sis-utility/src/main/java/org/apache/sis/util/ArraysExt.java index 3c78f18..9382504 100644 --- a/core/sis-utility/src/main/java/org/apache/sis/util/ArraysExt.java +++ b/core/sis-utility/src/main/java/org/apache/sis/util/ArraysExt.java @@ -1993,7 +1993,7 @@ public final class ArraysExt extends Static { /** * Returns {@code true} if all values in the specified array are equal to the specified value, - * which may be {@code null}. + * which may be {@code null}. If the given array is empty, then this method returns {@code true}. * * @param array the array to check. * @param value the expected value.