This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit cf73da5dd67af0042537b8e4e2fa4874b3f29975
Author: Martin Desruisseaux <martin.desruisse...@geomatys.com>
AuthorDate: Tue Sep 13 18:45:16 2022 +0200
Allow deferred loading of slice in a concatenated coverage.
---
.../org/apache/sis/internal/storage/Resources.java | 5 +
.../sis/internal/storage/Resources.properties | 1 +
.../sis/internal/storage/Resources_fr.properties | 3 +-
.../aggregate/ConcatenatedGridCoverage.java | 123 ++++++++++++++-----
.../aggregate/ConcatenatedGridResource.java | 133 +++++++++++++++++----
5 files changed, 213 insertions(+), 52 deletions(-)
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.java
index 5aabcbf31d..5934ebb244 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.java
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.java
@@ -124,6 +124,11 @@ public final class Resources extends IndexedResourceBundle
{
*/
public static final short CanNotReadPixel_3 = 68;
+ /**
+ * Can not read slice at index {0}.
+ */
+ public static final short CanNotReadSlice_1 = 78;
+
/**
* Can not remove resource “{1}” from aggregate “{0}”.
*/
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.properties
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.properties
index 662c15ae1b..c561b2bb2f 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.properties
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources.properties
@@ -32,6 +32,7 @@ CanNotReadFile_2 = Can not read
\u201c{1}\u201d as a file in th
CanNotReadFile_3 = Can not read line {2} of \u201c{1}\u201d
as part of a file in the {0} format.
CanNotReadFile_4 = Can not read after column {3} of line {2}
of \u201c{1}\u201d as part of a file in the {0} format.
CanNotReadPixel_3 = Can not read pixel at ({0}, {1}) indices
in the \u201c{2}\u201d file.
+CanNotReadSlice_1 = Can not read slice at index {0}.
CanNotRemoveResource_2 = Can not remove resource \u201c{1}\u201d
from aggregate \u201c{0}\u201d.
CanNotRenderImage_1 = Can not render an image for the
\u201c{0}\u201d coverage.
CanNotStoreResourceType_2 = Can not save resources of type
\u2018{1}\u2019 in a \u201c{0}\u201d store.
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources_fr.properties
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources_fr.properties
index 2d3a34bdc5..6c8d3708e5 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources_fr.properties
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/Resources_fr.properties
@@ -37,6 +37,7 @@ CanNotReadFile_2 = Ne peut pas lire
\u00ab\u202f{1}\u202f\u00bb
CanNotReadFile_3 = Ne peut pas lire la ligne {2} de
\u00ab\u202f{1}\u202f\u00bb comme une partie d\u2019un fichier au format {0}.
CanNotReadFile_4 = Ne peut pas lire apr\u00e8s la colonne {3}
de la ligne {2} de \u00ab\u202f{1}\u202f\u00bb comme une partie d\u2019un
fichier au format {0}.
CanNotReadPixel_3 = Ne peut pas lire le pixel aux indices
({0}, {1}) dans le fichier \u00ab\u202f{2}\u202f\u00bb.
+CanNotReadSlice_1 = Ne peut pas lire la tranche \u00e0
l\u2019index {0}.
CanNotRemoveResource_2 = Ne peut pas supprimer la ressource
\u00ab\u202f{1}\u202f\u00bb de l\u2019agr\u00e9gat \u00ab\u202f{0}\u202f\u00bb.
CanNotRenderImage_1 = Ne peut pas produire une image pour la
couverture de donn\u00e9es \u00ab\u202f{0}\u202f\u00bb.
CanNotWriteResource_1 = Ne peut pas \u00e9crire la ressource
\u00ab\u202f{0}\u202f\u00bb.
@@ -70,7 +71,7 @@ IncompatibleGridGeometry = Toutes les couvertures de
donn\u00e9es doive
InvalidExpression_2 = Expression \u00ab\u202f{1}\u202f\u00bb
invalide ou non-support\u00e9e \u00e0 l\u2019index {0}.
InvalidSampleDimensionIndex_2 = L\u2019index de dimension
d\u2019\u00e9chantillonnage {1} est invalide. On attendait un index de 0 \u00e0
{0} inclusif.
InconsistentNameComponents_2 = Les \u00e9l\u00e9ments qui composent le
nom \u00ab\u202f{1}\u202f\u00bb ne sont pas coh\u00e9rents avec ceux du nom qui
avait \u00e9t\u00e9 pr\u00e9c\u00e9demment li\u00e9 dans les donn\u00e9es de
\u00ab\u202f{0}\u202f\u00bb.
-ExceptionInListener_1 = Une exception est survenue dans un capteur
d'\u00e9v\u00e9nements de type \u2018{0}\u2019.
+ExceptionInListener_1 = Une exception est survenue dans un capteur
d\u2019\u00e9v\u00e9nements de type \u2018{0}\u2019.
LoadedGridCoverage_6 = Lecture d\u2019une couverture de
donn\u00e9es entre {1} \u2013 {2} et {3} \u2013 {4} \u00e0 partir du fichier
\u00ab\u202f{0}\u202f\u00bb en {5} secondes.
MarkNotSupported_1 = Les marques ne sont pas support\u00e9es
sur le flux \u00ab\u202f{0}\u202f\u00bb.
MissingResourceIdentifier_1 = La ressource \u00ab\u202f{0}\u202f\u00bb
n\u2019a pas d\u2019identifiant.
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java
index 44faa761f6..1a5a8d280f 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridCoverage.java
@@ -16,11 +16,20 @@
*/
package org.apache.sis.internal.storage.aggregate;
+import java.util.List;
+import java.util.ArrayList;
import java.awt.image.RenderedImage;
+import org.apache.sis.coverage.SampleDimension;
import org.apache.sis.coverage.grid.GridExtent;
import org.apache.sis.coverage.grid.GridGeometry;
import org.apache.sis.coverage.grid.GridCoverage;
import org.apache.sis.coverage.SubspaceNotSpecifiedException;
+import org.apache.sis.storage.GridCoverageResource;
+import org.apache.sis.storage.DataStoreException;
+import org.apache.sis.internal.storage.Resources;
+
+// Branch-dependent imports
+import org.opengis.coverage.CannotEvaluateException;
/**
@@ -37,9 +46,35 @@ final class ConcatenatedGridCoverage extends GridCoverage {
* The slices of this coverage, in the same order than {@link
#coordinatesOfSlices}.
* Each slice is not necessarily 1 cell tick; larger slices are accepted.
* The length of this array shall be at least 2.
+ *
+ * <p>Some elements in the array may be {@code null} if the coverage are
lazily loaded.</p>
*/
private final GridCoverage[] slices;
+ /**
+ * The resource from which load the coverages in the {@link #slices}
array, or {@code null} if none.
+ * This is non-null only if the {@linkplain #slices} are lazily loaded.
+ */
+ private final GridCoverageResource[] resources;
+
+ /**
+ * The domain to request when reading a coverage from the resource.
+ * This is non-null only if the {@linkplain #slices} are lazily loaded.
+ */
+ private final GridGeometry request;
+
+ /**
+ * The sample dimensions to request when loading slices from the
{@linkplain #resources}.
+ * This is non-null only if the {@linkplain #slices} are lazily loaded.
+ */
+ private final int[] ranges;
+
+ /**
+ * Whether this grid coverage should be considered as converted.
+ * This is used only if the {@linkplain #slices} are lazily loaded.
+ */
+ private final boolean isConverted;
+
/**
* The object for identifying indices in the {@link #slices} array.
*/
@@ -50,36 +85,46 @@ final class ConcatenatedGridCoverage extends GridCoverage {
*/
private final int startAt;
- /**
- * View over this grid coverage after conversion of sample values, or
{@code null} if not yet created.
- * May be {@code this} if we determined that there is no conversion or the
conversion is identity.
- *
- * @see #forConvertedValues(boolean)
- */
- private transient ConcatenatedGridCoverage convertedView;
-
/**
* Creates a new aggregated coverage.
+ *
+ * @param source the concatenated resource which is creating this
coverage.
+ * @param domain domain of the coverage to create.
+ * @param request grid geometry to request when loading data. Used only
if {@code resources} is non-null.
+ * @param slices grid coverages for each slice. May contain {@code
null} elements is lazy loading is applied.
+ * @param resources resources from which to load grid coverages, or
{@code null} if none.
+ * @param startAt index of the first slice in {@link #locator}.
+ * @param ranges bands to request when loading coverages. Used only if
{@code resources} is non-null.
*/
- ConcatenatedGridCoverage(final ConcatenatedGridResource source, final
GridGeometry domain,
- final GridCoverage[] slices, final int startAt)
+ ConcatenatedGridCoverage(final ConcatenatedGridResource source, final
GridGeometry domain, final GridGeometry request,
+ final GridCoverage[] slices, final
GridCoverageResource[] resources, final int startAt,
+ final int[] ranges)
{
super(domain, source.getSampleDimensions());
- this.slices = slices;
- this.startAt = startAt;
- this.locator = source.locator;
+ this.slices = slices;
+ this.resources = resources;
+ this.startAt = startAt;
+ this.request = request;
+ this.ranges = ranges;
+ this.isConverted = source.isConverted;
+ this.locator = source.locator;
}
/**
- * Creates a new aggregated coverage for the result of a conversion
from/to package values.
+ * Creates a new aggregated coverage for the result of a conversion
from/to packed values.
* This constructor assumes that all slices use the same sample dimensions.
*/
- private ConcatenatedGridCoverage(final ConcatenatedGridCoverage source,
final GridCoverage[] slices) {
- super(source.getGridGeometry(), slices[0].getSampleDimensions());
- this.slices = slices;
- this.startAt = source.startAt;
- this.locator = source.locator;
- convertedView = source;
+ private ConcatenatedGridCoverage(final ConcatenatedGridCoverage source,
final GridCoverage[] slices,
+ final List<SampleDimension> sampleDimensions, final boolean
converted)
+ {
+ super(source.getGridGeometry(), sampleDimensions);
+ this.slices = slices;
+ this.resources = source.resources;
+ this.startAt = source.startAt;
+ this.request = source.request;
+ this.ranges = source.ranges;
+ this.locator = source.locator;
+ this.isConverted = converted;
}
/**
@@ -92,17 +137,30 @@ final class ConcatenatedGridCoverage extends GridCoverage {
* @return a coverage containing requested values. May be {@code this} but
never {@code null}.
*/
@Override
- public synchronized GridCoverage forConvertedValues(final boolean
converted) {
- if (convertedView == null) {
- boolean changed = false;
- final GridCoverage[] c = new GridCoverage[slices.length];
- for (int i=0; i<c.length; i++) {
- final GridCoverage source = slices[i];
+ protected GridCoverage createConvertedValues(final boolean converted) {
+ boolean changed = false;
+ int template = -1; // Index of a grid coverage to use as
a template.
+ final GridCoverage[] c = new GridCoverage[slices.length];
+ for (int i=0; i<c.length; i++) {
+ final GridCoverage source = slices[i];
+ if (source != null) {
changed |= (c[i] = source.forConvertedValues(converted)) !=
source;
+ template = i;
+ } else {
+ changed |= (converted != isConverted);
}
- convertedView = changed ? new ConcatenatedGridCoverage(this, c) :
this;
}
- return convertedView;
+ if (!changed) {
+ return this;
+ }
+ final List<SampleDimension> sampleDimensions;
+ if (template >= 0) {
+ sampleDimensions = c[template].getSampleDimensions();
+ } else {
+ sampleDimensions = new ArrayList<>(getSampleDimensions());
+ sampleDimensions.replaceAll((b) ->
b.forConvertedValues(converted));
+ }
+ return new ConcatenatedGridCoverage(this, c, sampleDimensions,
converted);
}
/**
@@ -123,6 +181,13 @@ final class ConcatenatedGridCoverage extends GridCoverage {
if (upper - lower != 1) {
throw new SubspaceNotSpecifiedException();
}
- return slices[lower].render(locator.toSliceExtent(extent, lower));
+ GridCoverage slice = slices[lower];
+ if (slice == null) try {
+ slice = resources[lower].read(request,
ranges).forConvertedValues(isConverted);
+ slices[lower] = slice;
+ } catch (DataStoreException e) {
+ throw new
CannotEvaluateException(Resources.format(Resources.Keys.CanNotReadSlice_1,
lower + startAt), e);
+ }
+ return slice.render(locator.toSliceExtent(extent, lower));
}
}
diff --git
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java
index 773faa3169..9bf705d312 100644
---
a/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java
+++
b/storage/sis-storage/src/main/java/org/apache/sis/internal/storage/aggregate/ConcatenatedGridResource.java
@@ -37,8 +37,10 @@ import org.apache.sis.storage.RasterLoadingStrategy;
import org.apache.sis.storage.event.StoreListeners;
import org.apache.sis.internal.storage.MemoryGridResource;
import org.apache.sis.internal.storage.MetadataBuilder;
+import org.apache.sis.internal.storage.RangeArgument;
import org.apache.sis.internal.util.UnmodifiableArrayList;
import org.apache.sis.internal.util.CollectionsExt;
+import org.apache.sis.internal.util.Numerics;
import org.apache.sis.util.ArraysExt;
@@ -73,12 +75,35 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
*/
private final List<SampleDimension> sampleDimensions;
+ /**
+ * Whether all {@link SampleDimension} represent "real world" values.
+ */
+ final boolean isConverted;
+
/**
* The slices of this resource, in the same order than {@link
#coordinatesOfSlices}.
* Each slice is not necessarily 1 cell tick; larger slices are accepted.
*/
private final GridCoverageResource[] slices;
+ /**
+ * Whether loading of grid coverages should be deferred to rendering time.
+ * This is a bit set packed as {@code long} values. A bit value of 1 means
+ * that the coverages at the corresponding index should be loaded from the
+ * {@linkplain #slices} at same index only when first needed.
+ *
+ * <p>Whether a bit is set or not depends on two factor:</p>
+ * <ul>
+ * <li>Whether deferred loading has been requested by a call to {@link
#setLoadingStrategy(RasterLoadingStrategy)}.</li>
+ * <li>Whether the slice at the corresponding index can handle deferred
loading itself.
+ * In such case, we let the resource manages its own lazy
loading.</li>
+ * </ul>
+ *
+ * @see RasterLoadingStrategy#AT_READ_TIME
+ * @see RasterLoadingStrategy#AT_RENDER_TIME
+ */
+ private final long[] deferredLoading;
+
/**
* The object for identifying indices in the {@link #slices} array.
*/
@@ -133,6 +158,14 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
this.sampleDimensions = ranges;
this.slices = slices;
this.locator = locator;
+ this.deferredLoading = new long[Numerics.ceilDiv(slices.length,
Long.SIZE)];
+ for (final SampleDimension sd : ranges) {
+ if (sd.forConvertedValues(true) != sd) {
+ isConverted = false;
+ return;
+ }
+ }
+ isConverted = true;
}
/**
@@ -253,19 +286,30 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
*/
@Override
public RasterLoadingStrategy getLoadingStrategy() throws
DataStoreException {
- RasterLoadingStrategy common = null;
+ /*
+ * If at least one bit of `deferredLoading` is set, then it means that
+ * `setLoadingStrategy(…)` has been invoked with anything else than
`AT_READ_TIME`.
+ */
+ int bitx = 0;
+ long mask = 1;
+ RasterLoadingStrategy conservative =
RasterLoadingStrategy.AT_GET_TILE_TIME;
for (final GridCoverageResource slice : slices) {
- final RasterLoadingStrategy sr = slice.getLoadingStrategy();
- if (sr != null) { // Should never be null, but we are
paranoiac.
- if (common == null || sr.ordinal() < common.ordinal()) {
- common = sr;
- if (common.ordinal() == 0) {
- break;
- }
- }
+ RasterLoadingStrategy s = slice.getLoadingStrategy();
+ if (s == null || s.ordinal() == 0) { // Should
never be null, but we are paranoiac.
+ s = ((deferredLoading[bitx] & mask) != 0)
+ ? RasterLoadingStrategy.AT_RENDER_TIME
+ : RasterLoadingStrategy.AT_READ_TIME;
+ }
+ if (s.ordinal() < conservative.ordinal()) {
+ conservative = s;
+ if (s.ordinal() == 0) break;
+ }
+ if ((mask <<= 1) == 0) {
+ mask=1;
+ bitx++;
}
}
- return common;
+ return conservative;
}
/**
@@ -278,9 +322,20 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
*/
@Override
public boolean setLoadingStrategy(final RasterLoadingStrategy strategy)
throws DataStoreException {
- boolean accepted = false;
+ final boolean deferred = (strategy.ordinal() != 0);
+ Arrays.fill(deferredLoading, 0);
+ boolean accepted = true;
+ int bitx = 0;
+ long mask = 1;
for (final GridCoverageResource slice : slices) {
- accepted |= slice.setLoadingStrategy(strategy);
+ if (!slice.setLoadingStrategy(strategy)) {
+ if (deferred) deferredLoading[bitx] |= mask;
+ accepted = false;
+ }
+ if ((mask <<= 1) == 0) {
+ mask=1;
+ bitx++;
+ }
}
return accepted;
}
@@ -289,12 +344,18 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
* Loads a subset of the grid coverage represented by this resource.
*
* @param domain desired grid extent and resolution, or {@code null} for
reading the whole domain.
- * @param range 0-based indices of sample dimensions to read, or {@code
null} or an empty sequence for reading them all.
- * @return the grid coverage for the specified domain and range.
+ * @param ranges 0-based indices of sample dimensions to read, or {@code
null} or an empty sequence for reading them all.
+ * @return the grid coverage for the specified domain and ranges.
* @throws DataStoreException if an error occurred while reading the grid
coverage data.
*/
@Override
- public GridCoverage read(GridGeometry domain, final int... ranges) throws
DataStoreException {
+ public GridCoverage read(GridGeometry domain, int... ranges) throws
DataStoreException {
+ /*
+ * Validate arguments.
+ */
+ if (ranges != null) {
+ ranges = RangeArgument.validate(sampleDimensions.size(), ranges,
listeners).getSelectedBands();
+ }
int lower = 0, upper = slices.length;
if (domain != null) {
final GridDerivation subgrid =
gridGeometry.derive().rounding(GridRoundingMode.ENCLOSING).subgrid(domain);
@@ -308,19 +369,47 @@ final class ConcatenatedGridResource extends
AbstractGridCoverageResource implem
* Create arrays with only the requested range, without keeping
reference to this concatenated resource,
* for allowing garbage-collection of resources outside that range.
*/
- final GridCoverage[] coverages = new GridCoverage[upper - lower];
- for (int i=0; i < coverages.length; i++) {
+ final int count = upper - lower;
+ final GridGeometry[] geometries = new GridGeometry[count];
+ final GridCoverage[] coverages = new GridCoverage[count];
+ GridCoverageResource[] resources = null; //
Created when first needed.
+ int bitx = lower >>> Numerics.LONG_SHIFT;
+ long mask = 1L << lower; // No need for (lower
& 63) because high bits are ignored.
+ if (count <= 1) mask = 0; // Trick for forcing
coverage loading.
+ for (int i=0; i<count; i++) {
final GridCoverageResource slice = slices[lower + i];
if (slice instanceof MemoryGridResource) {
- coverages[i] = ((MemoryGridResource) slice).coverage;
+ final GridCoverage coverage = ((MemoryGridResource)
slice).coverage;
+ coverages [i] = coverage;
+ geometries[i] = coverage.getGridGeometry();
+ } else if ((deferredLoading[bitx] & mask) == 0) {
+ final GridCoverage coverage = slice.read(domain, ranges);
+ coverages [i] = coverage;
+ geometries[i] = coverage.getGridGeometry();
} else {
- coverages[i] = slice.read(domain, ranges);
+ if (resources == null) {
+ resources = new GridCoverageResource[count];
+ }
+ resources [i] = slice;
+ geometries[i] = slice.getGridGeometry();
+ }
+ if ((mask <<= 1) == 0) {
+ mask=1;
+ bitx++;
}
}
- if (coverages.length == 1) {
+ /*
+ * If it was not necessary to keep references to any resource, clear
references to information
+ * which were needed only for loading coverages from resources. Then
create create the coverage.
+ */
+ if (count == 1) {
return coverages[0];
}
- domain = locator.union(gridGeometry, Arrays.asList(coverages), (c) ->
c.getGridGeometry().getExtent());
- return new ConcatenatedGridCoverage(this, domain, coverages, lower);
+ if (resources == null) {
+ domain = null;
+ ranges = null;
+ }
+ final GridGeometry union = locator.union(gridGeometry,
Arrays.asList(geometries), GridGeometry::getExtent);
+ return new ConcatenatedGridCoverage(this, union, domain, coverages,
resources, lower, ranges);
}
}
