This is an automated email from the ASF dual-hosted git repository.
desruisseaux pushed a commit to branch geoapi-4.0
in repository https://gitbox.apache.org/repos/asf/sis.git
commit ddc4420c59b9df5d03af9b59dbe92e3f73ef7806
Author: Martin Desruisseaux <martin.desruisse...@geomatys.com>
AuthorDate: Tue Sep 17 12:55:04 2024 +0200
Allow to read more tiles in one GDAL read operation.
---
.../sis/storage/geotiff/CompressedSubset.java | 24 ++++++++----
.../org/apache/sis/storage/geotiff/DataSubset.java | 6 ++-
.../apache/sis/storage/base/TiledGridCoverage.java | 44 +++++++++++++---------
.../apache/sis/storage/base/TiledGridResource.java | 42 +++++++++++++++++++--
.../org/apache/sis/storage/gdal/TiledResource.java | 15 ++++++++
5 files changed, 99 insertions(+), 32 deletions(-)
diff --git
a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/CompressedSubset.java
b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/CompressedSubset.java
index 5ae22e39b6..5d418d27be 100644
---
a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/CompressedSubset.java
+++
b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/CompressedSubset.java
@@ -22,7 +22,6 @@ import java.nio.Buffer;
import java.awt.Point;
import java.awt.image.Raster;
import static java.lang.Math.toIntExact;
-import static java.lang.Math.multiplyFull;
import org.apache.sis.storage.DataStoreException;
import org.apache.sis.storage.base.TiledGridResource;
import org.apache.sis.storage.geotiff.inflater.Inflater;
@@ -53,7 +52,7 @@ final class CompressedSubset extends DataSubset {
* <em>first</em> pixel in a row. For computing the actual number of
sample values to skip,
* the number of sample values read or skipped before the last pixel must
be subtracted.
*/
- private final int afterLastBand;
+ private final long afterLastBand;
/**
* Number of sample values to skip after a chunk has been read, or {@code
null} if none.
@@ -108,9 +107,9 @@ final class CompressedSubset extends DataSubset {
@SuppressWarnings("LocalVariableHidesMemberVariable")
CompressedSubset(final DataCube source, final TiledGridResource.Subset
subset) throws DataStoreException {
super(source, subset);
- scanlineStride = multiplyFull(getTileSize(X_DIMENSION),
sourcePixelStride);
- final int between = sourcePixelStride * (getSubsampling(X_DIMENSION) -
1);
- int afterLastBand = sourcePixelStride * (getTileSize(X_DIMENSION) - 1);
+ scanlineStride = sourcePixelStride * getTileSize(X_DIMENSION);
+ final int between = sourcePixelStride * (getSubsampling(X_DIMENSION)
- 1);
+ long afterLastBand = scanlineStride - sourcePixelStride;
if (includedBands != null && sourcePixelStride > 1) {
final int[] skips = new int[includedBands.length];
final int m = skips.length - 1;
@@ -213,9 +212,18 @@ final class CompressedSubset extends DataSubset {
final int pixelsPerElement = getPixelsPerElement(); //
Always ≥ 1 and usually = 1.
assert (head % pixelsPerElement) == 0 : head;
if (inflater == null) {
- inflater = Inflater.create(source.listeners(), input(),
source.getCompression(), source.getPredictor(),
- sourcePixelStride, getTileSize(X_DIMENSION),
chunksPerRow, samplesPerChunk, skipAfterChunks,
- pixelsPerElement, dataType);
+ inflater = Inflater.create(
+ source.listeners(), // Object where to
report warnings.
+ input(), // The source of
data to decompress.
+ source.getCompression(), // The compression
method.
+ source.getPredictor(), // The mathematical
operator to apply after decompression.
+ sourcePixelStride, // Number of sample
values per pixel in the source image.
+ toIntExact(getTileSize(X_DIMENSION)), // Number of pixels
in a row of source image.
+ chunksPerRow, // Number of pixels
per row in target image.
+ samplesPerChunk, // Number of sample
values per pixel.
+ skipAfterChunks, // Number of sample
values to skip between pixels.
+ pixelsPerElement, // Number of pixels
per primitive element (for packed.
+ dataType); // Primitive type
used for storing data elements in the bank.
}
@SuppressWarnings("LocalVariableHidesMemberVariable")
final Inflater inflater = this.inflater;
diff --git
a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java
b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java
index 668cb11692..0ed305eedb 100644
---
a/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java
+++
b/endorsed/src/org.apache.sis.storage.geotiff/main/org/apache/sis/storage/geotiff/DataSubset.java
@@ -29,7 +29,6 @@ import java.awt.image.DataBufferDouble;
import java.awt.image.Raster;
import static java.lang.Math.subtractExact;
import static java.lang.Math.multiplyExact;
-import static java.lang.Math.multiplyFull;
import static java.lang.Math.toIntExact;
import org.opengis.util.GenericName;
import org.apache.sis.image.DataType;
@@ -497,7 +496,10 @@ class DataSubset extends TiledGridCoverage implements
Localized {
* This length is used only for verification purpose so it does not
need to be exact.
*/
final long length = ceilDiv(width * height * sourcePixelStride *
sampleSize, Byte.SIZE);
- final long[] size = new long[] {multiplyFull(sourcePixelStride,
getTileSize(X_DIMENSION)), getTileSize(Y_DIMENSION)};
+ final long[] size = new long[] {
+ multiplyExact(getTileSize(X_DIMENSION), sourcePixelStride),
+ getTileSize(Y_DIMENSION)
+ };
/*
* If we use an interleaved sample model, each "element" from
`HyperRectangleReader` perspective is actually a
* group of `sourcePixelStride` values. Note that in such case, we
cannot handle subsampling on the first axis.
diff --git
a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridCoverage.java
b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridCoverage.java
index eccfd9f5c5..684d9a5226 100644
---
a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridCoverage.java
+++
b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridCoverage.java
@@ -99,7 +99,7 @@ public abstract class TiledGridCoverage extends GridCoverage {
private final GridExtent readExtent;
/**
- * Whether to force the {@link #readExtent} tile intersection to the
{@link #tileSize}.
+ * Whether to force the {@link #readExtent} tile intersection to the
{@link #virtualTileSize}.
* This is relevant only for the last column of tile matrix, because those
tiles may be truncated
* if the image size is not a multiple of tile size. It is usually
necessary to read those tiles
* fully anyway because otherwise, the pixels read from the storage would
not be aligned with the
@@ -124,14 +124,19 @@ public abstract class TiledGridCoverage extends
GridCoverage {
* The length of this array is the number of dimensions in the source
{@link GridExtent}.
* This is often {@value #BIDIMENSIONAL} but can also be more.
*
+ * <p>The tile size may be virtual if the {@link TiledGridResource}
subclass decided to coalesce
+ * many real tiles in bigger virtual tiles. This is sometime useful when a
subsampling is applied,
+ * for avoiding that the subsampled tiles become too small.
+ * This strategy may be convenient when coalescing is easy.</p>
+ *
* @see #getTileSize(int)
*/
- private final int[] tileSize;
+ private final long[] virtualTileSize;
/**
* Values by which to multiply each tile coordinates for obtaining the
index in the tile vector.
- * The length of this array is the same as {@link #tileSize}. All
coverages created from the same
- * {@link TiledGridResource} have the same stride values.
+ * The length of this array is the same as {@link #virtualTileSize}. All
coverages created from
+ * the same {@link TiledGridResource} have the same stride values.
*/
private final int[] tileStrides;
@@ -198,8 +203,8 @@ public abstract class TiledGridCoverage extends
GridCoverage {
/**
* The sample model for all rasters. The width and height of this sample
model are the two first elements
- * of {@link #tileSize} divided by subsampling and clipped to the domain.
If user requested to read only
- * a subset of the bands, then this sample model is already the subset.
+ * of {@link #virtualTileSize} divided by subsampling and clipped to the
domain.
+ * If user requested to read only a subset of the bands, then this sample
model is already the subset.
*
* @see TiledGridResource#getSampleModel(int[])
*/
@@ -241,7 +246,7 @@ public abstract class TiledGridCoverage extends
GridCoverage {
subsamplingOffsets = subset.subsamplingOffsets;
includedBands = subset.includedBands;
rasters = subset.cache;
- tileSize = subset.tileSize;
+ virtualTileSize = subset.virtualTileSize;
tmcOfFirstTile = new long[dimension];
tileStrides = new int [dimension];
final int[] subSize = new int [dimension];
@@ -250,10 +255,10 @@ public abstract class TiledGridCoverage extends
GridCoverage {
long indexOfFirstTile = 0;
int tileStride = 1;
for (int i=0; i<dimension; i++) {
- final int ts = tileSize[i];
+ final long ts = virtualTileSize[i];
tmcOfFirstTile[i] = floorDiv(readExtent.getLow(i), ts);
tileStrides[i] = tileStride;
- subSize[i] = (int) Math.min(((ts-1) / subsampling[i]) + 1,
extent.getSize(i));
+ subSize[i] = toIntExact(Math.min(((ts-1) / subsampling[i])
+ 1, extent.getSize(i)));
indexOfFirstTile = addExact(indexOfFirstTile,
multiplyExact(tmcOfFirstTile[i], tileStride));
int tileCount =
toIntExact(ceilDiv(subset.sourceExtent.getSize(i), ts));
tileStride = multiplyExact(tileCount, tileStride);
@@ -272,7 +277,7 @@ public abstract class TiledGridCoverage extends
GridCoverage {
this.model = model;
this.colors = subset.colorsForBandSubset;
this.fillValues = subset.fillValues;
- forceTileSize = subSize[X_DIMENSION] * subsampling[X_DIMENSION] ==
tileSize[X_DIMENSION];
+ forceTileSize = multiplyFull(subSize[X_DIMENSION],
subsampling[X_DIMENSION]) == virtualTileSize[X_DIMENSION];
}
/**
@@ -294,12 +299,14 @@ public abstract class TiledGridCoverage extends
GridCoverage {
/**
* Returns the size of all tiles in the domain of this {@code
TiledGridCoverage}, without clipping and subsampling.
+ * It may be a virtual tile size if the {@link TiledGridResource} subclass
decided to coalesce many real tiles into
+ * fewer bigger virtual tiles.
*
* @param dimension dimension for which to get tile size.
* @return tile size in the given dimension, without clipping and
subsampling.
*/
- protected final int getTileSize(final int dimension) {
- return tileSize[dimension];
+ protected final long getTileSize(final int dimension) {
+ return virtualTileSize[dimension];
}
/**
@@ -358,7 +365,7 @@ public abstract class TiledGridCoverage extends
GridCoverage {
* @throws ArithmeticException if the coordinate cannot be represented as
an integer.
*/
private long toResourceTileMatrixCoordinate(final long coordinate, final
int dimension) {
- return floorDiv(pixelToResourceCoordinate(coordinate, dimension),
tileSize[dimension]);
+ return floorDiv(pixelToResourceCoordinate(coordinate, dimension),
virtualTileSize[dimension]);
}
/**
@@ -452,8 +459,8 @@ public abstract class TiledGridCoverage extends
GridCoverage {
}
}
// Lower and upper coordinates in subsampled image, rounded to
integer number of tiles and clipped to available data.
- final long lower = /* inclusive
*/Math.max(resourceToPixelCoordinate(/* inclusive */multiplyExact(tileLo,
tileSize[i]), i), min);
- final long upper =
incrementExact(Math.min(resourceToPixelCoordinate(decrementExact(multiplyExact(tileUp,
tileSize[i])), i), max));
+ final long lower = /* inclusive
*/Math.max(resourceToPixelCoordinate(/* inclusive */multiplyExact(tileLo,
virtualTileSize[i]), i), min);
+ final long upper =
incrementExact(Math.min(resourceToPixelCoordinate(decrementExact(multiplyExact(tileUp,
virtualTileSize[i])), i), max));
imageSize[i] = toIntExact(subtractExact(upper, lower));
offsetAOI[i] = toIntExact(subtractExact(lower, aoiMin));
tileLower[i] = toIntExact(subtractExact(tileLo,
tmcOfFirstTile[i]));
@@ -727,7 +734,7 @@ public abstract class TiledGridCoverage extends
GridCoverage {
System.arraycopy(coverage.subsampling, 0, subsampling, 0,
dimension);
}
while (--dimension >= 0) {
- final int tileSize = coverage.getTileSize(dimension);
+ final long tileSize = coverage.getTileSize(dimension);
final long tileIndex =
addExact(coverage.tmcOfFirstTile[dimension], tmcInSubset[dimension]);
final long tileBase = multiplyExact(tileIndex, tileSize);
/*
@@ -850,7 +857,7 @@ public abstract class TiledGridCoverage extends
GridCoverage {
* now instead of in a call to `next()` during iteration. A
negative value
* would mean that the AOI does not intersect the region
requested by user.
*/
- final int max = addExact(offsetAOI[i],
multiplyExact(getTileSize(i), count));
+ final long max = addExact(offsetAOI[i],
multiplyExact(getTileSize(i), count));
assert max > Math.max(offsetAOI[i], 0) : max;
}
this.tileCountInQuery = tileCountInQuery;
@@ -877,7 +884,8 @@ public abstract class TiledGridCoverage extends
GridCoverage {
if (s > base) {
lower[i] = s;
// Use of `ceilDiv(…)` is for consistency with
`getTileOrigin(int)`.
- offset[i] = addExact(offset[i], ceilDiv(multiplyExact(s -
base, getTileSize(i)), getSubsampling(i)));
+ long origin = ceilDiv(multiplyExact(s - base,
getTileSize(i)), getSubsampling(i));
+ offset[i] = toIntExact(addExact(offset[i], origin));
}
}
final int[] upper = this.tileUpper.clone();
diff --git
a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridResource.java
b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridResource.java
index f4322a3daf..b4bff677b3 100644
---
a/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridResource.java
+++
b/endorsed/src/org.apache.sis.storage/main/org/apache/sis/storage/base/TiledGridResource.java
@@ -171,13 +171,36 @@ public abstract class TiledGridResource extends
AbstractGridCoverageResource {
/**
* Returns the size of tiles in this resource.
- * The length of the returned array is the number of dimensions, which
must be 2 or more.
+ * The length of the returned array is the number of dimensions,
+ * which must be {@value TiledGridCoverage#BIDIMENSIONAL} or more.
*
* @return the size of tiles (in pixels) in this resource.
* @throws DataStoreException if an error occurred while fetching the tile
size.
*/
protected abstract int[] getTileSize() throws DataStoreException;
+ /**
+ * Returns the tile size to use for a read operation using the given
subsampling.
+ * The default implementation returns the real tile size, as returned by
{@link #getTileSize()}.
+ * Subclasses may override if it is easy for them to read many tiles as if
they were a single tile.
+ * Such coalescing can be useful for avoiding that a read operation
produces tiles that become too
+ * small after the subsampling.
+ *
+ * <p>Note that {@link TiledGridCoverage} aligns its {@link GridExtent} on
the boundaries of "real" tiles
+ * (i.e. on tiles of the size returned by {@link #getTileSize()}), not on
the boundaries of virtual tiles.
+ * Therefore, subclasses should override this method only if they are
prepared to read regions covering a
+ * fraction of the virtual tiles. A simple and efficient strategy is to
simply multiply {@code tileSize[i]}
+ * by {@code subsampling[i]} for each dimension <var>i</var> so that each
virtual tile contains only whole
+ * real tiles.
+ *
+ * @param subsampling the subsampling which will be applied in a read
operation.
+ * @return the size of tiles (in pixels) in this resource.
+ * @throws DataStoreException if an error occurred while fetching the tile
size.
+ */
+ protected long[] getVirtualTileSize(int[] subsampling) throws
DataStoreException {
+ return ArraysExt.copyAsLongs(getTileSize());
+ }
+
/**
* Returns the number of sample values in an indivisible element of a tile.
* An element is a primitive type such as {@code byte}, {@code int} or
{@code float}.
@@ -411,8 +434,12 @@ check: if (dataType.isInteger()) {
/**
* Size of tiles (or chunks) in the resource, without clipping and
subsampling.
+ * May be a virtual tile size (i.e., tiles larger than the real tiles)
if the
+ * resource can easily coalesce many tiles in a single read operation.
+ *
+ * @see #getVirtualTileSize(int[])
*/
- final int[] tileSize;
+ final long[] virtualTileSize;
/**
* The sample model for the bands to read (not the full set of bands
in the resource).
@@ -456,7 +483,7 @@ check: if (dataType.isInteger()) {
final RangeArgument rangeIndices =
RangeArgument.validate(bands.size(), range, listeners);
final GridGeometry gridGeometry = getGridGeometry();
sourceExtent = gridGeometry.getExtent();
- tileSize = getTileSize();
+ final int[] tileSize = getTileSize();
boolean sharedCache = true;
if (domain == null) {
domain = gridGeometry;
@@ -507,6 +534,13 @@ check: if (dataType.isInteger()) {
subsampling = target.getSubsampling();
subsamplingOffsets = target.getSubsamplingOffsets();
}
+ /*
+ * Virtual tile size is usually the same as the real tile size.
+ */
+ virtualTileSize = getVirtualTileSize(subsampling);
+ for (int i=0; i < virtualTileSize.length; i++) {
+ virtualTileSize[i] = Math.min(sourceExtent.getSize(i),
Math.max(tileSize[i], virtualTileSize[i]));
+ }
/*
* Get the bands selected by user in strictly increasing order of
source band index.
* If user has specified bands in a different order, that change
of band order will
@@ -580,7 +614,7 @@ check: if (dataType.isInteger()) {
return false;
}
for (int i = subsampling.length; --i >= 0;) {
- if (subsampling[i] >= tileSize[i]) {
+ if (subsampling[i] >= virtualTileSize[i]) {
return false;
}
}
diff --git
a/incubator/src/org.apache.sis.storage.gdal/main/org/apache/sis/storage/gdal/TiledResource.java
b/incubator/src/org.apache.sis.storage.gdal/main/org/apache/sis/storage/gdal/TiledResource.java
index 45e42ee8f6..1a71900078 100644
---
a/incubator/src/org.apache.sis.storage.gdal/main/org/apache/sis/storage/gdal/TiledResource.java
+++
b/incubator/src/org.apache.sis.storage.gdal/main/org/apache/sis/storage/gdal/TiledResource.java
@@ -551,6 +551,21 @@ final class TiledResource extends TiledGridResource {
return new int[] {tileWidth, tileHeight};
}
+ /**
+ * Returns virtual tile size to use during read operations. From the point
of view of this class,
+ * pretending that tiles have a size different than their real size is
easy because <abbr>GDAL</abbr>
+ * does the hard work of reading only the relevant parts of the real
tiles. Therefore, we compensate
+ * subsampling with larger virtual size for avoiding that subsampled tiles
become too small.
+ * This is useful in particular with pyramided images read with
potentially high subsampling values.
+ */
+ @Override
+ protected long[] getVirtualTileSize(final int[] subsampling) {
+ return new long[] {
+ Math.multiplyFull(subsampling[0], tileWidth),
+ Math.multiplyFull(subsampling[1], tileHeight)
+ };
+ }
+
/**
* Loads a subset of the grid coverage represented by this resource.
* The actual loading may be deferred until a tile is requested for the
first time.
