This is an automated email from the ASF dual-hosted git repository. desruisseaux pushed a commit to branch geoapi-4.0 in repository https://gitbox.apache.org/repos/asf/sis.git
commit 0a359acef97da1242e78840217478f4d2554a87c Author: jsorel <johann.so...@geomatys.com> AuthorDate: Wed Nov 8 12:17:56 2023 +0100 feat(Shapefile): add shp bbox filter, add dbf field selection filter --- .../sis/storage/shapefile/ShapefileStore.java | 65 ++++- .../sis/storage/shapefile/dbf/DBFReader.java | 29 +- .../shapefile/shp/ShapeGeometryEncoder.java | 303 +++++++++++++-------- .../sis/storage/shapefile/shp/ShapeReader.java | 12 +- .../sis/storage/shapefile/shp/ShapeRecord.java | 34 ++- .../sis/storage/shapefile/ShapefileStoreTest.java | 26 ++ .../sis/storage/shapefile/dbf/DBFIOTest.java | 26 +- .../sis/storage/shapefile/shp/ShapeIOTest.java | 56 +++- 8 files changed, 397 insertions(+), 154 deletions(-) diff --git a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/ShapefileStore.java b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/ShapefileStore.java index 33c46f120d..a95c5de705 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/ShapefileStore.java +++ b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/ShapefileStore.java @@ -16,6 +16,7 @@ */ package org.apache.sis.storage.shapefile; +import java.awt.geom.Rectangle2D; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; @@ -26,7 +27,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Iterator; +import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.Spliterator; import java.util.Spliterators; import java.util.concurrent.locks.ReadWriteLock; @@ -54,6 +57,7 @@ import org.apache.sis.referencing.CommonCRS; import org.apache.sis.storage.AbstractFeatureSet; import org.apache.sis.storage.DataStore; import org.apache.sis.storage.DataStoreException; +import org.apache.sis.storage.FeatureQuery; import org.apache.sis.storage.FeatureSet; import org.apache.sis.storage.Query; import org.apache.sis.storage.UnsupportedQueryException; @@ -72,6 +76,9 @@ import org.apache.sis.util.collection.BackingStoreException; // Specific to the geoapi-3.1 and geoapi-4.0 branches: import org.opengis.feature.Feature; import org.opengis.feature.FeatureType; +import org.opengis.filter.Expression; +import org.opengis.filter.Filter; +import org.opengis.filter.SpatialOperatorName; /** @@ -88,8 +95,7 @@ public final class ShapefileStore extends DataStore implements FeatureSet { /** * Internal class to inherit AbstractFeatureSet. */ - private final AsFeatureSet featureSetView = new AsFeatureSet(); - private FeatureType type; + private final AsFeatureSet featureSetView = new AsFeatureSet(null, null); private Charset charset; /** @@ -149,8 +155,19 @@ public final class ShapefileStore extends DataStore implements FeatureSet { private class AsFeatureSet extends AbstractFeatureSet implements WritableFeatureSet { - private AsFeatureSet() { + private final Rectangle2D.Double filter; + private final Set<String> dbfProperties; + private int[] dbfPropertiesIndex; + private FeatureType type; + + /** + * @param filter optional shape filter, must be in data CRS + * @param properties dbf properties to read, null for all properties + */ + private AsFeatureSet(Rectangle2D.Double filter, Set<String> properties) { super(null); + this.filter = filter; + this.dbfProperties = properties; } @Override @@ -165,7 +182,7 @@ public final class ShapefileStore extends DataStore implements FeatureSet { //read shp header to obtain geometry type final Class geometryClass; - try (final ShapeReader reader = new ShapeReader(ShpFiles.openReadChannel(shpPath))) { + try (final ShapeReader reader = new ShapeReader(ShpFiles.openReadChannel(shpPath), filter)) { final ShapeHeader header = reader.getHeader(); geometryClass = ShapeGeometryEncoder.getEncoder(header.shapeType).getValueClass(); } catch (IOException ex) { @@ -204,10 +221,25 @@ public final class ShapefileStore extends DataStore implements FeatureSet { //read dbf for attributes final Path dbfFile = files.getDbf(false); if (dbfFile != null) { - try (DBFReader reader = new DBFReader(ShpFiles.openReadChannel(dbfFile), charset)) { + try (DBFReader reader = new DBFReader(ShpFiles.openReadChannel(dbfFile), charset, null)) { final DBFHeader header = reader.getHeader(); boolean hasId = false; - for (DBFField field : header.fields) { + + if (dbfProperties == null) { + dbfPropertiesIndex = new int[header.fields.length]; + } else { + dbfPropertiesIndex = new int[dbfProperties.size()]; + } + + for (int i = 0,idx=0; i < header.fields.length; i++) { + final DBFField field = header.fields[i]; + if (dbfProperties != null && !dbfProperties.contains(field.fieldName)) { + //skip unwanted fields + continue; + } + dbfPropertiesIndex[idx] = i; + idx++; + final AttributeTypeBuilder atb = ftb.addAttribute(field.getEncoder().getValueClass()).setName(field.fieldName); //no official but 'id' field is common if (!hasId && "id".equalsIgnoreCase(field.fieldName) || "identifier".equalsIgnoreCase(field.fieldName)) { @@ -233,8 +265,8 @@ public final class ShapefileStore extends DataStore implements FeatureSet { final ShapeReader shpreader; final DBFReader dbfreader; try { - shpreader = new ShapeReader(ShpFiles.openReadChannel(files.shpFile)); - dbfreader = new DBFReader(ShpFiles.openReadChannel(files.getDbf(false)), charset); + shpreader = new ShapeReader(ShpFiles.openReadChannel(files.shpFile), filter); + dbfreader = new DBFReader(ShpFiles.openReadChannel(files.getDbf(false)), charset, dbfPropertiesIndex); } catch (IOException ex) { throw new DataStoreException("Faild to open shp and dbf files.", ex); } @@ -246,10 +278,12 @@ public final class ShapefileStore extends DataStore implements FeatureSet { try { final ShapeRecord shpRecord = shpreader.next(); if (shpRecord == null) return false; + //move dbf to record offset, some shp record might have been skipped because of filter + dbfreader.moveToOffset(header.headerSize + (shpRecord.recordNumber-1) * header.recordSize); final DBFRecord dbfRecord = dbfreader.next(); final Feature next = type.newInstance(); next.setPropertyValue(GEOMETRY_NAME, shpRecord.geometry); - for (int i = 0; i < header.fields.length; i++) { + for (int i = 0; i < dbfPropertiesIndex.length; i++) { next.setPropertyValue(header.fields[i].fieldName, dbfRecord.fields[i]); } action.accept(next); @@ -274,6 +308,19 @@ public final class ShapefileStore extends DataStore implements FeatureSet { } + @Override + public FeatureSet subset(Query query) throws UnsupportedQueryException, DataStoreException { + //try to optimise the query for common cases + if (query instanceof FeatureQuery) { + final FeatureQuery fq = (FeatureQuery) query; + //todo + } + + return super.subset(query); + } + + + @Override public void updateType(FeatureType newType) throws DataStoreException { throw new UnsupportedOperationException("Not supported yet."); diff --git a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/dbf/DBFReader.java b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/dbf/DBFReader.java index f7893632a8..3c235ec263 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/dbf/DBFReader.java +++ b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/dbf/DBFReader.java @@ -33,12 +33,19 @@ public final class DBFReader implements AutoCloseable { private final ChannelDataInput channel; private final DBFHeader header; + private final int[] fieldsToRead; private int nbRead = 0; - public DBFReader(ChannelDataInput channel, Charset charset) throws IOException { + /** + * @param channel to read from + * @param charset text encoding + * @param fieldsToRead fields index in the header to decode, other fields will be skipped. must be in increment order. + */ + public DBFReader(ChannelDataInput channel, Charset charset, int[] fieldsToRead) throws IOException { this.channel = channel; this.header = new DBFHeader(); this.header.read(channel, charset); + this.fieldsToRead = fieldsToRead; } public DBFHeader getHeader() { @@ -71,9 +78,25 @@ public final class DBFReader implements AutoCloseable { final DBFRecord record = new DBFRecord(); record.fields = new Object[header.fields.length]; - for (int i = 0; i < header.fields.length; i++) { - record.fields[i] = header.fields[i].getEncoder().read(channel); + if (fieldsToRead == null) { + //read all fields + record.fields = new Object[header.fields.length]; + for (int i = 0; i < header.fields.length; i++) { + record.fields[i] = header.fields[i].getEncoder().read(channel); + } + } else { + //read only selected fields + record.fields = new Object[fieldsToRead.length]; + for (int i = 0,k = 0; i < header.fields.length; i++) { + if (k < fieldsToRead.length && fieldsToRead[k] == i) { + record.fields[k++] = header.fields[i].getEncoder().read(channel); + } else { + //skip this field + channel.skipBytes(header.fields[i].fieldLength); + } + } } + return record; } diff --git a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeGeometryEncoder.java b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeGeometryEncoder.java index 1f122301fb..ff29bac57b 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeGeometryEncoder.java +++ b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeGeometryEncoder.java @@ -16,11 +16,13 @@ */ package org.apache.sis.storage.shapefile.shp; +import java.awt.geom.Rectangle2D; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.sis.geometry.Envelope2D; import org.apache.sis.geometry.GeneralEnvelope; import org.apache.sis.io.stream.ChannelDataInput; import org.apache.sis.io.stream.ChannelDataOutput; @@ -115,9 +117,23 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { return measures; } - public abstract void decode(ChannelDataInput ds, ShapeRecord record) throws IOException; + /** + * Decode geometry and store it in ShapeRecord. + * + * @param channel to read from + * @param record to read into + * @param filter optional filter envelope to stop geometry decoding as soon as possible + * @return true if geometry pass the filter + */ + public abstract boolean decode(ChannelDataInput channel, ShapeRecord record, Rectangle2D.Double filter) throws IOException; - public abstract void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException; + /** + * Encode geometry. + * + * @param channel to write into + * @param shape geometry to encode + */ + public abstract void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException; /** * Compute the encoded size of a geometry. @@ -126,26 +142,56 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { */ public abstract int getEncodedLength(Geometry geom); - protected void readBBox2D(ChannelDataInput ds, ShapeRecord shape) throws IOException { + /** + * Read 2D Bounding box from channel. + * + * @param channel to read from + * @param shape to write into + * @param filter optional filter envelope to stop geometry decoding as soon as possible + * @return true if filter match or is null + */ + protected boolean readBBox2D(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final double minX = channel.readDouble(); + if (filter != null && minX > (filter.x + filter.width)) return false; + final double minY = channel.readDouble(); + if (filter != null && minY > (filter.y + filter.height)) return false; + final double maxX = channel.readDouble(); + if (filter != null && maxX < filter.x) return false; + final double maxY = channel.readDouble(); + if (filter != null && maxY < filter.y) return false; shape.bbox = new GeneralEnvelope(getDimension()); - shape.bbox.getLowerCorner().setOrdinate(0, ds.readDouble()); - shape.bbox.getLowerCorner().setOrdinate(1, ds.readDouble()); - shape.bbox.getUpperCorner().setOrdinate(0, ds.readDouble()); - shape.bbox.getUpperCorner().setOrdinate(1, ds.readDouble()); + shape.bbox.getLowerCorner().setOrdinate(0, minX); + shape.bbox.getLowerCorner().setOrdinate(1, minY); + shape.bbox.getUpperCorner().setOrdinate(0, maxX); + shape.bbox.getUpperCorner().setOrdinate(1, maxY); + return true; } - protected void writeBBox2D(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - ds.writeDouble(shape.bbox.getMinimum(0)); - ds.writeDouble(shape.bbox.getMinimum(1)); - ds.writeDouble(shape.bbox.getMaximum(0)); - ds.writeDouble(shape.bbox.getMaximum(1)); + /** + * Write 2D Bounding box. + * + * @param channel to write into + * @param shape to read from + */ + protected void writeBBox2D(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + channel.writeDouble(shape.bbox.getMinimum(0)); + channel.writeDouble(shape.bbox.getMinimum(1)); + channel.writeDouble(shape.bbox.getMaximum(0)); + channel.writeDouble(shape.bbox.getMaximum(1)); } - protected LineString[] readLines(ChannelDataInput ds, ShapeRecord shape, boolean asRing) throws IOException { - readBBox2D(ds, shape); - final int numParts = ds.readInt(); - final int numPoints = ds.readInt(); - final int[] offsets = ds.readInts(numParts); + /** + * @param channel to read from + * @param shape to write into + * @param filter optional filter envelope to stop geometry decoding as soon as possible + * @param asRing true to produce LinearRing instead of LineString + * @return null if filter do no match + */ + protected LineString[] readLines(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter, boolean asRing) throws IOException { + if (!readBBox2D(channel, shape, filter)) return null; + final int numParts = channel.readInt(); + final int numPoints = channel.readInt(); + final int[] offsets = channel.readInts(numParts); final LineString[] lines = new LineString[numParts]; @@ -154,37 +200,37 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { final int nbValues = (i == numParts - 1) ? numPoints - offsets[i] : offsets[i + 1] - offsets[i]; final double[] values; if (nbOrdinates == 2) { - values = ds.readDoubles(nbValues * 2); + values = channel.readDoubles(nbValues * 2); } else { - values = ds.readDoubles(nbValues * nbOrdinates); + values = channel.readDoubles(nbValues * nbOrdinates); for (int k = 0; k < nbValues; k++) { - values[k * nbOrdinates ] = ds.readDouble(); - values[k * nbOrdinates + 1] = ds.readDouble(); + values[k * nbOrdinates ] = channel.readDouble(); + values[k * nbOrdinates + 1] = channel.readDouble(); } } final PackedCoordinateSequence.Double pc = new PackedCoordinateSequence.Double(values, getDimension(), getMeasures()); lines[i] = asRing ? GF.createLinearRing(pc) : GF.createLineString(pc); } //Z and M - if (nbOrdinates >= 3) readLineOrdinates(ds, shape, lines, 2); - if (nbOrdinates == 4) readLineOrdinates(ds, shape, lines, 3); + if (nbOrdinates >= 3) readLineOrdinates(channel, shape, lines, 2); + if (nbOrdinates == 4) readLineOrdinates(channel, shape, lines, 3); return lines; } - protected void readLineOrdinates(ChannelDataInput ds, ShapeRecord shape, LineString[] lines, int ordinateIndex) throws IOException { + protected void readLineOrdinates(ChannelDataInput channel, ShapeRecord shape, LineString[] lines, int ordinateIndex) throws IOException { final int nbDim = getDimension() + getMeasures(); - shape.bbox.setRange(ordinateIndex, ds.readDouble(), ds.readDouble()); + shape.bbox.setRange(ordinateIndex, channel.readDouble(), channel.readDouble()); for (LineString line : lines) { final double[] values = ((PackedCoordinateSequence.Double) line.getCoordinateSequence()).getRawCoordinates(); final int nbValues = values.length / nbDim; for (int k = 0; k < nbValues; k++) { - values[k * nbDim + ordinateIndex] = ds.readDouble(); + values[k * nbDim + ordinateIndex] = channel.readDouble(); } } } - protected void writeLines(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - writeBBox2D(ds, shape); + protected void writeLines(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + writeBBox2D(channel, shape); final List<LineString> lines = extractRings(shape.geometry); final int nbLines = lines.size(); final int[] offsets = new int[nbLines]; @@ -195,32 +241,32 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { offsets[i] = nbPts; nbPts += line.getCoordinateSequence().size(); } - ds.writeInt(nbLines); - ds.writeInt(nbPts); - ds.writeInts(offsets); + channel.writeInt(nbLines); + channel.writeInt(nbPts); + channel.writeInts(offsets); //second loop write points for (int i = 0; i < nbLines; i++) { final LineString line = lines.get(i); final CoordinateSequence cs = line.getCoordinateSequence(); for (int k = 0, kn =cs.size(); k < kn; k++) { - ds.writeDouble(cs.getX(k)); - ds.writeDouble(cs.getY(k)); + channel.writeDouble(cs.getX(k)); + channel.writeDouble(cs.getY(k)); } } //Z and M - if (nbOrdinates >= 3) writeLineOrdinates(ds, shape, lines, 2); - if (nbOrdinates == 4) writeLineOrdinates(ds, shape, lines, 3); + if (nbOrdinates >= 3) writeLineOrdinates(channel, shape, lines, 2); + if (nbOrdinates == 4) writeLineOrdinates(channel, shape, lines, 3); } - protected void writeLineOrdinates(ChannelDataOutput ds, ShapeRecord shape,List<LineString> lines, int ordinateIndex) throws IOException { - ds.writeDouble(shape.bbox.getMinimum(ordinateIndex)); - ds.writeDouble(shape.bbox.getMaximum(ordinateIndex)); + protected void writeLineOrdinates(ChannelDataOutput channel, ShapeRecord shape,List<LineString> lines, int ordinateIndex) throws IOException { + channel.writeDouble(shape.bbox.getMinimum(ordinateIndex)); + channel.writeDouble(shape.bbox.getMaximum(ordinateIndex)); for (LineString line : lines) { final CoordinateSequence cs = line.getCoordinateSequence(); for (int k = 0, kn =cs.size(); k < kn; k++) { - ds.writeDouble(cs.getOrdinate(k, ordinateIndex)); + channel.writeDouble(cs.getOrdinate(k, ordinateIndex)); } } } @@ -250,6 +296,11 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } } + /** + * Create a MultiPolygon from given set of rings. + * @param rings to create MultiPolygon from + * @return created MultiPolygon + */ protected MultiPolygon rebuild(List<LinearRing> rings) { final int nbRing = rings.size(); @@ -321,11 +372,12 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { } } @@ -339,21 +391,24 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final double x = channel.readDouble(); + if (filter != null && (x < filter.x || x > (filter.x + filter.width)) ) return false; + final double y = channel.readDouble(); + if (filter != null && (y < filter.y || y > (filter.y + filter.height)) ) return false; shape.bbox = new GeneralEnvelope(2); - final double x = ds.readDouble(); - final double y = ds.readDouble(); shape.bbox.setRange(0, x, x); shape.bbox.setRange(1, y, y); shape.geometry = GF.createPoint(new CoordinateXY(x, y)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { final Point pt = (Point) shape.geometry; final Coordinate coord = pt.getCoordinate(); - ds.writeDouble(coord.getX()); - ds.writeDouble(coord.getY()); + channel.writeDouble(coord.getX()); + channel.writeDouble(coord.getY()); } @Override @@ -370,24 +425,27 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final double x = channel.readDouble(); + if (filter != null && (x < filter.x || x > (filter.x + filter.width)) ) return false; + final double y = channel.readDouble(); + if (filter != null && (y < filter.y || y > (filter.y + filter.height)) ) return false; + final double z = channel.readDouble(); shape.bbox = new GeneralEnvelope(3); - final double x = ds.readDouble(); - final double y = ds.readDouble(); - final double z = ds.readDouble(); shape.bbox.setRange(0, x, x); shape.bbox.setRange(1, y, y); shape.bbox.setRange(2, z, z); shape.geometry = GF.createPoint(new CoordinateXYM(x, y, z)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { final Point pt = (Point) shape.geometry; final Coordinate coord = pt.getCoordinate(); - ds.writeDouble(coord.getX()); - ds.writeDouble(coord.getY()); - ds.writeDouble(coord.getM()); + channel.writeDouble(coord.getX()); + channel.writeDouble(coord.getY()); + channel.writeDouble(coord.getM()); } @Override @@ -405,27 +463,30 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final double x = channel.readDouble(); + if (filter != null && (x < filter.x || x > (filter.x + filter.width)) ) return false; + final double y = channel.readDouble(); + if (filter != null && (y < filter.y || y > (filter.y + filter.height)) ) return false; + final double z = channel.readDouble(); + final double m = channel.readDouble(); shape.bbox = new GeneralEnvelope(4); - final double x = ds.readDouble(); - final double y = ds.readDouble(); - final double z = ds.readDouble(); - final double m = ds.readDouble(); shape.bbox.setRange(0, x, x); shape.bbox.setRange(1, y, y); shape.bbox.setRange(2, z, z); shape.bbox.setRange(3, m, m); shape.geometry = GF.createPoint(new CoordinateXYZM(x, y, z, m)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { final Point pt = (Point) shape.geometry; final Coordinate coord = pt.getCoordinate(); - ds.writeDouble(coord.getX()); - ds.writeDouble(coord.getY()); - ds.writeDouble(coord.getZ()); - ds.writeDouble(coord.getM()); + channel.writeDouble(coord.getX()); + channel.writeDouble(coord.getY()); + channel.writeDouble(coord.getZ()); + channel.writeDouble(coord.getM()); } @Override @@ -442,23 +503,24 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { - readBBox2D(ds, shape); - int nbPt = ds.readInt(); - final double[] coords = ds.readDoubles(nbPt * 2); + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + if (!readBBox2D(channel, shape, filter)) return false; + int nbPt = channel.readInt(); + final double[] coords = channel.readDoubles(nbPt * 2); shape.geometry = GF.createMultiPoint(new PackedCoordinateSequence.Double(coords,2,0)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - writeBBox2D(ds, shape); + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + writeBBox2D(channel, shape); final MultiPoint geometry = (MultiPoint) shape.geometry; final int nbPts = geometry.getNumGeometries(); - ds.writeInt(nbPts); + channel.writeInt(nbPts); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getX()); - ds.writeDouble(pt.getY()); + channel.writeDouble(pt.getX()); + channel.writeDouble(pt.getY()); } } @@ -479,37 +541,38 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { - readBBox2D(ds, shape); - int nbPt = ds.readInt(); + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + if (!readBBox2D(channel, shape, filter)) return false; + int nbPt = channel.readInt(); final double[] coords = new double[nbPt * 3]; for (int i = 0; i < nbPt; i++) { - coords[i * 3 ] = ds.readDouble(); - coords[i * 3 + 1] = ds.readDouble(); + coords[i * 3 ] = channel.readDouble(); + coords[i * 3 + 1] = channel.readDouble(); } - shape.bbox.setRange(2, ds.readDouble(), ds.readDouble()); + shape.bbox.setRange(2, channel.readDouble(), channel.readDouble()); for (int i = 0; i < nbPt; i++) { - coords[i * 3 + 2] = ds.readDouble(); + coords[i * 3 + 2] = channel.readDouble(); } shape.geometry = GF.createMultiPoint(new PackedCoordinateSequence.Double(coords, 2, 1)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - writeBBox2D(ds, shape); + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + writeBBox2D(channel, shape); final MultiPoint geometry = (MultiPoint) shape.geometry; final int nbPts = geometry.getNumGeometries(); - ds.writeInt(nbPts); + channel.writeInt(nbPts); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getX()); - ds.writeDouble(pt.getY()); + channel.writeDouble(pt.getX()); + channel.writeDouble(pt.getY()); } - ds.writeDouble(shape.bbox.getMinimum(2)); - ds.writeDouble(shape.bbox.getMaximum(2)); + channel.writeDouble(shape.bbox.getMinimum(2)); + channel.writeDouble(shape.bbox.getMaximum(2)); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getCoordinate().getM()); + channel.writeDouble(pt.getCoordinate().getM()); } } @@ -530,47 +593,48 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { - readBBox2D(ds, shape); - int nbPt = ds.readInt(); + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + if (!readBBox2D(channel, shape, filter)) return false; + int nbPt = channel.readInt(); final double[] coords = new double[nbPt * 4]; for (int i = 0; i < nbPt; i++) { - coords[i * 4 ] = ds.readDouble(); - coords[i * 4 + 1] = ds.readDouble(); + coords[i * 4 ] = channel.readDouble(); + coords[i * 4 + 1] = channel.readDouble(); } - shape.bbox.setRange(2, ds.readDouble(), ds.readDouble()); + shape.bbox.setRange(2, channel.readDouble(), channel.readDouble()); for (int i = 0; i < nbPt; i++) { - coords[i * 4 + 2] = ds.readDouble(); + coords[i * 4 + 2] = channel.readDouble(); } - shape.bbox.setRange(3, ds.readDouble(), ds.readDouble()); + shape.bbox.setRange(3, channel.readDouble(), channel.readDouble()); for (int i = 0; i < nbPt; i++) { - coords[i * 4 + 3] = ds.readDouble(); + coords[i * 4 + 3] = channel.readDouble(); } shape.geometry = GF.createMultiPoint(new PackedCoordinateSequence.Double(coords, 3, 1)); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - writeBBox2D(ds, shape); + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + writeBBox2D(channel, shape); final MultiPoint geometry = (MultiPoint) shape.geometry; final int nbPts = geometry.getNumGeometries(); - ds.writeInt(nbPts); + channel.writeInt(nbPts); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getX()); - ds.writeDouble(pt.getY()); + channel.writeDouble(pt.getX()); + channel.writeDouble(pt.getY()); } - ds.writeDouble(shape.bbox.getMinimum(2)); - ds.writeDouble(shape.bbox.getMaximum(2)); + channel.writeDouble(shape.bbox.getMinimum(2)); + channel.writeDouble(shape.bbox.getMaximum(2)); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getCoordinate().getZ()); + channel.writeDouble(pt.getCoordinate().getZ()); } - ds.writeDouble(shape.bbox.getMinimum(3)); - ds.writeDouble(shape.bbox.getMaximum(3)); + channel.writeDouble(shape.bbox.getMinimum(3)); + channel.writeDouble(shape.bbox.getMaximum(3)); for (int i = 0; i < nbPts; i++) { final Point pt = (Point) geometry.getGeometryN(i); - ds.writeDouble(pt.getCoordinate().getM()); + channel.writeDouble(pt.getCoordinate().getM()); } } @@ -593,13 +657,16 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { - shape.geometry = GF.createMultiLineString(readLines(ds, shape, false)); + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final LineString[] lines = readLines(channel, shape, filter, false); + if (lines == null) return false; + shape.geometry = GF.createMultiLineString(lines); + return true; } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { - writeLines(ds, shape); + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { + writeLines(channel, shape); } @Override @@ -625,9 +692,11 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { - final LineString[] rings = readLines(ds, shape, true); + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { + final LineString[] rings = readLines(channel, shape, filter, true); + if (rings == null) return false; shape.geometry = rebuild(Stream.of(rings).map(LinearRing.class::cast).collect(Collectors.toList())); + return true; } @Override @@ -660,12 +729,12 @@ public abstract class ShapeGeometryEncoder<T extends Geometry> { } @Override - public void decode(ChannelDataInput ds, ShapeRecord shape) throws IOException { + public boolean decode(ChannelDataInput channel, ShapeRecord shape, Rectangle2D.Double filter) throws IOException { throw new UnsupportedOperationException("Not supported yet."); } @Override - public void encode(ChannelDataOutput ds, ShapeRecord shape) throws IOException { + public void encode(ChannelDataOutput channel, ShapeRecord shape) throws IOException { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeReader.java b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeReader.java index 904a02934f..9e4b325112 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeReader.java +++ b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeReader.java @@ -16,10 +16,12 @@ */ package org.apache.sis.storage.shapefile.shp; +import java.awt.geom.Rectangle2D; import org.apache.sis.io.stream.ChannelDataInput; import java.io.EOFException; import java.io.IOException; +import org.apache.sis.geometry.Envelope2D; /** * Seekable shape file reader. @@ -31,9 +33,11 @@ public final class ShapeReader implements AutoCloseable{ private final ChannelDataInput channel; private final ShapeHeader header; private final ShapeGeometryEncoder geomParser; + private final Rectangle2D.Double filter; - public ShapeReader(ChannelDataInput channel) throws IOException { + public ShapeReader(ChannelDataInput channel, Rectangle2D.Double filter) throws IOException { this.channel = channel; + this.filter = filter; header = new ShapeHeader(); header.read(channel); geomParser = ShapeGeometryEncoder.getEncoder(header.shapeType); @@ -48,10 +52,10 @@ public final class ShapeReader implements AutoCloseable{ } public ShapeRecord next() throws IOException { + final ShapeRecord record = new ShapeRecord(); try { - final ShapeRecord record = new ShapeRecord(); - record.read(channel); - record.parseGeometry(geomParser); + //read until we find a record matching the filter or EOF exception + while (!record.read(channel, geomParser, filter)) {} return record; } catch (EOFException ex) { //no more records diff --git a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeRecord.java b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeRecord.java index 8e10ceaf46..c291c9d85b 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeRecord.java +++ b/incubator/src/org.apache.sis.storage.shapefile/main/org/apache/sis/storage/shapefile/shp/ShapeRecord.java @@ -16,14 +16,15 @@ */ package org.apache.sis.storage.shapefile.shp; +import java.awt.geom.Rectangle2D; import org.apache.sis.geometry.GeneralEnvelope; import org.apache.sis.io.stream.ChannelDataInput; import org.apache.sis.io.stream.ChannelDataOutput; import org.locationtech.jts.geom.Geometry; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.apache.sis.geometry.Envelope2D; /** * @author Johann Sorel (Geomatys) @@ -45,20 +46,33 @@ public final class ShapeRecord { /** * Read this shape record. + * * @param channel input channel, not null + * @param io geometry decoder, if null gemetry content will be stored in content array, otherwise geometry will be parsed + * @param filter optional filter envelope to stop geometry decoding as soon as possible + * @return true if geometry pass the filter or if there is no filter * @throws IOException if an error occurred while reading. */ - public void read(final ChannelDataInput channel) throws IOException { + public boolean read(final ChannelDataInput channel, ShapeGeometryEncoder io, Rectangle2D.Double filter) throws IOException { + if (io == null && filter != null) throw new IllegalArgumentException("filter must be null if encoder is null"); + channel.buffer.order(ByteOrder.BIG_ENDIAN); recordNumber = channel.readInt(); - content = channel.readBytes(channel.readInt() * 2); // x2 because size is in 16bit words - } - - public void parseGeometry(ShapeGeometryEncoder io) throws IOException { - final ChannelDataInput di = new ChannelDataInput("", ByteBuffer.wrap(content)); - di.buffer.order(ByteOrder.LITTLE_ENDIAN); - int shapeType = di.readInt(); - io.decode(di,this); + final int byteSize = channel.readInt() * 2; // x2 because size is in 16bit words + final long position = channel.getStreamPosition(); + channel.buffer.order(ByteOrder.LITTLE_ENDIAN); + final int shapeType = channel.readInt(); + if (io == null) { + content = channel.readBytes(byteSize); + return true; + } else { + final boolean match = io.decode(channel,this, filter); + if (!match) { + //move to record end + channel.seek(position + byteSize); + } + return match; + } } /** diff --git a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/ShapefileStoreTest.java b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/ShapefileStoreTest.java index 0cdb00b890..1a6ba9d6d0 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/ShapefileStoreTest.java +++ b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/ShapefileStoreTest.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; import org.apache.sis.storage.DataStoreException; +import org.junit.Ignore; import org.junit.Test; import org.locationtech.jts.geom.Point; @@ -87,4 +88,29 @@ public class ShapefileStoreTest { assertFalse(iterator.hasNext()); } } + + /** + * TODO not implemented yet. + */ + @Ignore + @Test + public void testEnvelopeFilter() throws URISyntaxException, DataStoreException { + final URL url = ShapefileStoreTest.class.getResource("/org/apache/sis/storage/shapefile/point.shp"); + final ShapefileStore store = new ShapefileStore(Paths.get(url.toURI())); + + try (Stream<Feature> stream = store.features(false)) { + Iterator<Feature> iterator = stream.iterator(); + assertTrue(iterator.hasNext()); + Feature feature = iterator.next(); + assertEquals(2L, feature.getPropertyValue("id")); + assertEquals("text2", feature.getPropertyValue("text")); + assertEquals(40L, feature.getPropertyValue("integer")); + assertEquals(60.0, feature.getPropertyValue("float")); + assertEquals(LocalDate.of(2023, 10, 28), feature.getPropertyValue("date")); + Point pt2 = (Point) feature.getPropertyValue("geometry"); + + assertFalse(iterator.hasNext()); + } + } + } diff --git a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/dbf/DBFIOTest.java b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/dbf/DBFIOTest.java index 46fa04af28..4a822860f8 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/dbf/DBFIOTest.java +++ b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/dbf/DBFIOTest.java @@ -45,7 +45,7 @@ public class DBFIOTest { final String path = "/org/apache/sis/storage/shapefile/point.dbf"; final ChannelDataInput cdi = openRead(path); - try (DBFReader reader = new DBFReader(cdi, StandardCharsets.UTF_8)) { + try (DBFReader reader = new DBFReader(cdi, StandardCharsets.UTF_8, null)) { final DBFHeader header = reader.getHeader(); assertEquals(123, header.year); assertEquals(10, header.month); @@ -98,7 +98,29 @@ public class DBFIOTest { //no more records assertNull(reader.next()); } - } + /** + * Test reading only selected fields. + */ + @Test + public void readSelectionTest() throws DataStoreException, IOException { + final String path = "/org/apache/sis/storage/shapefile/point.dbf"; + final ChannelDataInput cdi = openRead(path); + + try (DBFReader reader = new DBFReader(cdi, StandardCharsets.UTF_8, new int[]{1,3})) { + final DBFHeader header = reader.getHeader(); + + final DBFRecord record1 = reader.next(); + assertEquals("text1", record1.fields[0]); + assertEquals(20.0, record1.fields[1]); + + final DBFRecord record2 = reader.next(); + assertEquals("text2", record2.fields[0]); + assertEquals(60.0, record2.fields[1]); + + //no more records + assertNull(reader.next()); + } + } } diff --git a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/shp/ShapeIOTest.java b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/shp/ShapeIOTest.java index 1425e6b226..eaec0af3d4 100644 --- a/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/shp/ShapeIOTest.java +++ b/incubator/src/org.apache.sis.storage.shapefile/test/org/apache/sis/storage/shapefile/shp/ShapeIOTest.java @@ -30,7 +30,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import org.apache.sis.geometry.Envelope2D; import org.apache.sis.io.stream.ChannelDataOutput; +import org.apache.sis.referencing.CommonCRS; import org.apache.sis.storage.DataStoreException; import org.locationtech.jts.geom.CoordinateSequence; import org.locationtech.jts.geom.LineString; @@ -78,7 +80,7 @@ public class ShapeIOTest { final ChannelDataOutput cdo = openWrite(tempFile); try { - try (ShapeReader reader = new ShapeReader(cdi); + try (ShapeReader reader = new ShapeReader(cdi, null); ShapeWriter writer = new ShapeWriter(cdo)) { writer.write(reader.getHeader()); @@ -105,9 +107,8 @@ public class ShapeIOTest { @Test public void testPoint() throws Exception { final String path = "/org/apache/sis/storage/shapefile/point.shp"; - final ChannelDataInput cdi = openRead(path); - try (ShapeReader reader = new ShapeReader(cdi)) { + try (ShapeReader reader = new ShapeReader(openRead(path), null)) { final ShapeRecord record1 = reader.next(); assertEquals(2, record1.bbox.getDimension()); assertEquals(-38.5, record1.bbox.getMinimum(0), 0.1); @@ -134,6 +135,16 @@ public class ShapeIOTest { assertNull(reader.next()); } + //test filter, envelope contains record 2 + final Envelope2D filter = new Envelope2D(CommonCRS.WGS84.normalizedGeographic(), 2, 42, 1, 1); + try (ShapeReader reader = new ShapeReader(openRead(path), filter)) { + final ShapeRecord record = reader.next(); + assertEquals(2, record.recordNumber); + + //no more records + assertNull(reader.next()); + } + testReadAndWrite(path); } @@ -143,9 +154,8 @@ public class ShapeIOTest { @Test public void testMultiPoint() throws Exception { final String path = "/org/apache/sis/storage/shapefile/multipoint.shp"; - final ChannelDataInput cdi = openRead(path); - try (ShapeReader reader = new ShapeReader(cdi)) { + try (ShapeReader reader = new ShapeReader(openRead(path), null)) { final ShapeRecord record1 = reader.next(); assertEquals(2, record1.bbox.getDimension()); assertEquals(-38.0, record1.bbox.getMinimum(0), 0.1); @@ -182,6 +192,16 @@ public class ShapeIOTest { assertNull(reader.next()); } + //test filter, envelope inside record 2 + final Envelope2D filter = new Envelope2D(CommonCRS.WGS84.normalizedGeographic(), 4, 15, 1, 1); + try (ShapeReader reader = new ShapeReader(openRead(path), filter)) { + final ShapeRecord record = reader.next(); + assertEquals(2, record.recordNumber); + + //no more records + assertNull(reader.next()); + } + testReadAndWrite(path); } @@ -191,9 +211,8 @@ public class ShapeIOTest { @Test public void testPolyline() throws Exception { final String path = "/org/apache/sis/storage/shapefile/polyline.shp"; - final ChannelDataInput cdi = openRead(path); - try (ShapeReader reader = new ShapeReader(cdi)) { + try (ShapeReader reader = new ShapeReader(openRead(path), null)) { //first record has a single 3 points line final ShapeRecord record1 = reader.next(); @@ -241,6 +260,16 @@ public class ShapeIOTest { assertNull(reader.next()); } + //test filter, envelope intersects record 2 + final Envelope2D filter = new Envelope2D(CommonCRS.WGS84.normalizedGeographic(), 0, 6, 1, 1); + try (ShapeReader reader = new ShapeReader(openRead(path), filter)) { + final ShapeRecord record = reader.next(); + assertEquals(2, record.recordNumber); + + //no more records + assertNull(reader.next()); + } + testReadAndWrite(path); } @@ -251,9 +280,8 @@ public class ShapeIOTest { @Test public void testPolygon() throws Exception { final String path = "/org/apache/sis/storage/shapefile/polygon.shp"; - final ChannelDataInput cdi = openRead(path); - try (ShapeReader reader = new ShapeReader(cdi)) { + try (ShapeReader reader = new ShapeReader(openRead(path), null)) { final ShapeRecord record1 = reader.next(); assertEquals(2, record1.bbox.getDimension()); assertEquals(-43.8, record1.bbox.getMinimum(0), 0.1); @@ -314,6 +342,16 @@ public class ShapeIOTest { assertNull(reader.next()); } + //test filter, envelope intersects record 1 + final Envelope2D filter = new Envelope2D(CommonCRS.WGS84.normalizedGeographic(), -35, 5, 1, 1); + try (ShapeReader reader = new ShapeReader(openRead(path), filter)) { + final ShapeRecord record = reader.next(); + assertEquals(1, record.recordNumber); + + //no more records + assertNull(reader.next()); + } + testReadAndWrite(path); } }