jpountz commented on a change in pull request #539: URL: https://github.com/apache/lucene/pull/539#discussion_r768421436
########## File path: lucene/core/src/test/org/apache/lucene/codecs/TestMinimalCodec.java ########## @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; + +import java.io.IOException; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Tests to ensure that {@link Codec}s won't need to implement all formats in case where only a + * small subset of Lucene's functionality is used. + */ +public class TestMinimalCodec extends LuceneTestCase { + + public void testMinimalCodec() throws IOException { + runMinimalCodecTest(false, false); + runMinimalCodecTest(false, true); + runMinimalCodecTest(true, true); + runMinimalCodecTest(true, false); + } + + private void runMinimalCodecTest(boolean useCompoundFile, boolean useDeletes) throws IOException { + try (BaseDirectoryWrapper dir = newDirectory()) { + dir.setCheckIndexOnClose(false); // MinimalCodec is not registered with SPI Review comment: should we register these 4 codecs via SPI? ########## File path: lucene/core/src/test/org/apache/lucene/codecs/TestMinimalCodec.java ########## @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; + +import java.io.IOException; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Tests to ensure that {@link Codec}s won't need to implement all formats in case where only a + * small subset of Lucene's functionality is used. + */ +public class TestMinimalCodec extends LuceneTestCase { + + public void testMinimalCodec() throws IOException { + runMinimalCodecTest(false, false); + runMinimalCodecTest(false, true); + runMinimalCodecTest(true, true); + runMinimalCodecTest(true, false); + } + + private void runMinimalCodecTest(boolean useCompoundFile, boolean useDeletes) throws IOException { + try (BaseDirectoryWrapper dir = newDirectory()) { + dir.setCheckIndexOnClose(false); // MinimalCodec is not registered with SPI + + IndexWriterConfig writerConfig = + newIndexWriterConfig(new MockAnalyzer(random())) + .setCodec(new MinimalCodec(useCompoundFile, useDeletes)) + .setUseCompoundFile(useCompoundFile); + if (!useCompoundFile) { + writerConfig.getMergePolicy().setNoCFSRatio(0.0); + writerConfig.getMergePolicy().setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); + } + + int expectedNumDocs = 0; + try (IndexWriter writer = new IndexWriter(dir, writerConfig)) { + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.flush(); // create second segment Review comment: this comment is misplaced? ########## File path: lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java ########## @@ -203,8 +203,13 @@ private void mergeTerms(SegmentWriteState segmentWriteState, SegmentReadState se // Use the merge instance in order to reuse the same IndexInput for all terms normsMergeInstance = norms.getMergeInstance(); } - try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) { - consumer.merge(mergeState, normsMergeInstance); + try (FieldsConsumer consumer = + mergeState.mergeFieldInfos.hasIndexed() + ? codec.postingsFormat().fieldsConsumer(segmentWriteState) + : null) { + if (consumer != null) { + consumer.merge(mergeState, normsMergeInstance); + } Review comment: I think it would read better if we had the hasIndexed check outside of the try block, so that we would no longer need to check that `consumer` is not null? ########## File path: lucene/core/src/test/org/apache/lucene/codecs/TestMinimalCodec.java ########## @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; + +import java.io.IOException; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Tests to ensure that {@link Codec}s won't need to implement all formats in case where only a + * small subset of Lucene's functionality is used. + */ +public class TestMinimalCodec extends LuceneTestCase { + + public void testMinimalCodec() throws IOException { + runMinimalCodecTest(false, false); + runMinimalCodecTest(false, true); + runMinimalCodecTest(true, true); + runMinimalCodecTest(true, false); + } + + private void runMinimalCodecTest(boolean useCompoundFile, boolean useDeletes) throws IOException { + try (BaseDirectoryWrapper dir = newDirectory()) { + dir.setCheckIndexOnClose(false); // MinimalCodec is not registered with SPI + + IndexWriterConfig writerConfig = + newIndexWriterConfig(new MockAnalyzer(random())) + .setCodec(new MinimalCodec(useCompoundFile, useDeletes)) + .setUseCompoundFile(useCompoundFile); + if (!useCompoundFile) { + writerConfig.getMergePolicy().setNoCFSRatio(0.0); + writerConfig.getMergePolicy().setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); + } + + int expectedNumDocs = 0; + try (IndexWriter writer = new IndexWriter(dir, writerConfig)) { + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.flush(); // create second segment + if (useDeletes && randomBoolean()) { + writer.deleteDocuments(new MatchAllDocsQuery()); + expectedNumDocs = 0; + } + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.forceMerge(1); // test merges + if (useDeletes && randomBoolean()) { + writer.deleteDocuments(new MatchAllDocsQuery()); Review comment: It looks like your test always deletes entire segments, which is a special case for Lucene. Can you only delete individual documents (and ideally less than 25% of the index to avoid triggering a merge immediately) to make sure live docs are correctly exercised? ########## File path: lucene/core/src/test/org/apache/lucene/codecs/TestMinimalCodec.java ########## @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; + +import java.io.IOException; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Tests to ensure that {@link Codec}s won't need to implement all formats in case where only a + * small subset of Lucene's functionality is used. + */ +public class TestMinimalCodec extends LuceneTestCase { + + public void testMinimalCodec() throws IOException { + runMinimalCodecTest(false, false); + runMinimalCodecTest(false, true); + runMinimalCodecTest(true, true); + runMinimalCodecTest(true, false); + } + + private void runMinimalCodecTest(boolean useCompoundFile, boolean useDeletes) throws IOException { + try (BaseDirectoryWrapper dir = newDirectory()) { + dir.setCheckIndexOnClose(false); // MinimalCodec is not registered with SPI + + IndexWriterConfig writerConfig = + newIndexWriterConfig(new MockAnalyzer(random())) + .setCodec(new MinimalCodec(useCompoundFile, useDeletes)) + .setUseCompoundFile(useCompoundFile); + if (!useCompoundFile) { + writerConfig.getMergePolicy().setNoCFSRatio(0.0); + writerConfig.getMergePolicy().setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); + } + + int expectedNumDocs = 0; + try (IndexWriter writer = new IndexWriter(dir, writerConfig)) { + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.flush(); // create second segment + if (useDeletes && randomBoolean()) { + writer.deleteDocuments(new MatchAllDocsQuery()); + expectedNumDocs = 0; + } + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.forceMerge(1); // test merges + if (useDeletes && randomBoolean()) { + writer.deleteDocuments(new MatchAllDocsQuery()); + expectedNumDocs = 0; + } + if (randomBoolean()) { + writer.commit(); + } + + try (DirectoryReader reader = DirectoryReader.open(writer)) { + assertEquals(expectedNumDocs, reader.numDocs()); + } + } + } + } + + /** returns a basic document with no indexed fields */ + private static Document basicDocument() { + Document doc = new Document(); + if (randomBoolean()) { + doc.add(new StoredField("field", "value")); + } + return doc; Review comment: should this return an empty document directly so that we can more easily iterate on this test case when we stop writing stored fields when no stored fields are indexed? ########## File path: lucene/core/src/java/org/apache/lucene/index/FieldInfos.java ########## @@ -200,6 +204,11 @@ public boolean hasFreq() { return hasFreq; } + /** Returns true if any fields are indexed */ + public boolean hasIndexed() { Review comment: +1 to hasPostings ########## File path: lucene/core/src/test/org/apache/lucene/codecs/TestMinimalCodec.java ########## @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean; + +import java.io.IOException; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Tests to ensure that {@link Codec}s won't need to implement all formats in case where only a + * small subset of Lucene's functionality is used. + */ +public class TestMinimalCodec extends LuceneTestCase { + + public void testMinimalCodec() throws IOException { + runMinimalCodecTest(false, false); + runMinimalCodecTest(false, true); + runMinimalCodecTest(true, true); + runMinimalCodecTest(true, false); + } + + private void runMinimalCodecTest(boolean useCompoundFile, boolean useDeletes) throws IOException { + try (BaseDirectoryWrapper dir = newDirectory()) { + dir.setCheckIndexOnClose(false); // MinimalCodec is not registered with SPI + + IndexWriterConfig writerConfig = + newIndexWriterConfig(new MockAnalyzer(random())) + .setCodec(new MinimalCodec(useCompoundFile, useDeletes)) + .setUseCompoundFile(useCompoundFile); + if (!useCompoundFile) { + writerConfig.getMergePolicy().setNoCFSRatio(0.0); + writerConfig.getMergePolicy().setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); + } + + int expectedNumDocs = 0; + try (IndexWriter writer = new IndexWriter(dir, writerConfig)) { + writer.addDocument(basicDocument()); + expectedNumDocs += 1; + writer.flush(); // create second segment + if (useDeletes && randomBoolean()) { Review comment: can we rely less on randomization for coverage? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
