Author: afuchs
Date: Thu Jun 28 15:30:29 2012
New Revision: 1355044
URL: http://svn.apache.org/viewvc?rev=1355044&view=rev
Log:
ACCUMULO-652 fixed stat aggregation at higher levels, fixed some formatting,
and fixed timestamp filtering of index blocks in the IndexIterator
Modified:
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
Modified:
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
(original)
+++
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
Thu Jun 28 15:30:29 2012
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.blo
import org.apache.accumulo.core.file.blockfile.BlockFileReader;
import org.apache.accumulo.core.file.blockfile.BlockFileWriter;
import org.apache.accumulo.core.file.rfile.bcfile.Utils;
+import org.apache.accumulo.core.iterators.predicates.ColumnVisibilityPredicate;
import org.apache.accumulo.core.iterators.predicates.TimestampRangePredicate;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.hadoop.io.WritableComparable;
@@ -406,7 +407,7 @@ public class MultiLevelIndex {
writer.close(out);
}
}
-
+
public static class Writer {
private int threshold;
@@ -424,8 +425,7 @@ public class MultiLevelIndex {
levels = new ArrayList<IndexBlock>();
}
- private void add(int level, Key key, BlockStats blockStats, long offset,
long compressedSize, long rawSize, boolean last, int version)
- throws IOException {
+ private void add(int level, Key key, BlockStats blockStats, long offset,
long compressedSize, long rawSize, boolean last, int version) throws
IOException {
if (level == levels.size()) {
levels.add(new IndexBlock(level, 0));
}
@@ -443,7 +443,7 @@ public class MultiLevelIndex {
iblock.write(out);
out.close();
- add(level + 1, key, blockStats, out.getStartPos(),
out.getCompressedSize(), out.getRawSize(), last, version);
+ add(level + 1, key, iblock.blockStats, out.getStartPos(),
out.getCompressedSize(), out.getRawSize(), last, version);
if (last)
levels.set(level, null);
@@ -501,9 +501,11 @@ public class MultiLevelIndex {
class IndexIterator implements Iterator<IndexEntry> {
private Stack<StackEntry> position = new Stack<StackEntry>();
private final TimestampRangePredicate timestampFilter;
+ private final ColumnVisibilityPredicate columnVisibilityPredicate;
- private IndexIterator(TimestampRangePredicate timestampFilter, Key
lookupKey) {
+ private IndexIterator(TimestampRangePredicate timestampFilter,
ColumnVisibilityPredicate columnVisibilityPredicate, Key lookupKey) {
this.timestampFilter = timestampFilter;
+ this.columnVisibilityPredicate = columnVisibilityPredicate;
try {
seek(lookupKey);
} catch (IOException e) {
@@ -512,10 +514,10 @@ public class MultiLevelIndex {
}
private final boolean checkFilterIndexEntry(IndexEntry ie) {
- if(timestampFilter == null)
- if (timestampFilter != null && (ie.blockStats.maxTimestamp <
timestampFilter.startTimestamp || ie.blockStats.minTimestamp >
timestampFilter.endTimestamp)) {
+ if(timestampFilter != null && (ie.blockStats.maxTimestamp <
timestampFilter.startTimestamp || ie.blockStats.minTimestamp >
timestampFilter.endTimestamp))
+ return false;
+ if(columnVisibilityPredicate != null &&
ie.blockStats.minimumVisibility != null &&
ie.blockStats.minimumVisibility.evaluate(columnVisibilityPredicate.auths) ==
false)
return false;
- }
return true;
}
@@ -532,7 +534,6 @@ public class MultiLevelIndex {
}
});
-
if (pos < 0) {
pos = (pos * -1) - 1;
} else if (pos < top.block.getKeyIndex().size()) {
@@ -542,14 +543,13 @@ public class MultiLevelIndex {
}
}
-
IndexEntry ie = null;
List<IndexEntry> index = top.block.getIndex();
- if(pos > 0)
- {
- // look backwards to find any initial previousEntry that might
match the timestamp range such that no entry within the given timestamp range
is between the seeked key and the previousKey
- previousEntry = index.get(pos-1);
+ if (pos > 0) {
+ // look backwards to find any initial previousEntry that might
match the timestamp range such that no entry within the given timestamp range is
+ // between the seeked key and the previousKey
+ previousEntry = index.get(pos - 1);
// TODO: find the offset for this block
previousIndex = Integer.MIN_VALUE;
}
@@ -562,7 +562,6 @@ public class MultiLevelIndex {
pos++;
}
-
if (pos == index.size()) {
position.pop();
goToNext();
@@ -581,7 +580,6 @@ public class MultiLevelIndex {
}
private void goToNext() throws IOException {
- int numSkippedBlocks = 0;
// traverse the index tree forwards
while (position.isEmpty() == false) {
StackEntry top = position.peek();
@@ -590,7 +588,6 @@ public class MultiLevelIndex {
while (top.offset < index.size()) {
if (checkFilterIndexEntry(index.get(top.offset)))
break;
- numSkippedBlocks++;
top.offset++;
}
if (top.offset == index.size()) {
@@ -651,8 +648,6 @@ public class MultiLevelIndex {
return nextEntry;
}
- private int blocksReturned = 0;
-
public IndexEntry next() {
prepNext();
previousEntry = nextEntry;
@@ -697,7 +692,7 @@ public class MultiLevelIndex {
}
IndexIterator lookup(Key key) throws IOException {
- return new IndexIterator(timestampRange, key);
+ return new IndexIterator(timestampRange, columnVisibilityPredicate, key);
}
public void readFields(DataInput in) throws IOException {
@@ -751,7 +746,7 @@ public class MultiLevelIndex {
return rootBlock.getIndex().get(rootBlock.getIndex().size() -
1).getKey();
}
- TimestampRangePredicate timestampRange;
+ TimestampRangePredicate timestampRange = null;
/**
* @param r
@@ -759,6 +754,12 @@ public class MultiLevelIndex {
public void setTimestampRange(TimestampRangePredicate r) {
this.timestampRange = r;
}
+
+ ColumnVisibilityPredicate columnVisibilityPredicate = null;
+
+ public void setColumnVisibilityPredicate(ColumnVisibilityPredicate
columnVisibilityPredicate) {
+ this.columnVisibilityPredicate = columnVisibilityPredicate;
+ }
}
}
Modified:
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
(original)
+++
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Thu Jun 28 15:30:29 2012
@@ -61,6 +61,7 @@ import org.apache.accumulo.core.iterator
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.Predicate;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.predicates.ColumnVisibilityPredicate;
import org.apache.accumulo.core.iterators.predicates.TimestampRangePredicate;
import org.apache.accumulo.core.iterators.system.HeapIterator;
import org.apache.accumulo.core.security.ColumnVisibility;
@@ -776,6 +777,7 @@ public class RFile {
}
private TimestampRangePredicate timestampRange;
+ private ColumnVisibilityPredicate columnVisibilityPredicate;
private boolean filterChanged = false;
/* (non-Javadoc)
@@ -795,6 +797,12 @@ public class RFile {
timestampRange = p;
index.setTimestampRange(timestampRange);
}
+ else if(filter instanceof ColumnVisibilityPredicate)
+ {
+ filterChanged = true;
+ columnVisibilityPredicate = (ColumnVisibilityPredicate)filter;
+ index.setColumnVisibilityPredicate(columnVisibilityPredicate);
+ }
else
{
throw new RuntimeException("yikes, not yet implemented");
Modified:
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
(original)
+++
accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
Thu Jun 28 15:30:29 2012
@@ -51,4 +51,8 @@ public class TimestampRangePredicate imp
return timestamp >= startTimestamp && timestamp <= endTimestamp;
}
+ @Override
+ public String toString() {
+ return "{"+startTimestamp+"-"+endTimestamp+"}";
+ }
}
Modified:
accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
---
accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
(original)
+++
accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
Thu Jun 28 15:30:29 2012
@@ -46,7 +46,7 @@ public class TimestampFilterTest {
@Test
public void testRFileTimestampFiltering() throws Exception {
// TODO create an RFile with increasing timestamp and random key order
- Predicate<Key,Value> timeRange = new TimestampRangePredicate(100, 110);
+ Predicate<Key,Value> timeRange = new TimestampRangePredicate(73, 117);
int expected = 0;
Random r = new Random();
Configuration conf = new Configuration();
