ignite-950: reworked footer, added metadata
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/02faa773 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/02faa773 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/02faa773 Branch: refs/heads/ignite-950 Commit: 02faa773d824a9ba1453e4ee4852182066a5e30e Parents: d2e247a Author: Denis Magda <dma...@gridgain.com> Authored: Wed Jun 10 16:29:04 2015 +0300 Committer: Denis Magda <dma...@gridgain.com> Committed: Wed Jun 10 16:29:04 2015 +0300 ---------------------------------------------------------------------- .../processors/cache/CacheObjectContext.java | 15 +++- .../IgniteCacheObjectProcessorImpl.java | 2 + .../optimized/OptimizedClassDescriptor.java | 4 +- .../optimized/OptimizedMarshallerUtils.java | 14 +-- .../optimized/OptimizedObjectInputStream.java | 91 ++++++++++++++------ .../optimized/OptimizedObjectMetadata.java | 52 +++++++++-- .../optimized/OptimizedObjectOutputStream.java | 10 +-- .../optimized/OptimizedMarshallerSelfTest.java | 8 +- 8 files changed, 138 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java index e85a04a..cf35177 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectContext.java @@ -45,21 +45,27 @@ public class CacheObjectContext { /** */ private boolean p2pEnabled; + /** */ + private boolean systemCache; + /** * @param kernalCtx Kernal context. * @param dfltAffMapper Default affinity mapper. * @param cpyOnGet Copy on get flag. * @param storeVal {@code True} if should store unmarshalled value in cache. + * @param systemCache Whether the object will be stored in a system cache or not. */ public CacheObjectContext(GridKernalContext kernalCtx, AffinityKeyMapper dfltAffMapper, boolean cpyOnGet, - boolean storeVal) { + boolean storeVal, + boolean systemCache) { this.kernalCtx = kernalCtx; this.p2pEnabled = kernalCtx.config().isPeerClassLoadingEnabled(); this.dfltAffMapper = dfltAffMapper; this.cpyOnGet = cpyOnGet; this.storeVal = storeVal; + this.systemCache = systemCache; proc = kernalCtx.cacheObjects(); } @@ -86,6 +92,13 @@ public class CacheObjectContext { } /** + * @return {@code true} if the object is stored in a system cache. + */ + public boolean systemCache() { + return systemCache; + } + + /** * @return Default affinity mapper. */ public AffinityKeyMapper defaultAffMapper() { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java index bf96dd8..2b9e69b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.cacheobject; +import com.sun.javaws.*; import org.apache.ignite.*; import org.apache.ignite.cache.*; import org.apache.ignite.configuration.*; @@ -262,6 +263,7 @@ public class IgniteCacheObjectProcessorImpl extends GridProcessorAdapter impleme return new CacheObjectContext(ctx, ccfg.getAffinityMapper() != null ? ccfg.getAffinityMapper() : new GridCacheDefaultAffinityKeyMapper(), ccfg.isCopyOnRead() && memMode == ONHEAP_TIERED, + GridCacheUtils.isSystemCache(ccfg.getName()), storeVal); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java index 29709fa..ba3dfcf 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedClassDescriptor.java @@ -134,7 +134,7 @@ class OptimizedClassDescriptor { this.clsMap = clsMap; this.ctx = ctx; this.mapper = mapper; - this.metaHandler = this.metaHandler; + this.metaHandler = metaHandler; name = cls.getName(); @@ -492,7 +492,7 @@ class OptimizedClassDescriptor { for (ClassFields clsFields : this.fields.fields) for (FieldInfo info : clsFields.fields) - meta.addMeta(info.id(), (byte)info.type().ordinal()); + meta.addMeta(info.id(), info.type()); U.debug("putting to cache: " + typeId); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java index 60141ee..bf6a4fa 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerUtils.java @@ -149,6 +149,9 @@ class OptimizedMarshallerUtils { /** */ static final byte FOOTER_LEN_OFF = 4; + /** */ + static final byte VARIABLE_LEN = -1; + /** UTF-8 character name. */ static final Charset UTF_8 = Charset.forName("UTF-8"); @@ -221,7 +224,7 @@ class OptimizedMarshallerUtils { * @param mapper Mapper. * @return Type ID. */ - private static int resolveTypeId(String clsName, OptimizedMarshallerIdMapper mapper) { + public static int resolveTypeId(String clsName, OptimizedMarshallerIdMapper mapper) { int typeId; if (mapper != null) { @@ -237,15 +240,6 @@ class OptimizedMarshallerUtils { } /** - * Checks whether a given type supports and should include footer with fields info into the serialized form. - * - * @param type Type. - * @return {@code true} if supported, {@code false} otherwise. - */ - public static boolean supportsFooter(int type) { - return type == SERIALIZABLE; - } - /** * @param fieldName Field name. * @return Field ID. */ http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java index 34fd539..846fb08 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectInputStream.java @@ -965,14 +965,16 @@ class OptimizedObjectInputStream extends ObjectInputStream { int pos = in.position(); // TODO: IGNITE-950, do we need move to start position? - if (!supportsFooter(in.readByte())) + if (in.readByte() != SERIALIZABLE) { + in.position(pos); return false; + } - int off = fieldOffset(fieldName); + FieldRange range = fieldRange(fieldName); in.position(pos); - return off > 0; + return range != null && range.start > 0; } /** @@ -986,15 +988,17 @@ class OptimizedObjectInputStream extends ObjectInputStream { int pos = in.position(); // TODO: IGNITE-950, do we need move to start position? - if (!supportsFooter(in.readByte())) + if (in.readByte() != SERIALIZABLE) { + in.position(pos); return false; + } - int off = fieldOffset(fieldName); + FieldRange range = fieldRange(fieldName); Object obj = null; - if (off > 0) { - in.position(off); + if (range != null && range.start > 0) { + in.position(range.start); obj = readObject(); } @@ -1007,12 +1011,30 @@ class OptimizedObjectInputStream extends ObjectInputStream { * Returns field offset in the byte stream. * * @param fieldName Field name. - * @return positive offset or -1 if the object doesn't have such a field. + * @return positive range or {@code null} if the object doesn't have such a field. * @throws IOException in case of error. */ - private int fieldOffset(String fieldName) throws IOException { + private FieldRange fieldRange(String fieldName) throws IOException { int fieldId = resolveFieldId(fieldName); + int typeId = readInt(); + + int clsNameLen = 0; + + if (typeId == 0) { + int pos = in.position(); + + typeId = OptimizedMarshallerUtils.resolveTypeId(readUTF(), mapper); + + clsNameLen = in.position() - pos; + } + + OptimizedObjectMetadata meta = metaHandler.metadata(typeId); + + if (meta == null) + // TODO: IGNITE-950 add warning! + return null; + int end = in.size(); in.position(end - FOOTER_LEN_OFF); @@ -1020,33 +1042,28 @@ class OptimizedObjectInputStream extends ObjectInputStream { int footerLen = in.readInt(); if (footerLen == EMPTY_FOOTER) - return -1; - - int footerStartOff = end - footerLen; - in.position(footerStartOff); + return null; - int fieldsDataPos = in.readInt(); + // 4 - skipping length at the beginning + int footerOff = (end - footerLen) + 4; + in.position(footerOff); - int fieldOff = -1; + int fieldOff = 0; - // 8 - size of footer len and object total len fields. - int bound = end - 8; + for (OptimizedObjectMetadata.FieldInfo info : meta.getMeta()) { + if (info.id == fieldId) { + //object header len: 1 - for type, 4 - for type ID, 2 - for checksum. + fieldOff += 1 + 4 + clsNameLen + 2; - while (footerStartOff < bound) { - int id = in.readInt(); + FieldRange range = new FieldRange(fieldOff, info.len == VARIABLE_LEN ? in.readShort() : info.len); - if (fieldId == id) { - fieldOff = fieldsDataPos + in.readInt(); - break; + return range; } else - // skip offset and len - in.skipBytes(8); - - footerStartOff += 12; + fieldOff += info.len == VARIABLE_LEN ? in.readShort() : info.len; } - return fieldOff; + return null; } /** @@ -1294,4 +1311,24 @@ class OptimizedObjectInputStream extends ObjectInputStream { return objs[fieldInfo.getIndex(name)] != null ? (T)objs[fieldInfo.getIndex(name)] : dflt; } } + + /** + * + */ + private static class FieldRange { + /** */ + private int start; + + /** */ + private int len; + + /** + * @param start Start. + * @param len Length. + */ + public FieldRange(int start, int len) { + this.start = start; + this.len = len; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectMetadata.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectMetadata.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectMetadata.java index e156c4c..5b476da 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectMetadata.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectMetadata.java @@ -17,9 +17,12 @@ package org.apache.ignite.marshaller.optimized; +import org.apache.ignite.*; + import java.io.*; import java.util.*; +import static org.apache.ignite.marshaller.optimized.OptimizedFieldType.*; /** * Metadata that keeps fields information. Used in conjunction with the footer that is added to some objects during * marshalling. @@ -39,11 +42,44 @@ public class OptimizedObjectMetadata implements Externalizable { * @param fieldId Field ID. * @param fieldType Field type. */ - public void addMeta(int fieldId, byte fieldType) { + public void addMeta(int fieldId, OptimizedFieldType fieldType) { if (fieldsInfo == null) fieldsInfo = new ArrayList<>(); - fieldsInfo.add(new FieldInfo(fieldId, fieldType)); + int len = 1; + + switch (fieldType) { + case BYTE: + case BOOLEAN: + len += 1; + break; + + case SHORT: + case CHAR: + len += 2; + break; + + case INT: + case FLOAT: + len += 4; + break; + + case LONG: + case DOUBLE: + len += 8; + break; + + case OTHER: + len = OptimizedMarshallerUtils.VARIABLE_LEN; + break; + + default: + throw new IgniteException("Unknown field type: " + fieldType); + } + + assert len != 1; + + fieldsInfo.add(new FieldInfo(fieldId, len)); } /** @@ -75,7 +111,7 @@ public class OptimizedObjectMetadata implements Externalizable { for (FieldInfo fieldInfo : fieldsInfo) { out.writeInt(fieldInfo.id); - out.writeByte(fieldInfo.type); + out.writeInt(fieldInfo.len); } } @@ -86,7 +122,7 @@ public class OptimizedObjectMetadata implements Externalizable { fieldsInfo = new ArrayList<>(size); for (int i = 0; i < size; i++) - fieldsInfo.add(new FieldInfo(in.readInt(), in.readByte())); + fieldsInfo.add(new FieldInfo(in.readInt(), in.readInt())); } /** @@ -97,17 +133,17 @@ public class OptimizedObjectMetadata implements Externalizable { int id; /** Field type. */ - byte type; + int len; /** * Constructor. * * @param id Field ID. - * @param type Field type. + * @param len Field len. */ - public FieldInfo(int id, byte type) { + public FieldInfo(int id, int len) { this.id = id; - this.type = type; + this.len = len; } } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java index 288ddef..2ead955 100644 --- a/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java +++ b/modules/core/src/main/java/org/apache/ignite/marshaller/optimized/OptimizedObjectOutputStream.java @@ -561,7 +561,7 @@ class OptimizedObjectOutputStream extends ObjectOutputStream { if (t.field() != null) { int fieldLen = out.size() - size; - footer.put(t.id(), (byte)t.type().ordinal(), fieldLen); + footer.put(t.id(), t.type(), fieldLen); } } } @@ -819,7 +819,7 @@ class OptimizedObjectOutputStream extends ObjectOutputStream { int fieldLen = out.size() - size; - footer.put(t.get1().id(), (byte)t.get1().type().ordinal(), fieldLen); + footer.put(t.get1().id(), t.get1().type(), fieldLen); } } @@ -986,12 +986,12 @@ class OptimizedObjectOutputStream extends ObjectOutputStream { * @param fieldType Field type. * @param len Total number of bytes occupied by type's value. */ - private void put(int fieldId, byte fieldType, int len) { + private void put(int fieldId, OptimizedFieldType fieldType, int len) { if (data == null) return; // Considering that field's length will be no longer 2^15 (32 MB) - if (fieldType == OptimizedFieldType.OTHER.ordinal()) + if (fieldType == OptimizedFieldType.OTHER) data.add((short)len); } @@ -1011,8 +1011,6 @@ class OptimizedObjectOutputStream extends ObjectOutputStream { if (data == null) writeInt(EMPTY_FOOTER); else { - int footerStartPos = out.size(); - //12 - 4 bytes for len at the beginning, 4 bytes for len at the end, 4 bytes for object len. int footerLen = data.size() * 2 + 12; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/02faa773/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java index d890331..6c78506 100644 --- a/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/marshaller/optimized/OptimizedMarshallerSelfTest.java @@ -114,13 +114,13 @@ public class OptimizedMarshallerSelfTest extends GridMarshallerAbstractTest { * @throws Exception If failed. */ public void testFieldUnmarshalling() throws Exception { - //TestObject2 obj = new TestObject2(5); + TestObject2 obj = new TestObject2(5); - //byte[] data = marshal(obj); + byte[] data = marshal(obj); - //Integer i = ((OptimizedMarshaller)marsh).unmarshal("i", data, Thread.currentThread().getContextClassLoader()); + Integer i = ((OptimizedMarshaller)marsh).unmarshal("i", data, Thread.currentThread().getContextClassLoader()); - //assertEquals(obj.i, (int)i); + assertEquals(obj.i, (int)i); } /**
