ignite-1258: open sourced portables implementation
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/1f2be19d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/1f2be19d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/1f2be19d Branch: refs/heads/ignite-1258 Commit: 1f2be19d4cafd94687500618f16c85c873210238 Parents: fcd9d61 Author: Denis Magda <dma...@gridgain.com> Authored: Tue Aug 18 12:31:26 2015 +0300 Committer: Denis Magda <dma...@gridgain.com> Committed: Tue Aug 18 12:31:26 2015 +0300 ---------------------------------------------------------------------- .../java/org/apache/ignite/IgnitePortables.java | 364 +++ .../ignite/internal/GridKernalContextImpl.java | 3 +- .../apache/ignite/internal/IgniteKernal.java | 3 +- .../portable/GridGridPortableReaderImpl.java | 2972 ++++++++++++++++++ .../portable/GridPortableAbstractLazyValue.java | 57 + .../portable/GridPortableBuilderEnum.java | 114 + .../portable/GridPortableBuilderImpl.java | 519 +++ .../portable/GridPortableBuilderReader.java | 775 +++++ .../GridPortableBuilderSerializationAware.java | 29 + .../portable/GridPortableBuilderSerializer.java | 210 ++ .../portable/GridPortableClassDescriptor.java | 1344 ++++++++ .../internal/portable/GridPortableContext.java | 1089 +++++++ .../GridPortableEnumArrayLazyValue.java | 111 + .../portable/GridPortableLazyArrayList.java | 156 + .../portable/GridPortableLazyLinkedList.java | 210 ++ .../internal/portable/GridPortableLazyMap.java | 214 ++ .../portable/GridPortableLazyMapEntry.java | 66 + .../internal/portable/GridPortableLazySet.java | 89 + .../portable/GridPortableLazyValue.java | 28 + .../portable/GridPortableMarshaller.java | 304 ++ .../portable/GridPortableMetaDataCollector.java | 253 ++ .../portable/GridPortableMetaDataHandler.java | 43 + .../portable/GridPortableMetaDataImpl.java | 140 + .../GridPortableObjectArrayLazyValue.java | 89 + .../internal/portable/GridPortableObjectEx.java | 213 ++ .../portable/GridPortableObjectImpl.java | 383 +++ .../portable/GridPortableObjectOffheapImpl.java | 238 ++ .../portable/GridPortablePlainLazyValue.java | 47 + .../GridPortablePlainPortableObject.java | 50 + .../portable/GridPortablePrimitives.java | 773 +++++ .../portable/GridPortableRawReader.java | 33 + .../portable/GridPortableRawWriter.java | 44 + .../portable/GridPortableReaderContext.java | 83 + .../GridPortableThreadLocalMemoryAllocator.java | 163 + .../internal/portable/GridPortableUtils.java | 380 +++ .../portable/GridPortableValueWithType.java | 74 + .../portable/GridPortableWriterImpl.java | 1770 +++++++++++ .../ignite/internal/portable/package-info.java | 22 + .../GridPortableAbstractInputStream.java | 344 ++ .../GridPortableAbstractOutputStream.java | 325 ++ .../streams/GridPortableAbstractStream.java | 83 + .../streams/GridPortableHeapInputStream.java | 134 + .../streams/GridPortableHeapOutputStream.java | 155 + .../streams/GridPortableMemoryAllocator.java | 76 + .../streams/GridPortableOffheapInputStream.java | 129 + .../GridPortableOffheapOutputStream.java | 169 + .../GridPortableSimpleMemoryAllocator.java | 67 + .../internal/portable/streams/package-info.java | 22 + .../CacheDefaultPortableAffinityKeyMapper.java | 51 + .../portable/CacheObjectPortableContext.java | 164 + .../portable/CacheObjectPortableProcessor.java | 101 + .../CacheObjectPortableProcessorImpl.java | 949 ++++++ .../cache/portable/IgnitePortablesImpl.java | 176 ++ .../cache/portable/PortableMetaDataKey.java | 80 + .../processors/cache/portable/package-info.java | 22 + .../marshaller/portable/PortableMarshaller.java | 348 ++ .../marshaller/portable/package-info.java | 22 + .../apache/ignite/portable/PortableBuilder.java | 137 + .../ignite/portable/PortableException.java | 58 + .../ignite/portable/PortableIdMapper.java | 47 + .../portable/PortableInvalidClassException.java | 50 + .../ignite/portable/PortableMarshalAware.java | 40 + .../ignite/portable/PortableMetadata.java | 53 + .../apache/ignite/portable/PortableObject.java | 160 + .../portable/PortableProtocolVersion.java | 32 + .../ignite/portable/PortableRawReader.java | 241 ++ .../ignite/portable/PortableRawWriter.java | 210 ++ .../apache/ignite/portable/PortableReader.java | 294 ++ .../ignite/portable/PortableSerializer.java | 39 + .../portable/PortableTypeConfiguration.java | 189 ++ .../apache/ignite/portable/PortableWriter.java | 257 ++ .../apache/ignite/portable/package-info.java | 22 + 72 files changed, 18699 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java b/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java new file mode 100644 index 0000000..efb6e6a --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/IgnitePortables.java @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite; + +import org.apache.ignite.internal.processors.cache.*; +import org.apache.ignite.portable.*; +import org.apache.ignite.marshaller.portable.*; + +import org.jetbrains.annotations.*; + +import java.sql.*; +import java.util.*; +import java.util.Date; + +/** + * Defines portable objects functionality. With portable objects you are able to: + * <ul> + * <li>Seamlessly interoperate between Java, .NET, and C++.</li> + * <li>Make any object portable with zero code change to your existing code.</li> + * <li>Nest portable objects within each other.</li> + * <li>Automatically handle {@code circular} or {@code null} references.</li> + * <li>Automatically convert collections and maps between Java, .NET, and C++.</li> + * <li> + * Optionally avoid deserialization of objects on the server side + * (objects are stored in {@link PortableObject} format). + * </li> + * <li>Avoid need to have concrete class definitions on the server side.</li> + * <li>Dynamically change structure of the classes without having to restart the cluster.</li> + * <li>Index into portable objects for querying purposes.</li> + * </ul> + * <h1 class="header">Working With Portables Directly</h1> + * Once an object is defined as portable, + * Ignite will always store it in memory in the portable (i.e. binary) format. + * User can choose to work either with the portable format or with the deserialized form + * (assuming that class definitions are present in the classpath). + * <p> + * To work with the portable format directly, user should create a special cache projection + * using {@link IgniteInternalCache#keepPortable()} method and then retrieve individual fields as needed: + * <pre name=code class=java> + * CacheProjection<GridPortableObject.class, GridPortableObject.class> prj = cache.keepPortable(); + * + * // Convert instance of MyKey to portable format. + * // We could also use GridPortableBuilder to create + * // the key in portable format directly. + * GridPortableObject key = grid.portables().toPortable(new MyKey()); + * + * GridPortableObject val = prj.get(key); + * + * String field = val.field("myFieldName"); + * </pre> + * Alternatively, if we have class definitions in the classpath, we may choose to work with deserialized + * typed objects at all times. In this case we do incur the deserialization cost, however, + * Ignite will only deserialize on the first access and will cache the deserialized object, + * so it does not have to be deserialized again: + * <pre name=code class=java> + * CacheProjection<MyKey.class, MyValue.class> prj = + * cache.projection(MyKey.class, MyValue.class); + * + * MyValue val = prj.get(new MyKey()); + * + * // Normal java getter. + * String fieldVal = val.getMyFieldName(); + * </pre> + * If we used, for example, one of the automatically handled portable types for a key, like integer, + * and still wanted to work with binary portable format for values, then we would declare cache projection + * as follows: + * <pre name=code class=java> + * CacheProjection<Integer.class, GridPortableObject.class> prj = cache.keepPortable(); + * </pre> + * <h1 class="header">Automatic Portable Types</h1> + * Note that only portable classes are converted to {@link PortableObject} format. Following + * classes are never converted (e.g., {@link #toPortable(Object)} method will return original + * object, and instances of these classes will be stored in cache without changes): + * <ul> + * <li>All primitives (byte, int, ...) and there boxed versions (Byte, Integer, ...)</li> + * <li>Arrays of primitives (byte[], int[], ...)</li> + * <li>{@link String} and array of {@link String}s</li> + * <li>{@link UUID} and array of {@link UUID}s</li> + * <li>{@link Date} and array of {@link Date}s</li> + * <li>{@link Timestamp} and array of {@link Timestamp}s</li> + * <li>Enums and array of enums</li> + * <li> + * Maps, collections and array of objects (but objects inside + * them will still be converted if they are portable) + * </li> + * </ul> + * <h1 class="header">Working With Maps and Collections</h1> + * All maps and collections in the portable objects are serialized automatically. When working + * with different platforms, e.g. C++ or .NET, Ignite will automatically pick the most + * adequate collection or map in either language. For example, {@link ArrayList} in Java will become + * {@code List} in C#, {@link LinkedList} in Java is {@link LinkedList} in C#, {@link HashMap} + * in Java is {@code Dictionary} in C#, and {@link TreeMap} in Java becomes {@code SortedDictionary} + * in C#, etc. + * <h1 class="header">Building Portable Objects</h1> + * Ignite comes with {@link PortableBuilder} which allows to build portable objects dynamically: + * <pre name=code class=java> + * GridPortableBuilder builder = Ignition.ignite().portables().builder(); + * + * builder.typeId("MyObject"); + * + * builder.stringField("fieldA", "A"); + * build.intField("fieldB", "B"); + * + * GridPortableObject portableObj = builder.build(); + * </pre> + * For the cases when class definition is present + * in the class path, it is also possible to populate a standard POJO and then + * convert it to portable format, like so: + * <pre name=code class=java> + * MyObject obj = new MyObject(); + * + * obj.setFieldA("A"); + * obj.setFieldB(123); + * + * GridPortableObject portableObj = Ignition.ignite().portables().toPortable(obj); + * </pre> + * NOTE: you don't need to convert typed objects to portable format before storing + * them in cache, Ignite will do that automatically. + * <h1 class="header">Portable Metadata</h1> + * Even though Ignite portable protocol only works with hash codes for type and field names + * to achieve better performance, Ignite provides metadata for all portable types which + * can be queried ar runtime via any of the {@link IgnitePortables#metadata(Class) GridPortables.metadata(...)} + * methods. Having metadata also allows for proper formatting of {@code GridPortableObject.toString()} method, + * even when portable objects are kept in binary format only, which may be necessary for audit reasons. + * <h1 class="header">Dynamic Structure Changes</h1> + * Since objects are always cached in the portable binary format, server does not need to + * be aware of the class definitions. Moreover, if class definitions are not present or not + * used on the server, then clients can continuously change the structure of the portable + * objects without having to restart the cluster. For example, if one client stores a + * certain class with fields A and B, and another client stores the same class with + * fields B and C, then the server-side portable object will have the fields A, B, and C. + * As the structure of a portable object changes, the new fields become available for SQL queries + * automatically. + * <h1 class="header">Configuration</h1> + * By default all your objects are considered as portables and no specific configuration is needed. + * However, in some cases, like when an object is used by both Java and .Net, you may need to specify portable objects + * explicitly by calling {@link PortableMarshaller#setClassNames(Collection)}. + * The only requirement Ignite imposes is that your object has an empty + * constructor. Note, that since server side does not have to know the class definition, + * you only need to list portable objects in configuration on the client side. However, if you + * list them on the server side as well, then you get the ability to deserialize portable objects + * into concrete types on the server as well as on the client. + * <p> + * Here is an example of portable configuration (note that star (*) notation is supported): + * <pre name=code class=xml> + * ... + * <!-- Explicit portable objects configuration. --> + * <property name="marshaller"> + * <bean class="org.gridgain.grid.marshaller.portable.PortableMarshaller"> + * <property name="classNames"> + * <list> + * <value>my.package.for.portable.objects.*</value> + * <value>org.apache.ignite.examples.client.portable.Employee</value> + * </list> + * </property> + * </bean> + * </property> + * ... + * </pre> + * or from code: + * <pre name=code class=java> + * IgniteConfiguration cfg = new IgniteConfiguration(); + * + * PortableMarshaller marsh = new PortableMarshaller(); + * + * marsh.setClassNames(Arrays.asList( + * Employee.class.getName(), + * Address.class.getName()) + * ); + * + * cfg.setMarshaller(marsh); + * </pre> + * You can also specify class name for a portable object via {@link PortableTypeConfiguration}. + * Do it in case if you need to override other configuration properties on per-type level, like + * ID-mapper, or serializer. + * <h1 class="header">Custom Affinity Keys</h1> + * Often you need to specify an alternate key (not the cache key) for affinity routing whenever + * storing objects in cache. For example, if you are caching {@code Employee} object with + * {@code Organization}, and want to colocate employees with organization they work for, + * so you can process them together, you need to specify an alternate affinity key. + * With portable objects you would have to do it as following: + * <pre name=code class=xml> + * <property name="marshaller"> + * <bean class="org.gridgain.grid.marshaller.portable.PortableMarshaller"> + * ... + * <property name="typeConfigurations"> + * <list> + * <bean class="org.apache.ignite.portables.PortableTypeConfiguration"> + * <property name="className" value="org.apache.ignite.examples.client.portable.EmployeeKey"/> + * <property name="affinityKeyFieldName" value="organizationId"/> + * </bean> + * </list> + * </property> + * ... + * </bean> + * </property> + * </pre> + * <h1 class="header">Serialization</h1> + * Serialization and deserialization works out-of-the-box in Ignite. However, you can provide your own custom + * serialization logic by optionally implementing {@link PortableMarshalAware} interface, like so: + * <pre name=code class=java> + * public class Address implements GridPortableMarshalAware { + * private String street; + * private int zip; + * + * // Empty constructor required for portable deserialization. + * public Address() {} + * + * @Override public void writePortable(GridPortableWriter writer) throws GridPortableException { + * writer.writeString("street", street); + * writer.writeInt("zip", zip); + * } + * + * @Override public void readPortable(GridPortableReader reader) throws GridPortableException { + * street = reader.readString("street"); + * zip = reader.readInt("zip"); + * } + * } + * </pre> + * Alternatively, if you cannot change class definitions, you can provide custom serialization + * logic in {@link PortableSerializer} either globally in {@link PortableMarshaller} or + * for a specific type via {@link PortableTypeConfiguration} instance. + * <p> + * Similar to java serialization you can use {@code writeReplace()} and {@code readResolve()} methods. + * <ul> + * <li> + * {@code readResolve} is defined as follows: {@code ANY-ACCESS-MODIFIER Object readResolve()}. + * It may be used to replace the de-serialized object by another one of your choice. + * </li> + * <li> + * {@code writeReplace} is defined as follows: {@code ANY-ACCESS-MODIFIER Object writeReplace()}. This method + * allows the developer to provide a replacement object that will be serialized instead of the original one. + * </li> + * </ul> + * + * <h1 class="header">Custom ID Mappers</h1> + * Ignite implementation uses name hash codes to generate IDs for class names or field names + * internally. However, in cases when you want to provide your own ID mapping schema, + * you can provide your own {@link PortableIdMapper} implementation. + * <p> + * ID-mapper may be provided either globally in {@link PortableMarshaller}, + * or for a specific type via {@link PortableTypeConfiguration} instance. + * <h1 class="header">Query Indexing</h1> + * Portable objects can be indexed for querying by specifying index fields in + * {@link org.apache.ignite.cache.CacheTypeMetadata} inside of specific + * {@link org.apache.ignite.configuration.CacheConfiguration} instance, + * like so: + * <pre name=code class=xml> + * ... + * <bean class="org.apache.ignite.cache.CacheConfiguration"> + * ... + * <property name="typeMetadata"> + * <list> + * <bean class="CacheTypeMetadata"> + * <property name="type" value="Employee"/> + * + * <!-- Fields to index in ascending order. --> + * <property name="ascendingFields"> + * <map> + * <entry key="name" value="java.lang.String"/> + * + * <!-- Nested portable objects can also be indexed. --> + * <entry key="address.zip" value="java.lang.Integer"/> + * </map> + * </property> + * </bean> + * </list> + * </property> + * </bean> + * </pre> + */ +public interface IgnitePortables { + /** + * Gets type ID for given type name. + * + * @param typeName Type name. + * @return Type ID. + */ + public int typeId(String typeName); + + /** + * Converts provided object to instance of {@link PortableObject}. + * + * @param obj Object to convert. + * @return Converted object. + * @throws PortableException In case of error. + */ + public <T> T toPortable(@Nullable Object obj) throws PortableException; + + /** + * Creates new portable builder. + * + * @param typeId ID of the type. + * @return Newly portable builder. + */ + public PortableBuilder builder(int typeId); + + /** + * Creates new portable builder. + * + * @param typeName Type name. + * @return Newly portable builder. + */ + public PortableBuilder builder(String typeName); + + /** + * Creates portable builder initialized by existing portable object. + * + * @param portableObj Portable object to initialize builder. + * @return Portable builder. + */ + public PortableBuilder builder(PortableObject portableObj); + + /** + * Gets metadata for provided class. + * + * @param cls Class. + * @return Metadata. + * @throws PortableException In case of error. + */ + @Nullable public PortableMetadata metadata(Class<?> cls) throws PortableException; + + /** + * Gets metadata for provided class name. + * + * @param typeName Type name. + * @return Metadata. + * @throws PortableException In case of error. + */ + @Nullable public PortableMetadata metadata(String typeName) throws PortableException; + + /** + * Gets metadata for provided type ID. + * + * @param typeId Type ID. + * @return Metadata. + * @throws PortableException In case of error. + */ + @Nullable public PortableMetadata metadata(int typeId) throws PortableException; + + /** + * Gets metadata for all known types. + * + * @return Metadata. + * @throws PortableException In case of error. + */ + public Collection<PortableMetadata> metadata() throws PortableException; +} + http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index fd8b50c..01dadfd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -32,6 +32,7 @@ import org.apache.ignite.internal.managers.loadbalancer.*; import org.apache.ignite.internal.managers.swapspace.*; import org.apache.ignite.internal.processors.affinity.*; import org.apache.ignite.internal.processors.cache.*; +import org.apache.ignite.internal.processors.cache.portable.*; import org.apache.ignite.internal.processors.cacheobject.*; import org.apache.ignite.internal.processors.clock.*; import org.apache.ignite.internal.processors.closure.*; @@ -803,7 +804,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable return res; if (cls.equals(IgniteCacheObjectProcessor.class)) - return (T)new IgniteCacheObjectProcessorImpl(this); + return (T)new CacheObjectPortableProcessorImpl(this); if (cls.equals(CacheConflictResolutionManager.class)) return null; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/1f2be19d/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java index bf47f63..e32d5e8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java @@ -36,6 +36,7 @@ import org.apache.ignite.internal.managers.swapspace.*; import org.apache.ignite.internal.processors.*; import org.apache.ignite.internal.processors.affinity.*; import org.apache.ignite.internal.processors.cache.*; +import org.apache.ignite.internal.processors.cache.portable.*; import org.apache.ignite.internal.processors.cacheobject.*; import org.apache.ignite.internal.processors.clock.*; import org.apache.ignite.internal.processors.closure.*; @@ -2960,7 +2961,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable { return comp; if (cls.equals(IgniteCacheObjectProcessor.class)) - return (T)new IgniteCacheObjectProcessorImpl(ctx); + return (T)new CacheObjectPortableProcessorImpl(ctx); if (cls.equals(DiscoveryNodeValidationProcessor.class)) return (T)new OsDiscoveryNodeValidationProcessor(ctx);
