IGNITE-794 WIP on memory estimation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/5bcac1c7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/5bcac1c7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/5bcac1c7 Branch: refs/heads/ignite-794 Commit: 5bcac1c73f5528f10fffbf27353dce464df1568e Parents: 779f341 Author: AKuznetsov <akuznet...@gridgain.com> Authored: Mon Apr 27 17:46:50 2015 +0700 Committer: AKuznetsov <akuznet...@gridgain.com> Committed: Mon Apr 27 17:46:50 2015 +0700 ---------------------------------------------------------------------- modules/core/pom.xml | 6 - .../ignite/internal/visor/cache/VisorCache.java | 81 ++++++--- .../processors/query/h2/IgniteH2Indexing.java | 9 +- modules/memory/pom.xml | 75 +++++++++ .../apache/ignite/memory/MemoryEstimator.java | 168 +++++++++++++++++++ pom.xml | 1 + 6 files changed, 310 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/core/pom.xml ---------------------------------------------------------------------- diff --git a/modules/core/pom.xml b/modules/core/pom.xml index b374edf..62612f8 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -41,12 +41,6 @@ </dependency> <dependency> - <groupId>org.ehcache</groupId> - <artifactId>sizeof</artifactId> - <version>0.3.0-SNAPSHOT</version> - </dependency> - - <dependency> <groupId>mx4j</groupId> <artifactId>mx4j-tools</artifactId> <version>3.0.1</version> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java index 1ba003f..a2f24a7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java @@ -27,14 +27,13 @@ import org.apache.ignite.internal.processors.cache.*; import org.apache.ignite.internal.processors.cache.distributed.dht.*; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.*; import org.apache.ignite.internal.processors.cache.distributed.near.*; -import org.apache.ignite.internal.processors.query.*; import org.apache.ignite.internal.util.lang.*; import org.apache.ignite.internal.util.typedef.internal.*; import org.apache.ignite.lang.*; -import org.ehcache.sizeof.*; import org.jetbrains.annotations.*; import java.io.*; +import java.lang.reflect.*; import java.util.*; import java.util.concurrent.*; @@ -45,8 +44,17 @@ public class VisorCache implements Serializable { /** */ private static final long serialVersionUID = 0L; - /** Default cache size sampling. */ - private static final int DFLT_CACHE_SIZE_SAMPLING = 10; + /** */ + private static final String MEM_ESTIMATOR = "MemoryEstimator"; + + /** */ + private static final String MEM_ESTIMATOR_INSTALLED = "MemoryEstimatorInstalled"; + + /** */ + private static final String MEM_ESTIMATOR_CACHE_SZ_MTHD = "MemoryEstimatorCacheSizeMethod"; + + /** */ + private static final String MEM_ESTIMATOR_INDEXES_SZ_MTHD = "MemoryEstimatorIndexesSizeMethod"; /** Cache name. */ private String name; @@ -60,6 +68,9 @@ public class VisorCache implements Serializable { /** Cache size in bytes. */ private long memorySize; + /** Cache size in bytes. */ + private long indexesSize; + /** Number of all entries in cache. */ private int size; @@ -196,7 +207,6 @@ public class VisorCache implements Serializable { nearSize = ca.nearSize(); dynamicDeploymentId = ca.context().dynamicDeploymentId(); - memorySize = estimateMemorySize(ignite, ca, sample); dhtSize = size - nearSize; primarySize = ca.primarySize(); offHeapAllocatedSize = ca.offHeapAllocatedSize(); @@ -204,6 +214,8 @@ public class VisorCache implements Serializable { partitions = ca.affinity().partitions(); metrics = VisorCacheMetrics.from(ignite, ca); + estimateMemorySize(ignite, ca, sample); + return this; } @@ -216,34 +228,51 @@ public class VisorCache implements Serializable { * @return Estimation of memory size occupied by cache. * @throws IgniteCheckedException If estimation failed. */ - protected long estimateMemorySize(IgniteEx ignite, GridCacheAdapter ca, int sample) throws IgniteCheckedException { - ConcurrentMap<String, SizeOf> storage = ignite.cluster().nodeLocalMap(); + protected void estimateMemorySize(IgniteEx ignite, GridCacheAdapter ca, int sample) throws IgniteCheckedException { + ConcurrentMap<String, Object> storage = ignite.cluster().nodeLocalMap(); - SizeOf sizeOf = storage.get("SizeOf"); + Boolean installed = (Boolean)storage.get(MEM_ESTIMATOR_INSTALLED); - if (sizeOf == null) - sizeOf = SizeOf.newInstance(); + if (installed == null) { + try { + Class<?> cls = Class.forName("org.apache.ignite.memory.MemoryEstimator"); - int cnt = Math.min(sample > 0 ? sample : DFLT_CACHE_SIZE_SAMPLING, ca.size()); + Object estimator = cls.newInstance(); - long memSz = 0; + Method mtdCacheSz = cls.getMethod("estimateCacheSize", GridCacheAdapter.class, int.class); + Method mtdIndexesSz = cls.getMethod("estimateIndexesSize", IgniteEx.class, GridCacheAdapter.class); - if (cnt > 0) { - for (int i = 0; i < cnt; i++) { - GridCacheMapEntry entry = ca.randomInternalEntry(); + memorySize = (Long)mtdCacheSz.invoke(estimator, ca, sample); + indexesSize = (Long)mtdIndexesSz.invoke(estimator, ignite, ca); - long ksz = sizeOf.deepSizeOf(Integer.MAX_VALUE, false, entry.key()).getCalculated(); - long vsz = sizeOf.deepSizeOf(Integer.MAX_VALUE, false, entry.rawGet()).getCalculated(); - - memSz += (ksz + vsz); + storage.putIfAbsent(MEM_ESTIMATOR, estimator); + storage.putIfAbsent(MEM_ESTIMATOR_CACHE_SZ_MTHD, mtdCacheSz); + storage.putIfAbsent(MEM_ESTIMATOR_INDEXES_SZ_MTHD, mtdIndexesSz); + storage.putIfAbsent(MEM_ESTIMATOR_INSTALLED, Boolean.TRUE); } + catch (Exception e) { + storage.putIfAbsent(MEM_ESTIMATOR_INSTALLED, Boolean.FALSE); - memSz = (long)((double)memSz / cnt * size); + throw U.cast(e); + } } + else if (installed) { + Object estimator = storage.get(MEM_ESTIMATOR); + Method mtdCacheSz = (Method)storage.get(MEM_ESTIMATOR_CACHE_SZ_MTHD); + Method mtdIndexesSz = (Method)storage.get(MEM_ESTIMATOR_INDEXES_SZ_MTHD); + + assert estimator != null && mtdCacheSz != null && mtdIndexesSz != null; - GridQueryIndexing indexing = ignite.context().query().indexing(); + try { + memorySize = (Long)mtdCacheSz.invoke(estimator, ca, sample); + indexesSize = (Long)mtdIndexesSz.invoke(estimator, ignite, ca); + } + catch (Exception e) { + throw U.cast(e); + } + } - return memSz; + System.out.println("Cache: " + name + ", entries memory=" + memorySize + ", indexes memory=" + indexesSize); } /** @@ -255,6 +284,7 @@ public class VisorCache implements Serializable { c.name = name; c.mode = mode; c.memorySize = memorySize; + c.indexesSize = indexesSize; c.size = size; c.nearSize = nearSize; c.dhtSize = dhtSize; @@ -300,6 +330,13 @@ public class VisorCache implements Serializable { } /** + * @return Indexes size in bytes. + */ + public long indexesSize() { + return indexesSize; + } + + /** * @return Number of all entries in cache. */ public int size() { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 1bc3409..4fcbe10 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1044,7 +1044,7 @@ public class IgniteH2Indexing implements GridQueryIndexing { * @param schema Schema name. * @return Collection of table descriptors. */ - private Collection<TableDescriptor> tables(String schema) { + public Collection<TableDescriptor> tables(String schema) { Schema s = schemas.get(schema); if (s == null) @@ -1480,7 +1480,7 @@ public class IgniteH2Indexing implements GridQueryIndexing { /** * Information about table in database. */ - private class TableDescriptor implements GridH2Table.IndexesFactory { + public class TableDescriptor implements GridH2Table.IndexesFactory { /** */ private final String fullTblName; @@ -1541,6 +1541,11 @@ public class IgniteH2Indexing implements GridQueryIndexing { return S.toString(TableDescriptor.class, this); } + public GridH2Table h2Table() { + return tbl; + } + + /** {@inheritDoc} */ @Override public ArrayList<Index> createIndexes(GridH2Table tbl) { this.tbl = tbl; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/memory/pom.xml ---------------------------------------------------------------------- diff --git a/modules/memory/pom.xml b/modules/memory/pom.xml new file mode 100644 index 0000000..4d373a1 --- /dev/null +++ b/modules/memory/pom.xml @@ -0,0 +1,75 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + POM file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-parent</artifactId> + <version>1</version> + <relativePath>../../parent</relativePath> + </parent> + + <artifactId>ignite-memory</artifactId> + <version>1.0.3-SNAPSHOT</version> + + <dependencies> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-indexing</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.ehcache</groupId> + <artifactId>sizeof</artifactId> + <version>0.3.0-SNAPSHOT</version> + </dependency> + </dependencies> + + <build> + <resources> + <resource> + <directory>src/main/java</directory> + <excludes> + <exclude>**/*.java</exclude> + </excludes> + </resource> + </resources> + + <testResources> + <testResource> + <directory>src/test/java</directory> + <excludes> + <exclude>**/*.java</exclude> + </excludes> + </testResource> + </testResources> + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java ---------------------------------------------------------------------- diff --git a/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java b/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java new file mode 100644 index 0000000..80c68c3 --- /dev/null +++ b/modules/memory/src/main/java/org/apache/ignite/memory/MemoryEstimator.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.memory; + +import org.apache.ignite.internal.*; +import org.apache.ignite.internal.processors.cache.*; +import org.apache.ignite.internal.processors.query.*; +import org.apache.ignite.internal.processors.query.h2.*; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.*; +import org.apache.ignite.internal.processors.query.h2.opt.*; +import org.ehcache.sizeof.*; +import org.ehcache.sizeof.filters.*; +import org.h2.index.*; + +import java.lang.reflect.*; +import java.util.*; + +/** + * Utility class that will estimate memory occupied by cache. + */ +public class MemoryEstimator { + /** Default cache size sampling. */ + private static final int DFLT_CACHE_SIZE_SAMPLING = 10; + + /** */ + private final SizeOf sizeOfCache; + /** */ + private final SizeOf sizeOfIndex1; + + private final Set<String> ignored = new HashSet<>(); +// /** */ +// private final SizeOf sizeOfIndex2; + + /** + * Default constructor. + */ + public MemoryEstimator() { + ignored.add("org.h2.engine.Database"); + ignored.add("org.h2.message.TraceSystem"); + + SizeOfFilter filter1 = new SizeOfFilter() { + @Override public Collection<Field> filterFields(Class<?> klazz, Collection<Field> fields) { + return fields; + } + + @Override public boolean filterClass(Class<?> klazz) { + String klazzName = klazz.getName(); + + System.out.println("]]] " + klazzName); + + return true; + } + }; + + SizeOfFilter filter = new SizeOfFilter() { + @Override public Collection<Field> filterFields(Class<?> klazz, Collection<Field> fields) { + return fields; + } + + @Override public boolean filterClass(Class<?> klazz) { + String klazzName = klazz.getName(); + + boolean f = (!klazzName.startsWith("org.apache.ignite") || + klazzName.contains("h2") || klazzName.contains("H2") || + klazzName.contains("snaptree")) && + !ignored.contains(klazzName); + + System.out.println("))) " + klazzName + " " + f); + + return f; + } + }; + + sizeOfCache = SizeOf.newInstance(filter1); + sizeOfIndex1 = SizeOf.newInstance(filter); + } + + /** + * Estimate memory size for entries in cache. + * + * @param ca Cache to estimate. + * @param sample Number of entries to sample. + * @return Memory size in bytes under cache entries. + */ + public long estimateCacheSize(GridCacheAdapter ca, int sample) { + int size = ca.size(); + + int cnt = Math.min(sample > 0 ? sample : DFLT_CACHE_SIZE_SAMPLING, size); + + long memSz = 0; + + if (cnt > 0) { + for (int i = 0; i < cnt; i++) { + GridCacheMapEntry entry = ca.randomInternalEntry(); + + long ksz = sizeOfCache.deepSizeOf(Integer.MAX_VALUE, false, entry.key()).getCalculated(); + long vsz = sizeOfCache.deepSizeOf(Integer.MAX_VALUE, false, entry.rawGet()).getCalculated(); + + memSz += (ksz + vsz); + } + + memSz = (long)((double)memSz / cnt * size); + } + + return memSz; + } + + /** + * Estimate memory size for cache indexes. + * + * @param ignite Ignite. + * @param ca Cache to estimate. + * @return Memory size in bytes under indexes. + */ + public long estimateIndexesSize(IgniteEx ignite, GridCacheAdapter ca) { + long memSz = 0; + + GridQueryIndexing indexing = ignite.context().query().indexing(); + + if (indexing instanceof IgniteH2Indexing) { + IgniteH2Indexing h2Indexing = (IgniteH2Indexing)indexing; + + String cacheName = ca.name(); + + if (cacheName == null) + cacheName = ""; + + Collection<TableDescriptor> tbls = h2Indexing.tables(cacheName); + + for (TableDescriptor tbl : tbls) { + GridH2Table h2Tbl = tbl.h2Table(); + + Iterator<Index> idxs = h2Tbl.getIndexes().iterator(); + + // Skip first index, as it is special wrapper. + if (idxs.hasNext()) + idxs.next(); + + while (idxs.hasNext()) { + Index idx = idxs.next(); + + long idxSz = sizeOfIndex1.deepSizeOf(Integer.MAX_VALUE, false, idx).getCalculated(); + + System.out.println("Index=" + idx + ", size=" + idxSz); + + memSz += idxSz; + } + } + } + + return memSz; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/5bcac1c7/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8810b17..bc83d73 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ <module>modules/slf4j</module> <module>modules/jcl</module> <module>modules/schema-import</module> + <module>modules/memory</module> <module>modules/codegen</module> </modules>
