Author: ogusakov
Date: Mon May 18 23:17:24 2009
New Revision: 776133
URL: http://svn.apache.org/viewvc?rev=776133&view=rev
Log:
[MERCURY-123] - trying to cache results from DependencyProcessor() to enable
once-only processing on POMs. This should be linked to session-like behavior
because next session may happen in quite some time and data should be properly
aged
Added:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
(with props)
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
(with props)
Modified:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
maven/mercury/trunk/mercury-it/pom.xml
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
(props changed)
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
Modified:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
---
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
(original)
+++
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/cache/fs/MetadataCacheFs.java
Mon May 18 23:17:24 2009
@@ -62,6 +62,16 @@
public static final String EVENT_SAVE_RAW = "save.raw";
+// public static final String SYSTEM_PROPERTY_CACHE_METADATA =
"mercury.cache.metadata";
+//
+// /** by default - cache metadata in memory */
+// private static final boolean cacheMetadata = Boolean.valueOf(
System.getProperty( SYSTEM_PROPERTY_CACHE_METADATA, "true" ) );
+
+ public static final String SYSTEM_PROPERTY_CACHE_RAW = "mercury.cache.raw";
+
+ /** by default - do not cache raw data in memory to preserve RAM */
+ private static final boolean cacheRaw = Boolean.valueOf(
System.getProperty( SYSTEM_PROPERTY_CACHE_RAW, "false" ) );
+
private static final Language LANG = new DefaultLanguage(
RepositoryGAVMetadata.class );
static volatile Map<String, MetadataCacheFs> fsCaches =
@@ -72,11 +82,12 @@
(Map<String, RepositoryGAMetadata>) Collections.synchronizedMap( new
HashMap<String, RepositoryGAMetadata>( 512 ) );
private volatile Map<String, RepositoryGAVMetadata> gavCache =
- (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new
HashMap<String, RepositoryGAVMetadata>(
-
1024 ) );
+ (Map<String, RepositoryGAVMetadata>) Collections.synchronizedMap( new
HashMap<String, RepositoryGAVMetadata>( 1024 ) );
- private volatile Map<String, byte[]> rawCache =
- (Map<String, byte[]>) Collections.synchronizedMap( new HashMap<String,
byte[]>( 1024 ) );
+ private volatile Map<String, byte[]> rawCache = cacheRaw
+ ? (Map<String, byte[]>) Collections.synchronizedMap( new
HashMap<String, byte[]>( 1024 ) )
+ : null
+ ;
private File root;
@@ -374,7 +385,10 @@
if ( _eventManager != null )
event = new GenericEvent( EventTypeEnum.fsCache,
EVENT_FIND_RAW, rawKey );
- byte[] res = rawCache.get( rawKey );
+ byte[] res = cacheRaw
+ ? rawCache.get( rawKey )
+ : null
+ ;
if ( res != null )
{
@@ -394,7 +408,8 @@
res = FileUtil.readRawData( f );
- rawCache.put( rawKey, res );
+ if( cacheRaw)
+ rawCache.put( rawKey, res );
if ( _eventManager != null )
event.setResult( "found on disk" );
@@ -428,7 +443,8 @@
if ( _eventManager != null )
event = new GenericEvent( EventTypeEnum.fsCache,
EVENT_SAVE_RAW, rawKey );
- rawCache.put( rawKey, rawBytes );
+ if( cacheRaw )
+ rawCache.put( rawKey, rawBytes );
File f =
new File( getGAVDir( md.getEffectiveCoordinates() ),
md.getArtifactId() + FileUtil.DASH
@@ -519,7 +535,9 @@
public void clearSession()
throws MetadataCacheException
{
- rawCache.clear();
+ if( cacheRaw )
+ rawCache.clear();
+
gaCache.clear();
gavCache.clear();
}
Modified:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
---
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
(original)
+++
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/repository/virtual/VirtualRepositoryReader.java
Mon May 18 23:17:24 2009
@@ -57,6 +57,8 @@
import org.apache.maven.mercury.repository.cache.fs.MetadataCacheFs;
import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryM2;
import org.apache.maven.mercury.repository.remote.m2.RemoteRepositoryReaderM2;
+import org.apache.maven.mercury.util.LruMemCache;
+import org.apache.maven.mercury.util.MemCache;
import org.apache.maven.mercury.util.Util;
import org.codehaus.plexus.lang.DefaultLanguage;
import org.codehaus.plexus.lang.Language;
@@ -119,6 +121,15 @@
private boolean _initialized = false;
private EventManager _eventManager;
+
+ public static final String SYSTEM_PROPERTY_VERSION_CACHE_SIZE =
"mercury.version.cache.size";
+
+ private static final int _versionCacheSize = Integer.valueOf(
System.getProperty( SYSTEM_PROPERTY_VERSION_CACHE_SIZE, "1024" ) );
+
+ private static final MemCache<ArtifactMetadata, List<ArtifactMetadata> >
_cachedVersions =
+ _versionCacheSize == 0 ? null
+ : new LruMemCache<ArtifactMetadata, List<ArtifactMetadata>>(
_versionCacheSize )
+ ;
//
----------------------------------------------------------------------------------------------------------------------------
public VirtualRepositoryReader( Collection<Repository> repositories )
@@ -301,6 +312,59 @@
}
}
+
//----------------------------------------------------------------------------------------------------------------------------
+ private MetadataResults readCachedVersions( Collection<ArtifactMetadata>
query, List<ArtifactMetadata> leftOvers )
+ {
+ if( _cachedVersions == null )
+ {
+ leftOvers.addAll( query );
+ return null;
+ }
+
+ MetadataResults res = null;
+
+ for( ArtifactMetadata key : query )
+ {
+ List<ArtifactMetadata> vl = _cachedVersions.get( key );
+ if( Util.isEmpty( vl ) )
+ leftOvers.add( key );
+ else
+ {
+ if( res == null )
+ res = new MetadataResults( key, vl );
+ else
+ res.add( key, vl );
+ }
+ }
+
+ return res;
+ }
+
//----------------------------------------------------------------------------------------------------------------------------
+ private MetadataResults cacheVersions( MetadataResults res )
+ {
+ if(true)
+ return res;
+
+ if( _cachedVersions == null || res == null || res.hasExceptions() || !
res.hasResults() )
+ {
+ return res;
+ }
+
+ for( ArtifactMetadata key :res.getResults().keySet() )
+ {
+ List<ArtifactMetadata> vl = res.getResult( key );
+
+ if( Util.isEmpty( vl ) )
+ continue;
+
+ for( ArtifactMetadata md : vl )
+ md.setTracker( null );
+
+ _cachedVersions.put( key, vl );
+ }
+
+ return res;
+ }
//
----------------------------------------------------------------------------------------------------------------------------
public MetadataResults readVersions( Collection<ArtifactMetadata> query )
throws IllegalArgumentException, RepositoryException
@@ -321,13 +385,16 @@
event = new GenericEvent(
EventTypeEnum.virtualRepositoryReader, EVENT_READ_VERSIONS );
}
- MetadataResults res = null;
ArtifactListProcessor tp = _processors == null ? null :
_processors.get( ArtifactListProcessor.FUNCTION_TP );
GenericEvent eventRead = null;
List<ArtifactMetadata> qList = new ArrayList<ArtifactMetadata>(
query.size() );
- qList.addAll( query );
+// qList.addAll( query );
+ MetadataResults res = readCachedVersions( query, qList );
+
+ if( Util.isEmpty( qList ) )
+ return res;
for ( RepositoryReader rr : _repositoryReaders )
{
@@ -425,7 +492,7 @@
processSingletons( res );
}
- return res;
+ return cacheVersions( res );
}
finally
{
Added:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java?rev=776133&view=auto
==============================================================================
---
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
(added)
+++
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
Mon May 18 23:17:24 2009
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public class LruMemCache<K, V>
+ implements MemCache<K, V>
+{
+ ConcurrentHashMap<K, V> _cache;
+
+ K _lastKey;
+
+ int _sz;
+
+ public LruMemCache()
+ {
+ this( DEFAULT_CACHE_SIZE );
+ }
+
+ public LruMemCache( int sz )
+ {
+ _sz = sz;
+ _cache = new ConcurrentHashMap<K, V>( sz );
+ }
+
+ public V get( K key )
+ {
+ return _cache.get( key );
+ }
+
+ public void put( K key, V val )
+ {
+ // MRU is easier - do it first ..
+ synchronized( _cache )
+ {
+ if( _cache.size() == _sz )
+ {// free up one slot
+ _cache.remove( _lastKey );
+ _lastKey = key;
+ }
+ }
+
+ _cache.put( key, val );
+ }
+
+}
Propchange:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/LruMemCache.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Added:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java?rev=776133&view=auto
==============================================================================
---
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
(added)
+++
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
Mon May 18 23:17:24 2009
@@ -0,0 +1,36 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
+package org.apache.maven.mercury.util;
+
+/**
+ *
+ *
+ * @author Oleg Gusakov
+ * @version $Id$
+ *
+ */
+public interface MemCache<K extends Object, V extends Object>
+{
+ public static final int DEFAULT_CACHE_SIZE = 1024;
+
+ void put( K key, V val );
+
+ V get( K key );
+}
Propchange:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
maven/mercury/trunk/mercury-core/src/main/java/org/apache/maven/mercury/util/MemCache.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision
Modified: maven/mercury/trunk/mercury-it/pom.xml
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-it/pom.xml?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
--- maven/mercury/trunk/mercury-it/pom.xml (original)
+++ maven/mercury/trunk/mercury-it/pom.xml Mon May 18 23:17:24 2009
@@ -182,7 +182,7 @@
</dependencies>
<build>
-
+ <!--
<filters>
<filter>src/test/filters/filter.txt</filter>
</filters>
@@ -209,7 +209,8 @@
</testResources>
-
+ -->
+
<plugins>
<plugin>
Propchange:
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
('svn:eol-style' removed)
Propchange:
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified:
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
URL:
http://svn.apache.org/viewvc/maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1?rev=776133&r1=776132&r2=776133&view=diff
==============================================================================
---
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
(original)
+++
maven/mercury/trunk/mercury-it/src/test/resources/remoteRepo/asm/asm-parent/3.0/asm-parent-3.0.pom.sha1
Mon May 18 23:17:24 2009
@@ -1 +1 @@
-526bfebc865ac047ff3fa7d77924a4edff7ba468
\ No newline at end of file
+5ab8cde950b8b29bc5c8d79b8df5a804a76e573e
