cstamas commented on code in PR #161: URL: https://github.com/apache/maven-resolver/pull/161#discussion_r845929757
########## src/site/markdown/configuration.md: ########## @@ -42,9 +42,10 @@ Option | Type | Description | Default Value | Supports Repo ID Suffix `aether.connector.smartChecksums` | boolean | Flag indicating that instead of comparing the external checksum fetched from the remote repo with the calculated one, it should try to extract the reference checksum from the actual artifact requests's response headers (several (strategies supported)[included-checksum-strategies.html]). This only works for transport-http transport. | `true` | no `aether.connector.userAgent` | String | The user agent that repository connectors should report to servers. | `"Aether"` | no `aether.connector.wagon.config` | Object | The configuration to use for the Wagon provider. | - | yes (must be used) -`aether.dependencyCollector.useSkip` | boolean | Flag controlling whether to skip resolving duplicate/conflicting nodes during the dependency collection process. | `true` | no `aether.dependencyCollector.maxCycles` | int | Only up to the given amount cyclic dependencies are emitted. | `10` | no `aether.dependencyCollector.maxExceptions` | int | Only exceptions up to the number given in this configuration property are emitted. Exceptions which exceed that number are swallowed. | `50` | no +`aether.dependencyCollector.impl` | String | The name of the dependency collector implementation to use: depth-first (original) named `df`, and breadth-first (new in 1.8) named `bf`. Both collectors produce equivalent results, but they may differ performance wise, depending on project being applied to. Our experience shows that existing `df` is well suited for smaller to medium size projects, while `bf` may perform better on huge projects with many dependencies. Experiment (and come back to us!) to figure out which one suits you the better. | `"df"` | no Review Comment: fixed ########## maven-resolver-impl/src/main/java/org/eclipse/aether/internal/impl/collect/df/DfDependencyCollector.java: ########## @@ -0,0 +1,866 @@ +package org.eclipse.aether.internal.impl.collect.df; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import static java.util.Objects.requireNonNull; + +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositoryException; +import org.eclipse.aether.RepositorySystemSession; +import org.eclipse.aether.RequestTrace; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.ArtifactProperties; +import org.eclipse.aether.collection.CollectRequest; +import org.eclipse.aether.collection.CollectResult; +import org.eclipse.aether.collection.DependencyCollectionException; +import org.eclipse.aether.collection.DependencyGraphTransformer; +import org.eclipse.aether.collection.DependencyManagement; +import org.eclipse.aether.collection.DependencyManager; +import org.eclipse.aether.collection.DependencySelector; +import org.eclipse.aether.collection.DependencyTraverser; +import org.eclipse.aether.collection.VersionFilter; +import org.eclipse.aether.graph.DefaultDependencyNode; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.graph.Exclusion; +import org.eclipse.aether.impl.ArtifactDescriptorReader; +import org.eclipse.aether.impl.RemoteRepositoryManager; +import org.eclipse.aether.impl.VersionRangeResolver; +import org.eclipse.aether.internal.impl.collect.CachingArtifactTypeRegistry; +import org.eclipse.aether.internal.impl.collect.DataPool; +import org.eclipse.aether.internal.impl.collect.DefaultDependencyCollectionContext; +import org.eclipse.aether.internal.impl.collect.DefaultDependencyCycle; +import org.eclipse.aether.internal.impl.collect.DefaultDependencyGraphTransformationContext; +import org.eclipse.aether.internal.impl.collect.DefaultVersionFilterContext; +import org.eclipse.aether.internal.impl.collect.DependencyCollectorDelegate; +import org.eclipse.aether.repository.ArtifactRepository; +import org.eclipse.aether.repository.RemoteRepository; +import org.eclipse.aether.resolution.ArtifactDescriptorException; +import org.eclipse.aether.resolution.ArtifactDescriptorRequest; +import org.eclipse.aether.resolution.ArtifactDescriptorResult; +import org.eclipse.aether.resolution.VersionRangeRequest; +import org.eclipse.aether.resolution.VersionRangeResolutionException; +import org.eclipse.aether.resolution.VersionRangeResult; +import org.eclipse.aether.spi.locator.Service; +import org.eclipse.aether.util.ConfigUtils; +import org.eclipse.aether.util.graph.manager.DependencyManagerUtils; +import org.eclipse.aether.util.graph.transformer.TransformationContextKeys; +import org.eclipse.aether.version.Version; + +/** + * Depth-first {@link org.eclipse.aether.impl.DependencyCollector} (the "original" default). Originally + * this class was located a package higher (as "default" implementation). + * + * @since 1.8.0 + */ +@Singleton +@Named( DfDependencyCollector.NAME ) +public class DfDependencyCollector + extends DependencyCollectorDelegate implements Service +{ + public static final String NAME = "df"; + + public DfDependencyCollector() + { + // enables default constructor + } + + @Inject + DfDependencyCollector( RemoteRepositoryManager remoteRepositoryManager, + ArtifactDescriptorReader artifactDescriptorReader, + VersionRangeResolver versionRangeResolver ) + { + super( remoteRepositoryManager, artifactDescriptorReader, versionRangeResolver ); + } + + @SuppressWarnings( "checkstyle:methodlength" ) + public CollectResult collectDependencies( RepositorySystemSession session, CollectRequest request ) + throws DependencyCollectionException + { + requireNonNull( session, "session cannot be null" ); + requireNonNull( request, "request cannot be null" ); + session = optimizeSession( session ); + + RequestTrace trace = RequestTrace.newChild( request.getTrace(), request ); + + CollectResult result = new CollectResult( request ); + + DependencySelector depSelector = session.getDependencySelector(); + DependencyManager depManager = session.getDependencyManager(); + DependencyTraverser depTraverser = session.getDependencyTraverser(); + VersionFilter verFilter = session.getVersionFilter(); + + Dependency root = request.getRoot(); + List<RemoteRepository> repositories = request.getRepositories(); + List<Dependency> dependencies = request.getDependencies(); + List<Dependency> managedDependencies = request.getManagedDependencies(); + + Map<String, Object> stats = new LinkedHashMap<>(); + long time1 = System.nanoTime(); + + DefaultDependencyNode node; + if ( root != null ) + { + List<? extends Version> versions; + VersionRangeResult rangeResult; + try + { + VersionRangeRequest rangeRequest = + new VersionRangeRequest( root.getArtifact(), request.getRepositories(), + request.getRequestContext() ); + rangeRequest.setTrace( trace ); + rangeResult = versionRangeResolver.resolveVersionRange( session, rangeRequest ); + versions = filterVersions( root, rangeResult, verFilter, new DefaultVersionFilterContext( session ) ); + } + catch ( VersionRangeResolutionException e ) + { + result.addException( e ); + throw new DependencyCollectionException( result, e.getMessage() ); + } + + Version version = versions.get( versions.size() - 1 ); + root = root.setArtifact( root.getArtifact().setVersion( version.toString() ) ); + + ArtifactDescriptorResult descriptorResult; + try + { + ArtifactDescriptorRequest descriptorRequest = new ArtifactDescriptorRequest(); + descriptorRequest.setArtifact( root.getArtifact() ); + descriptorRequest.setRepositories( request.getRepositories() ); + descriptorRequest.setRequestContext( request.getRequestContext() ); + descriptorRequest.setTrace( trace ); + if ( isLackingDescriptor( root.getArtifact() ) ) + { + descriptorResult = new ArtifactDescriptorResult( descriptorRequest ); + } + else + { + descriptorResult = descriptorReader.readArtifactDescriptor( session, descriptorRequest ); + } + } + catch ( ArtifactDescriptorException e ) + { + result.addException( e ); + throw new DependencyCollectionException( result, e.getMessage() ); + } + + root = root.setArtifact( descriptorResult.getArtifact() ); + + if ( !session.isIgnoreArtifactDescriptorRepositories() ) + { + repositories = remoteRepositoryManager.aggregateRepositories( session, repositories, + descriptorResult.getRepositories(), + true ); + } + dependencies = mergeDeps( dependencies, descriptorResult.getDependencies() ); + managedDependencies = mergeDeps( managedDependencies, descriptorResult.getManagedDependencies() ); + + node = new DefaultDependencyNode( root ); + node.setRequestContext( request.getRequestContext() ); + node.setRelocations( descriptorResult.getRelocations() ); + node.setVersionConstraint( rangeResult.getVersionConstraint() ); + node.setVersion( version ); + node.setAliases( descriptorResult.getAliases() ); + node.setRepositories( request.getRepositories() ); + } + else + { + node = new DefaultDependencyNode( request.getRootArtifact() ); + node.setRequestContext( request.getRequestContext() ); + node.setRepositories( request.getRepositories() ); + } + + result.setRoot( node ); + + boolean traverse = root == null || depTraverser == null || depTraverser.traverseDependency( root ); + String errorPath = null; + if ( traverse && !dependencies.isEmpty() ) + { + DataPool pool = new DataPool( session ); + + NodeStack nodes = new NodeStack(); + nodes.push( node ); + + DefaultDependencyCollectionContext context = + new DefaultDependencyCollectionContext( session, request.getRootArtifact(), root, managedDependencies ); + + DefaultVersionFilterContext versionContext = new DefaultVersionFilterContext( session ); + + Args args = new Args( session, trace, pool, nodes, context, versionContext, request ); + Results results = new Results( result, session ); + + process( args, results, dependencies, repositories, + depSelector != null ? depSelector.deriveChildSelector( context ) : null, + depManager != null ? depManager.deriveChildManager( context ) : null, + depTraverser != null ? depTraverser.deriveChildTraverser( context ) : null, + verFilter != null ? verFilter.deriveChildFilter( context ) : null ); + + errorPath = results.errorPath; + } + + long time2 = System.nanoTime(); + + DependencyGraphTransformer transformer = session.getDependencyGraphTransformer(); + if ( transformer != null ) + { + try + { + DefaultDependencyGraphTransformationContext context = + new DefaultDependencyGraphTransformationContext( session ); + context.put( TransformationContextKeys.STATS, stats ); + result.setRoot( transformer.transformGraph( node, context ) ); + } + catch ( RepositoryException e ) + { + result.addException( e ); + } + } + + long time3 = System.nanoTime(); + stats.put( "DefaultDependencyCollector.collectTime", time2 - time1 ); + stats.put( "DefaultDependencyCollector.transformTime", time3 - time2 ); + logger.debug( "Dependency collection stats {}", stats ); + + if ( errorPath != null ) + { + throw new DependencyCollectionException( result, "Failed to collect dependencies at " + errorPath ); + } + if ( !result.getExceptions().isEmpty() ) + { + throw new DependencyCollectionException( result ); + } + + return result; + } + + private static RepositorySystemSession optimizeSession( RepositorySystemSession session ) + { + DefaultRepositorySystemSession optimized = new DefaultRepositorySystemSession( session ); + optimized.setArtifactTypeRegistry( CachingArtifactTypeRegistry.newInstance( session ) ); + return optimized; + } + + private List<Dependency> mergeDeps( List<Dependency> dominant, List<Dependency> recessive ) + { + List<Dependency> result; + if ( dominant == null || dominant.isEmpty() ) + { + result = recessive; + } + else if ( recessive == null || recessive.isEmpty() ) + { + result = dominant; + } + else + { + int initialCapacity = dominant.size() + recessive.size(); + result = new ArrayList<>( initialCapacity ); + Collection<String> ids = new HashSet<>( initialCapacity, 1.0f ); + for ( Dependency dependency : dominant ) + { + ids.add( getId( dependency.getArtifact() ) ); + result.add( dependency ); + } + for ( Dependency dependency : recessive ) + { + if ( !ids.contains( getId( dependency.getArtifact() ) ) ) + { + result.add( dependency ); + } + } + } + return result; + } + + private static String getId( Artifact a ) + { + return a.getGroupId() + ':' + a.getArtifactId() + ':' + a.getClassifier() + ':' + a.getExtension(); + } + + @SuppressWarnings( "checkstyle:parameternumber" ) + private void process( final Args args, Results results, List<Dependency> dependencies, + List<RemoteRepository> repositories, DependencySelector depSelector, + DependencyManager depManager, DependencyTraverser depTraverser, VersionFilter verFilter ) + { + for ( Dependency dependency : dependencies ) + { + processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter, + dependency ); + } + } + + @SuppressWarnings( "checkstyle:parameternumber" ) + private void processDependency( Args args, Results results, List<RemoteRepository> repositories, + DependencySelector depSelector, DependencyManager depManager, + DependencyTraverser depTraverser, VersionFilter verFilter, Dependency dependency ) + { + + List<Artifact> relocations = Collections.emptyList(); + processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter, dependency, + relocations, false ); + } + + @SuppressWarnings( "checkstyle:parameternumber" ) + private void processDependency( Args args, Results results, List<RemoteRepository> repositories, + DependencySelector depSelector, DependencyManager depManager, + DependencyTraverser depTraverser, VersionFilter verFilter, Dependency dependency, + List<Artifact> relocations, boolean disableVersionManagement ) + { + + if ( depSelector != null && !depSelector.selectDependency( dependency ) ) + { + return; + } + + PremanagedDependency preManaged = + PremanagedDependency.create( depManager, dependency, disableVersionManagement, args.premanagedState ); + dependency = preManaged.managedDependency; + + boolean noDescriptor = isLackingDescriptor( dependency.getArtifact() ); + + boolean traverse = !noDescriptor && ( depTraverser == null || depTraverser.traverseDependency( dependency ) ); + + List<? extends Version> versions; + VersionRangeResult rangeResult; + try + { + VersionRangeRequest rangeRequest = createVersionRangeRequest( args, repositories, dependency ); + + rangeResult = cachedResolveRangeResult( rangeRequest, args.pool, args.session ); + + versions = filterVersions( dependency, rangeResult, verFilter, args.versionContext ); + } + catch ( VersionRangeResolutionException e ) + { + results.addException( dependency, e, args.nodes ); + return; + } + + for ( Version version : versions ) + { + Artifact originalArtifact = dependency.getArtifact().setVersion( version.toString() ); + Dependency d = dependency.setArtifact( originalArtifact ); + + ArtifactDescriptorRequest descriptorRequest = createArtifactDescriptorRequest( args, repositories, d ); + + final ArtifactDescriptorResult descriptorResult = + getArtifactDescriptorResult( args, results, noDescriptor, d, descriptorRequest ); + if ( descriptorResult != null ) + { + d = d.setArtifact( descriptorResult.getArtifact() ); + + DependencyNode node = args.nodes.top(); + + int cycleEntry = DefaultDependencyCycle.find( args.nodes.nodes, d.getArtifact() ); + if ( cycleEntry >= 0 ) + { + results.addCycle( args.nodes, cycleEntry, d ); + DependencyNode cycleNode = args.nodes.get( cycleEntry ); + if ( cycleNode.getDependency() != null ) + { + DefaultDependencyNode child = + createDependencyNode( relocations, preManaged, rangeResult, version, d, descriptorResult, + cycleNode ); + node.getChildren().add( child ); + continue; + } + } + + if ( !descriptorResult.getRelocations().isEmpty() ) + { + boolean disableVersionManagementSubsequently = + originalArtifact.getGroupId().equals( d.getArtifact().getGroupId() ) + && originalArtifact.getArtifactId().equals( d.getArtifact().getArtifactId() ); + + processDependency( args, results, repositories, depSelector, depManager, depTraverser, verFilter, d, + descriptorResult.getRelocations(), disableVersionManagementSubsequently ); + return; + } + else + { + d = args.pool.intern( d.setArtifact( args.pool.intern( d.getArtifact() ) ) ); + + List<RemoteRepository> repos = + getRemoteRepositories( rangeResult.getRepository( version ), repositories ); + + DefaultDependencyNode child = + createDependencyNode( relocations, preManaged, rangeResult, version, d, + descriptorResult.getAliases(), repos, args.request.getRequestContext() ); + + node.getChildren().add( child ); + + boolean recurse = traverse && !descriptorResult.getDependencies().isEmpty(); + if ( recurse ) + { + doRecurse( args, results, repositories, depSelector, depManager, depTraverser, verFilter, d, + descriptorResult, child ); + } + } + } + else + { + DependencyNode node = args.nodes.top(); + List<RemoteRepository> repos = + getRemoteRepositories( rangeResult.getRepository( version ), repositories ); + DefaultDependencyNode child = + createDependencyNode( relocations, preManaged, rangeResult, version, d, null, repos, + args.request.getRequestContext() ); + node.getChildren().add( child ); + } + } + } + + @SuppressWarnings( "checkstyle:parameternumber" ) + private void doRecurse( Args args, Results results, List<RemoteRepository> repositories, + DependencySelector depSelector, DependencyManager depManager, + DependencyTraverser depTraverser, VersionFilter verFilter, Dependency d, + ArtifactDescriptorResult descriptorResult, DefaultDependencyNode child ) + { + DefaultDependencyCollectionContext context = args.collectionContext; + context.set( d, descriptorResult.getManagedDependencies() ); + + DependencySelector childSelector = depSelector != null ? depSelector.deriveChildSelector( context ) : null; + DependencyManager childManager = depManager != null ? depManager.deriveChildManager( context ) : null; + DependencyTraverser childTraverser = depTraverser != null ? depTraverser.deriveChildTraverser( context ) : null; + VersionFilter childFilter = verFilter != null ? verFilter.deriveChildFilter( context ) : null; + + final List<RemoteRepository> childRepos = + args.ignoreRepos + ? repositories + : remoteRepositoryManager.aggregateRepositories( args.session, repositories, + descriptorResult.getRepositories(), true ); + + Object key = + args.pool.toKey( d.getArtifact(), childRepos, childSelector, childManager, childTraverser, childFilter ); + + List<DependencyNode> children = args.pool.getChildren( key ); + if ( children == null ) + { + args.pool.putChildren( key, child.getChildren() ); + + args.nodes.push( child ); + + process( args, results, descriptorResult.getDependencies(), childRepos, childSelector, childManager, + childTraverser, childFilter ); + + args.nodes.pop(); + } + else + { + child.setChildren( children ); + } + } + + private ArtifactDescriptorResult getArtifactDescriptorResult( Args args, Results results, boolean noDescriptor, + Dependency d, + ArtifactDescriptorRequest descriptorRequest ) + { + return noDescriptor + ? new ArtifactDescriptorResult( descriptorRequest ) + : resolveCachedArtifactDescriptor( args.pool, descriptorRequest, args.session, d, results, args ); + + } + + private ArtifactDescriptorResult resolveCachedArtifactDescriptor( DataPool pool, + ArtifactDescriptorRequest descriptorRequest, + RepositorySystemSession session, Dependency d, + Results results, Args args ) + { + Object key = pool.toKey( descriptorRequest ); + ArtifactDescriptorResult descriptorResult = pool.getDescriptor( key, descriptorRequest ); + if ( descriptorResult == null ) + { + try + { + descriptorResult = descriptorReader.readArtifactDescriptor( session, descriptorRequest ); + pool.putDescriptor( key, descriptorResult ); + } + catch ( ArtifactDescriptorException e ) + { + results.addException( d, e, args.nodes ); + pool.putDescriptor( key, e ); + return null; + } + + } + else if ( descriptorResult == DataPool.NO_DESCRIPTOR ) + { + return null; + } + + return descriptorResult; + } + + @SuppressWarnings( "checkstyle:parameternumber" ) + private static DefaultDependencyNode createDependencyNode( List<Artifact> relocations, + PremanagedDependency preManaged, + VersionRangeResult rangeResult, Version version, + Dependency d, Collection<Artifact> aliases, + List<RemoteRepository> repos, String requestContext ) + { + DefaultDependencyNode child = new DefaultDependencyNode( d ); + preManaged.applyTo( child ); + child.setRelocations( relocations ); + child.setVersionConstraint( rangeResult.getVersionConstraint() ); + child.setVersion( version ); + child.setAliases( aliases ); + child.setRepositories( repos ); + child.setRequestContext( requestContext ); + return child; + } + + private static DefaultDependencyNode createDependencyNode( List<Artifact> relocations, + PremanagedDependency preManaged, + VersionRangeResult rangeResult, Version version, + Dependency d, ArtifactDescriptorResult descriptorResult, + DependencyNode cycleNode ) + { + DefaultDependencyNode child = + createDependencyNode( relocations, preManaged, rangeResult, version, d, descriptorResult.getAliases(), + cycleNode.getRepositories(), cycleNode.getRequestContext() ); + child.setChildren( cycleNode.getChildren() ); + return child; + } + + private static ArtifactDescriptorRequest createArtifactDescriptorRequest( Args args, + List<RemoteRepository> repositories, + Dependency d ) + { + ArtifactDescriptorRequest descriptorRequest = new ArtifactDescriptorRequest(); + descriptorRequest.setArtifact( d.getArtifact() ); + descriptorRequest.setRepositories( repositories ); + descriptorRequest.setRequestContext( args.request.getRequestContext() ); + descriptorRequest.setTrace( args.trace ); + return descriptorRequest; + } + + private static VersionRangeRequest createVersionRangeRequest( Args args, List<RemoteRepository> repositories, + Dependency dependency ) + { + VersionRangeRequest rangeRequest = new VersionRangeRequest(); + rangeRequest.setArtifact( dependency.getArtifact() ); + rangeRequest.setRepositories( repositories ); + rangeRequest.setRequestContext( args.request.getRequestContext() ); + rangeRequest.setTrace( args.trace ); + return rangeRequest; + } + + private VersionRangeResult cachedResolveRangeResult( VersionRangeRequest rangeRequest, DataPool pool, + RepositorySystemSession session ) + throws VersionRangeResolutionException + { + Object key = pool.toKey( rangeRequest ); + VersionRangeResult rangeResult = pool.getConstraint( key, rangeRequest ); + if ( rangeResult == null ) + { + rangeResult = versionRangeResolver.resolveVersionRange( session, rangeRequest ); + pool.putConstraint( key, rangeResult ); + } + return rangeResult; + } + + private static boolean isLackingDescriptor( Artifact artifact ) + { + return artifact.getProperty( ArtifactProperties.LOCAL_PATH, null ) != null; + } + + private static List<RemoteRepository> getRemoteRepositories( ArtifactRepository repository, + List<RemoteRepository> repositories ) + { + if ( repository instanceof RemoteRepository ) + { + return Collections.singletonList( (RemoteRepository) repository ); + } + if ( repository != null ) + { + return Collections.emptyList(); + } + return repositories; + } + + private static List<? extends Version> filterVersions( Dependency dependency, VersionRangeResult rangeResult, + VersionFilter verFilter, + DefaultVersionFilterContext verContext ) + throws VersionRangeResolutionException + { + if ( rangeResult.getVersions().isEmpty() ) + { + throw new VersionRangeResolutionException( rangeResult, + "No versions available for " + dependency.getArtifact() + + " within specified range" ); + } + + List<? extends Version> versions; + if ( verFilter != null && rangeResult.getVersionConstraint().getRange() != null ) + { + verContext.set( dependency, rangeResult ); + try + { + verFilter.filterVersions( verContext ); + } + catch ( RepositoryException e ) + { + throw new VersionRangeResolutionException( rangeResult, + "Failed to filter versions for " + dependency.getArtifact() + + ": " + e.getMessage(), e ); Review Comment: fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
