diff --git a/pkg/metadata/dependencies.go b/pkg/metadata/dependencies.go deleted file mode 100644 index f70efdd1..00000000 --- a/pkg/metadata/dependencies.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metadata - -import ( - "regexp" - "sort" - "strings" - - "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" - "github.com/apache/camel-k/pkg/util/camel" -) - -var ( - additionalDependencies = map[string]string{ - ".*JsonLibrary\\.Jackson.*": "camel:jackson", - } -) - -// discoverDependencies returns a list of dependencies required by the given source code -func discoverDependencies(source v1alpha1.SourceSpec, fromURIs []string, toURIs []string) []string { - candidateMap := make(map[string]bool) - uris := make([]string, 0, len(fromURIs)+len(toURIs)) - uris = append(uris, fromURIs...) - uris = append(uris, toURIs...) - for _, uri := range uris { - candidateComp := decodeComponent(uri) - if candidateComp != "" { - candidateMap[candidateComp] = true - } - } - additional := findAdditionalDependencies(source) - for _, dep := range additional { - candidateMap[dep] = true - } - // Remove duplicates and sort - candidateComponents := make([]string, 0, len(candidateMap)) - for cmp := range candidateMap { - candidateComponents = append(candidateComponents, cmp) - } - sort.Strings(candidateComponents) - return candidateComponents -} - -func decodeComponent(uri string) string { - uriSplit := strings.SplitN(uri, ":", 2) - if len(uriSplit) < 2 { - return "" - } - uriStart := uriSplit[0] - if component := camel.Runtime.GetArtifactByScheme(uriStart); component != nil { - artifactID := component.ArtifactID - if component.GroupID == "org.apache.camel" && strings.HasPrefix(artifactID, "camel-") { - return "camel:" + artifactID[6:] - } - if component.GroupID == "org.apache.camel.k" && strings.HasPrefix(artifactID, "camel-") { - return "camel-k:" + artifactID[6:] - } - return "mvn:" + component.GroupID + ":" + artifactID + ":" + component.Version - } - return "" -} - -func findAdditionalDependencies(source v1alpha1.SourceSpec) []string { - additional := make([]string, 0) - for pattern, dep := range additionalDependencies { - pat := regexp.MustCompile(pattern) - if pat.MatchString(source.Content) { - additional = append(additional, dep) - } - } - return additional -} diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go index bbdacfdb..5b2940c5 100644 --- a/pkg/metadata/metadata.go +++ b/pkg/metadata/metadata.go @@ -28,9 +28,11 @@ import ( func ExtractAll(sources []v1alpha1.SourceSpec) IntegrationMetadata { // neutral metadata meta := IntegrationMetadata{ - Dependencies: []string{}, - FromURIs: []string{}, - ToURIs: []string{}, + Metadata: src.Metadata{ + FromURIs: []string{}, + ToURIs: []string{}, + Dependencies: []string{}, + }, PassiveEndpoints: true, RequiresHTTPService: false, } @@ -54,9 +56,11 @@ func merge(m1 IntegrationMetadata, m2 IntegrationMetadata) IntegrationMetadata { } sort.Strings(allDependencies) return IntegrationMetadata{ - FromURIs: append(m1.FromURIs, m2.FromURIs...), - ToURIs: append(m1.ToURIs, m2.ToURIs...), - Dependencies: allDependencies, + Metadata: src.Metadata{ + FromURIs: append(m1.FromURIs, m2.FromURIs...), + ToURIs: append(m1.ToURIs, m2.ToURIs...), + Dependencies: allDependencies, + }, RequiresHTTPService: m1.RequiresHTTPService || m2.RequiresHTTPService, PassiveEndpoints: m1.PassiveEndpoints && m2.PassiveEndpoints, } @@ -65,20 +69,16 @@ func merge(m1 IntegrationMetadata, m2 IntegrationMetadata) IntegrationMetadata { // Extract returns metadata information from the source code func Extract(source v1alpha1.SourceSpec) IntegrationMetadata { language := source.InferLanguage() + + m := IntegrationMetadata{} + // TODO: handle error - fromURIs, _ := src.InspectorForLanguage(language).FromURIs(source) - // TODO:: handle error - toURIs, _ := src.InspectorForLanguage(language).ToURIs(source) - dependencies := discoverDependencies(source, fromURIs, toURIs) - requiresHTTPService := requiresHTTPService(source, fromURIs) - passiveEndpoints := hasOnlyPassiveEndpoints(source, fromURIs) - return IntegrationMetadata{ - FromURIs: fromURIs, - ToURIs: toURIs, - Dependencies: dependencies, - RequiresHTTPService: requiresHTTPService, - PassiveEndpoints: passiveEndpoints, - } + _ = src.InspectorForLanguage(language).Extract(source, &m.Metadata) + + m.RequiresHTTPService = requiresHTTPService(source, m.FromURIs) + m.PassiveEndpoints = hasOnlyPassiveEndpoints(source, m.FromURIs) + + return m } // Each -- diff --git a/pkg/metadata/types.go b/pkg/metadata/types.go index af81ef11..538553fb 100644 --- a/pkg/metadata/types.go +++ b/pkg/metadata/types.go @@ -17,14 +17,11 @@ limitations under the License. package metadata +import "github.com/apache/camel-k/pkg/util/source" + // IntegrationMetadata contains aggregate metadata about all Camel routes in a integrations type IntegrationMetadata struct { - // All starting URIs of defined routes - FromURIs []string - // All end URIs of defined routes - ToURIs []string - // All inferred dependencies required to run the integration - Dependencies []string + source.Metadata // RequiresHTTPService indicates if the integration needs to be invoked through HTTP RequiresHTTPService bool // PassiveEndpoints indicates that the integration contains only passive endpoints that are activated from diff --git a/pkg/util/source/inspector.go b/pkg/util/source/inspector.go index db76c490..550f0643 100644 --- a/pkg/util/source/inspector.go +++ b/pkg/util/source/inspector.go @@ -19,8 +19,13 @@ package source import ( "regexp" + "sort" + "strings" "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + "github.com/apache/camel-k/pkg/util" + "github.com/apache/camel-k/pkg/util/camel" + "github.com/scylladb/go-set/strset" ) var ( @@ -32,12 +37,15 @@ var ( doubleQuotedTo = regexp.MustCompile(`\.to\s*\(\s*"([a-z0-9-]+:[^"]+)"\s*\)`) doubleQuotedToD = regexp.MustCompile(`\.toD\s*\(\s*"([a-z0-9-]+:[^"]+)"\s*\)`) doubleQuotedToF = regexp.MustCompile(`\.toF\s*\(\s*"([a-z0-9-]+:[^"]+)"[^)]*\)`) + + additionalDependencies = map[string]string{ + ".*JsonLibrary\\.Jackson.*": "camel:jackson", + } ) // Inspector -- type Inspector interface { - FromURIs(v1alpha1.SourceSpec) ([]string, error) - ToURIs(v1alpha1.SourceSpec) ([]string, error) + Extract(v1alpha1.SourceSpec, *Metadata) error } // InspectorForLanguage -- @@ -56,34 +64,57 @@ func InspectorForLanguage(language v1alpha1.Language) Inspector { case v1alpha1.LanguageYamlFlow: return &YAMLFlowInspector{} } - return &noInspector{} + return &baseInspector{} } -func findAllDistinctStringSubmatch(data string, regexps ...*regexp.Regexp) []string { - candidates := make([]string, 0) - alreadyFound := make(map[string]bool) - for _, reg := range regexps { - hits := reg.FindAllStringSubmatch(data, -1) - for _, hit := range hits { - if len(hit) > 1 { - for _, match := range hit[1:] { - if _, ok := alreadyFound[match]; !ok { - alreadyFound[match] = true - candidates = append(candidates, match) - } - } - } - } - } - return candidates +type baseInspector struct { } -type noInspector struct { +func (i baseInspector) Extract(v1alpha1.SourceSpec, *Metadata) error { + return nil } -func (i noInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { - return []string{}, nil +// discoverDependencies returns a list of dependencies required by the given source code +func (i *baseInspector) discoverDependencies(source v1alpha1.SourceSpec, meta *Metadata) []string { + uris := util.StringSliceJoin(meta.FromURIs, meta.ToURIs) + candidates := strset.New() + + for _, uri := range uris { + candidateComp := i.decodeComponent(uri) + if candidateComp != "" { + candidates.Add(candidateComp) + } + } + + for pattern, dep := range additionalDependencies { + pat := regexp.MustCompile(pattern) + if pat.MatchString(source.Content) { + candidates.Add(dep) + } + } + + components := candidates.List() + + sort.Strings(components) + + return components } -func (i noInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - return []string{}, nil + +func (i *baseInspector) decodeComponent(uri string) string { + uriSplit := strings.SplitN(uri, ":", 2) + if len(uriSplit) < 2 { + return "" + } + uriStart := uriSplit[0] + if component := camel.Runtime.GetArtifactByScheme(uriStart); component != nil { + artifactID := component.ArtifactID + if component.GroupID == "org.apache.camel" && strings.HasPrefix(artifactID, "camel-") { + return "camel:" + artifactID[6:] + } + if component.GroupID == "org.apache.camel.k" && strings.HasPrefix(artifactID, "camel-") { + return "camel-k:" + artifactID[6:] + } + return "mvn:" + component.GroupID + ":" + artifactID + ":" + component.Version + } + return "" } diff --git a/pkg/util/source/inspector_groovy.go b/pkg/util/source/inspector_groovy.go index ddcd8748..62db6293 100644 --- a/pkg/util/source/inspector_groovy.go +++ b/pkg/util/source/inspector_groovy.go @@ -17,26 +17,24 @@ limitations under the License. package source -import "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +import ( + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + "github.com/apache/camel-k/pkg/util" +) // GroovyInspector -- type GroovyInspector struct { + baseInspector } -// FromURIs -- -func (i GroovyInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( +// Extract -- +func (i GroovyInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { + from := util.FindAllDistinctStringSubmatch( source.Content, singleQuotedFrom, doubleQuotedFrom, ) - - return answer, nil -} - -// ToURIs -- -func (i GroovyInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( + to := util.FindAllDistinctStringSubmatch( source.Content, singleQuotedTo, doubleQuotedTo, @@ -46,5 +44,9 @@ func (i GroovyInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { doubleQuotedToF, ) - return answer, nil + meta.FromURIs = append(meta.FromURIs, from...) + meta.ToURIs = append(meta.ToURIs, to...) + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/inspector_java_script.go b/pkg/util/source/inspector_java_script.go index 913fc8df..87d53eae 100644 --- a/pkg/util/source/inspector_java_script.go +++ b/pkg/util/source/inspector_java_script.go @@ -17,26 +17,24 @@ limitations under the License. package source -import "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +import ( + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + "github.com/apache/camel-k/pkg/util" +) // JavaScriptInspector -- type JavaScriptInspector struct { + baseInspector } -// FromURIs -- -func (i JavaScriptInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( +// Extract -- +func (i JavaScriptInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { + from := util.FindAllDistinctStringSubmatch( source.Content, singleQuotedFrom, doubleQuotedFrom, ) - - return answer, nil -} - -// ToURIs -- -func (i JavaScriptInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( + to := util.FindAllDistinctStringSubmatch( source.Content, singleQuotedTo, doubleQuotedTo, @@ -46,5 +44,9 @@ func (i JavaScriptInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error doubleQuotedToF, ) - return answer, nil + meta.FromURIs = append(meta.FromURIs, from...) + meta.ToURIs = append(meta.ToURIs, to...) + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/inspector_java_source.go b/pkg/util/source/inspector_java_source.go index 4a229539..938e75bf 100644 --- a/pkg/util/source/inspector_java_source.go +++ b/pkg/util/source/inspector_java_source.go @@ -17,30 +17,32 @@ limitations under the License. package source -import "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +import ( + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + "github.com/apache/camel-k/pkg/util" +) // JavaSourceInspector -- type JavaSourceInspector struct { + baseInspector } -// FromURIs -- -func (i JavaSourceInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( +// Extract -- +func (i JavaSourceInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { + from := util.FindAllDistinctStringSubmatch( source.Content, doubleQuotedFrom, ) - - return answer, nil -} - -// ToURIs -- -func (i JavaSourceInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( + to := util.FindAllDistinctStringSubmatch( source.Content, doubleQuotedTo, doubleQuotedToD, doubleQuotedToF, ) - return answer, nil + meta.FromURIs = append(meta.FromURIs, from...) + meta.ToURIs = append(meta.ToURIs, to...) + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/inspector_kotlin.go b/pkg/util/source/inspector_kotlin.go index 111bf14d..9277f9b2 100644 --- a/pkg/util/source/inspector_kotlin.go +++ b/pkg/util/source/inspector_kotlin.go @@ -17,30 +17,32 @@ limitations under the License. package source -import "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +import ( + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + "github.com/apache/camel-k/pkg/util" +) // KotlinInspector -- type KotlinInspector struct { + baseInspector } -// FromURIs -- -func (i KotlinInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( +// Extract -- +func (i KotlinInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { + from := util.FindAllDistinctStringSubmatch( source.Content, doubleQuotedFrom, ) - - return answer, nil -} - -// ToURIs -- -func (i KotlinInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - answer := findAllDistinctStringSubmatch( + to := util.FindAllDistinctStringSubmatch( source.Content, doubleQuotedTo, doubleQuotedToD, doubleQuotedToF, ) - return answer, nil + meta.FromURIs = append(meta.FromURIs, from...) + meta.ToURIs = append(meta.ToURIs, to...) + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/inspector_xml.go b/pkg/util/source/inspector_xml.go index b2bd2e8a..923bd405 100644 --- a/pkg/util/source/inspector_xml.go +++ b/pkg/util/source/inspector_xml.go @@ -26,15 +26,14 @@ import ( // XMLInspector -- type XMLInspector struct { + baseInspector } -// FromURIs -- -func (i XMLInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { +// Extract -- +func (i XMLInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { content := strings.NewReader(source.Content) decoder := xml.NewDecoder(content) - uris := make([]string, 0) - for { // Read tokens from the XML document in a stream. t, _ := decoder.Token() @@ -47,41 +46,20 @@ func (i XMLInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { case "from", "fromF": for _, a := range se.Attr { if a.Name.Local == "uri" { - uris = append(uris, a.Value) + meta.FromURIs = append(meta.FromURIs, a.Value) } } - } - } - } - - return uris, nil -} - -// ToURIs -- -func (i XMLInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - content := strings.NewReader(source.Content) - decoder := xml.NewDecoder(content) - - uris := make([]string, 0) - - for { - // Read tokens from the XML document in a stream. - t, _ := decoder.Token() - if t == nil { - break - } - - if se, ok := t.(xml.StartElement); ok { - switch se.Name.Local { case "to", "toD", "toF": for _, a := range se.Attr { if a.Name.Local == "uri" { - uris = append(uris, a.Value) + meta.ToURIs = append(meta.ToURIs, a.Value) } } } } } - return uris, nil + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/inspector_yaml_flow.go b/pkg/util/source/inspector_yaml_flow.go index e677b4de..89475fcd 100644 --- a/pkg/util/source/inspector_yaml_flow.go +++ b/pkg/util/source/inspector_yaml_flow.go @@ -19,49 +19,35 @@ package source import ( "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2" ) // YAMLFlowInspector -- type YAMLFlowInspector struct { + baseInspector } -// FromURIs -- -func (i YAMLFlowInspector) FromURIs(source v1alpha1.SourceSpec) ([]string, error) { +// Extract -- +func (i YAMLFlowInspector) Extract(source v1alpha1.SourceSpec, meta *Metadata) error { var flows []v1alpha1.Flow if err := yaml.Unmarshal([]byte(source.Content), &flows); err != nil { - return []string{}, nil + return nil } - uris := make([]string, 0) - for _, flow := range flows { if flow.Steps[0].URI != "" { - uris = append(uris, flow.Steps[0].URI) + meta.FromURIs = append(meta.FromURIs, flow.Steps[0].URI) } - } - return uris, nil -} - -// ToURIs -- -func (i YAMLFlowInspector) ToURIs(source v1alpha1.SourceSpec) ([]string, error) { - var flows []v1alpha1.Flow - - if err := yaml.Unmarshal([]byte(source.Content), &flows); err != nil { - return []string{}, nil - } - - uris := make([]string, 0) - - for _, flow := range flows { for i := 1; i < len(flow.Steps); i++ { if flow.Steps[i].URI != "" { - uris = append(uris, flow.Steps[i].URI) + meta.ToURIs = append(meta.ToURIs, flow.Steps[i].URI) } } } - return uris, nil + meta.Dependencies = i.discoverDependencies(source, meta) + + return nil } diff --git a/pkg/util/source/types.go b/pkg/util/source/types.go new file mode 100644 index 00000000..1b89b12f --- /dev/null +++ b/pkg/util/source/types.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package source + +// Metadata -- +type Metadata struct { + // All starting URIs of defined routes + FromURIs []string + // All end URIs of defined routes + ToURIs []string + // All inferred dependencies required to run the integration + Dependencies []string +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 7cd9712b..150b1ab2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -21,13 +21,33 @@ import ( "os" "os/signal" "path" + "regexp" "syscall" + "github.com/scylladb/go-set/strset" + "k8s.io/api/core/v1" "github.com/pkg/errors" ) +// StringSliceJoin -- +func StringSliceJoin(slices ...[]string) []string { + size := 0 + + for _, s := range slices { + size += len(s) + } + + result := make([]string, 0, size) + + for _, s := range slices { + result = append(result, s...) + } + + return result +} + // StringSliceContains -- func StringSliceContains(slice []string, items []string) bool { for i := 0; i < len(items); i++ { @@ -116,3 +136,20 @@ func LookupEnvVar(vars []v1.EnvVar, name string) *v1.EnvVar { return nil } + +// FindAllDistinctStringSubmatch .. +func FindAllDistinctStringSubmatch(data string, regexps ...*regexp.Regexp) []string { + submatchs := strset.New() + + for _, reg := range regexps { + hits := reg.FindAllStringSubmatch(data, -1) + for _, hit := range hits { + if len(hit) > 1 { + for _, match := range hit[1:] { + submatchs.Add(match) + } + } + } + } + return submatchs.List() +}
With regards, Apache Git Services
