This is an automated email from the ASF dual-hosted git repository.
nferraro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 6234cca5ea022890a142013791ef62ff30e18d36
Author: lburgazzoli <lburgazz...@gmail.com>
AuthorDate: Thu Sep 20 01:37:17 2018 +0200
maven: use maven like structure to store artifacts at runtime and pre
computed classpath file
---
deploy/platform-integration-context-core.yaml | 7 +-
deploy/resources.go | 5 +-
pkg/apis/camel/v1alpha1/types.go | 1 +
pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go | 5 +
pkg/build/build_manager.go | 6 +-
pkg/build/build_types.go | 12 +--
pkg/build/local/local_builder.go | 72 +++++++-------
pkg/build/local/local_builder_test.go | 10 +-
pkg/discover/dependencies_test.go | 13 +--
pkg/discover/languages_test.go | 7 +-
pkg/stub/action/context/build.go | 1 +
pkg/stub/action/integration/deploy.go | 18 ++++
pkg/util/maven/maven.go | 121 ++++++++++++++++-------
pkg/util/maven/{types.go => maven_project.go} | 0
14 files changed, 185 insertions(+), 93 deletions(-)
diff --git a/deploy/platform-integration-context-core.yaml
b/deploy/platform-integration-context-core.yaml
index 836d195..3fe794c 100644
--- a/deploy/platform-integration-context-core.yaml
+++ b/deploy/platform-integration-context-core.yaml
@@ -1,9 +1,12 @@
apiVersion: camel.apache.org/v1alpha1
kind: IntegrationContext
metadata:
- name: root.integrationcontexts.camel.apache.org
+ name: core.integrationcontexts.camel.apache.org
labels:
app: "camel-k"
camel.apache.org/context.created.by.kind: Operator
camel.apache.org/context.created.by.name: core
- camel.apache.org/context.type: platform
\ No newline at end of file
+ camel.apache.org/context.type: platform
+spec:
+ dependencies:
+ - camel:core
\ No newline at end of file
diff --git a/deploy/resources.go b/deploy/resources.go
index e9525d6..ec79a89 100644
--- a/deploy/resources.go
+++ b/deploy/resources.go
@@ -2441,12 +2441,15 @@ status:
apiVersion: camel.apache.org/v1alpha1
kind: IntegrationContext
metadata:
- name: root.integrationcontexts.camel.apache.org
+ name: core.integrationcontexts.camel.apache.org
labels:
app: "camel-k"
camel.apache.org/context.created.by.kind: Operator
camel.apache.org/context.created.by.name: core
camel.apache.org/context.type: platform
+spec:
+ dependencies:
+ - camel:core
`
Resources["platform-integration-context-groovy.yaml"] =
`
diff --git a/pkg/apis/camel/v1alpha1/types.go b/pkg/apis/camel/v1alpha1/types.go
index 3a91be6..ee89870 100644
--- a/pkg/apis/camel/v1alpha1/types.go
+++ b/pkg/apis/camel/v1alpha1/types.go
@@ -125,6 +125,7 @@ type IntegrationContext struct {
// IntegrationContextSpec --
type IntegrationContextSpec struct {
Dependencies []string `json:"dependencies,omitempty"`
+ Classpath []string `json:"classpath,omitempty"`
Configuration []ConfigurationSpec `json:"configuration,omitempty"`
}
diff --git a/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go
b/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go
index 9feb415..dfd91cb 100644
--- a/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/camel/v1alpha1/zz_generated.deepcopy.go
@@ -138,6 +138,11 @@ func (in *IntegrationContextSpec) DeepCopyInto(out
*IntegrationContextSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
+ if in.Classpath != nil {
+ in, out := &in.Classpath, &out.Classpath
+ *out = make([]string, len(*in))
+ copy(*out, *in)
+ }
if in.Configuration != nil {
in, out := &in.Configuration, &out.Configuration
*out = make([]ConfigurationSpec, len(*in))
diff --git a/pkg/build/build_manager.go b/pkg/build/build_manager.go
index 325e995..50d1187 100644
--- a/pkg/build/build_manager.go
+++ b/pkg/build/build_manager.go
@@ -57,7 +57,7 @@ func (m *Manager) Start(source Request) {
resChannel := m.builder.Build(source)
go func() {
res := <-resChannel
- m.builds.Store(res.Source.Identifier, &res)
+ m.builds.Store(res.Request.Identifier, &res)
}()
}
@@ -69,7 +69,7 @@ func noBuildInfo() Result {
func initialBuildInfo(source *Request) Result {
return Result{
- Source: source,
- Status: StatusStarted,
+ Request: source,
+ Status: StatusStarted,
}
}
diff --git a/pkg/build/build_types.go b/pkg/build/build_types.go
index a552aa1..0fd7309 100644
--- a/pkg/build/build_types.go
+++ b/pkg/build/build_types.go
@@ -30,7 +30,7 @@ type Identifier struct {
Qualifier string
}
-// Source represent the integration code
+// Request represent the integration code
type Source struct {
Name string
Content string
@@ -39,11 +39,11 @@ type Source struct {
// Result represents the result of a build
type Result struct {
- Source *Request
- Status Status
- Image string
- Error error
- ResolvedDependencies []string
+ Request *Request
+ Status Status
+ Image string
+ Error error
+ Classpath []string
}
// Builder is supertype of all builders
diff --git a/pkg/build/local/local_builder.go b/pkg/build/local/local_builder.go
index 65e035f..2bb59ce 100644
--- a/pkg/build/local/local_builder.go
+++ b/pkg/build/local/local_builder.go
@@ -51,8 +51,8 @@ type localBuilder struct {
}
type buildOperation struct {
- source build.Request
- output chan build.Result
+ request build.Request
+ output chan build.Result
}
// NewLocalBuilder create a new builder
@@ -65,11 +65,11 @@ func NewLocalBuilder(ctx context.Context, namespace string)
build.Builder {
return &builder
}
-func (b *localBuilder) Build(source build.Request) <-chan build.Result {
+func (b *localBuilder) Build(request build.Request) <-chan build.Result {
res := make(chan build.Result, 1)
op := buildOperation{
- source: source,
- output: res,
+ request: request,
+ output: res,
}
b.buffer <- op
return res
@@ -84,53 +84,57 @@ func (b *localBuilder) buildCycle(ctx context.Context) {
case op := <-b.buffer:
now := time.Now()
logrus.Info("Starting new build")
- image, err := b.execute(op.source)
+ res := b.execute(&op.request)
elapsed := time.Now().Sub(now)
- if err != nil {
- logrus.Error("Error during build (total time ",
elapsed.Seconds(), " seconds): ", err)
- } else {
- logrus.Info("Build completed in ",
elapsed.Seconds(), " seconds")
- }
- if err != nil {
- op.output <- build.Result{
- Source: &op.source,
- Status: build.StatusError,
- Error: err,
- }
+ if res.Error != nil {
+ logrus.Error("Error during build (total time ",
elapsed.Seconds(), " seconds): ", res.Error)
} else {
- op.output <- build.Result{
- Source: &op.source,
- Status: build.StatusCompleted,
- Image: image,
- }
+ logrus.Info("Build completed in ",
elapsed.Seconds(), " seconds")
}
+ op.output <- res
}
}
}
-func (b *localBuilder) execute(source build.Request) (string, error) {
- project, err := generateProject(source)
+func (b *localBuilder) execute(request *build.Request) build.Result {
+ project, err := generateProject(request)
if err != nil {
- return "", err
+ return build.Result{
+ Error: err,
+ Status: build.StatusError,
+ }
}
- tarFileName, err := maven.Build(project)
+
+ res, err := maven.Build(project)
if err != nil {
- return "", err
+ return build.Result{
+ Error: err,
+ Status: build.StatusError,
+ }
}
- logrus.Info("Created tar file ", tarFileName)
+ logrus.Info("Created tar file ", res.TarFilePath)
- image, err := b.publish(tarFileName, source)
+ image, err := b.publish(res.TarFilePath, request)
if err != nil {
- return "", errors.Wrap(err, "could not publish docker image")
+ return build.Result{
+ Error: errors.Wrap(err, "could not publish docker
image"),
+ Status: build.StatusError,
+ }
}
- return image, nil
+ return build.Result{
+ Request: request,
+ Image: image,
+ Error: nil,
+ Status: build.StatusCompleted,
+ Classpath: res.Classpath,
+ }
}
-func (b *localBuilder) publish(tarFile string, source build.Request) (string,
error) {
+func (b *localBuilder) publish(tarFile string, source *build.Request) (string,
error) {
bc := buildv1.BuildConfig{
TypeMeta: metav1.TypeMeta{
APIVersion: buildv1.SchemeGroupVersion.String(),
@@ -253,7 +257,7 @@ func (b *localBuilder) publish(tarFile string, source
build.Request) (string, er
return is.Status.DockerImageRepository + ":" +
source.Identifier.Qualifier, nil
}
-func generateProject(source build.Request) (maven.Project, error) {
+func generateProject(source *build.Request) (maven.Project, error) {
project := maven.Project{
XMLName: xml.Name{Local: "project"},
XMLNs: "http://maven.apache.org/POM/4.0.0";,
@@ -267,7 +271,7 @@ func generateProject(source build.Request) (maven.Project,
error) {
Dependencies: maven.Dependencies{
Dependencies: []maven.Dependency{
{
- //TODO: camel version should be
retrieved from an external source or provided as static version
+ //TODO: camel version should be
retrieved from an external request or provided as static version
GroupID: "org.apache.camel",
ArtifactID: "camel-bom",
Version: "2.22.1",
diff --git a/pkg/build/local/local_builder_test.go
b/pkg/build/local/local_builder_test.go
index e6ec79b..ddb928e 100644
--- a/pkg/build/local/local_builder_test.go
+++ b/pkg/build/local/local_builder_test.go
@@ -25,7 +25,7 @@ import (
)
func TestProjectGeneration(t *testing.T) {
- source := build.Request{
+ request := build.Request{
Identifier: build.Identifier{
Name: "my-integration",
Qualifier: "",
@@ -43,7 +43,8 @@ func TestProjectGeneration(t *testing.T) {
},
}
- prj, err := generateProject(source)
+ prj, err := generateProject(&request)
+
assert.Nil(t, err)
assert.NotNil(t, prj)
assert.Equal(t, len(prj.Dependencies.Dependencies), 5)
@@ -57,7 +58,7 @@ func TestProjectGeneration(t *testing.T) {
}
func TestProjectGenerationWithFailure(t *testing.T) {
- source := build.Request{
+ request := build.Request{
Identifier: build.Identifier{
Name: "my-integration",
Qualifier: "",
@@ -75,6 +76,7 @@ func TestProjectGenerationWithFailure(t *testing.T) {
},
}
- _, err := generateProject(source)
+ _, err := generateProject(&request)
+
assert.NotNil(t, err)
}
diff --git a/pkg/discover/dependencies_test.go
b/pkg/discover/dependencies_test.go
index ca5fff2..8e48cb1 100644
--- a/pkg/discover/dependencies_test.go
+++ b/pkg/discover/dependencies_test.go
@@ -18,14 +18,15 @@ limitations under the License.
package discover
import (
+ "testing"
+
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
"github.com/stretchr/testify/assert"
- "testing"
)
func TestDependenciesJavaSource(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "Source.java",
+ Name: "Request.java",
Language: v1alpha1.LanguageJavaSource,
Content: `
from("telegram:bots/cippa").to("log:stash");
@@ -40,7 +41,7 @@ func TestDependenciesJavaSource(t *testing.T) {
func TestDependenciesJavaClass(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "Source.class",
+ Name: "Request.class",
Language: v1alpha1.LanguageJavaClass,
Content: `
from("telegram:bots/cippa").to("log:stash");
@@ -54,7 +55,7 @@ func TestDependenciesJavaClass(t *testing.T) {
func TestDependenciesJavaScript(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "source.js",
+ Name: "source.js",
Language: v1alpha1.LanguageJavaScript,
Content: `
from('telegram:bots/cippa').to("log:stash");
@@ -70,7 +71,7 @@ func TestDependenciesJavaScript(t *testing.T) {
func TestDependenciesGroovy(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "source.groovy",
+ Name: "source.groovy",
Language: v1alpha1.LanguageGroovy,
Content: `
from('telegram:bots/cippa').to("log:stash");
@@ -82,4 +83,4 @@ func TestDependenciesGroovy(t *testing.T) {
dependencies := Dependencies(code)
// assert all dependencies are found and sorted (removing duplicates)
assert.Equal(t, []string{"camel:amqp", "camel:core", "camel:telegram"},
dependencies)
-}
\ No newline at end of file
+}
diff --git a/pkg/discover/languages_test.go b/pkg/discover/languages_test.go
index 919de9d..bbe0c45 100644
--- a/pkg/discover/languages_test.go
+++ b/pkg/discover/languages_test.go
@@ -18,14 +18,15 @@ limitations under the License.
package discover
import (
+ "testing"
+
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
"github.com/stretchr/testify/assert"
- "testing"
)
func TestLanguageJavaSource(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "Source.java",
+ Name: "Request.java",
}
language := Language(code)
assert.Equal(t, v1alpha1.LanguageJavaSource, language)
@@ -33,7 +34,7 @@ func TestLanguageJavaSource(t *testing.T) {
func TestLanguageAlreadySet(t *testing.T) {
code := v1alpha1.SourceSpec{
- Name: "Source.java",
+ Name: "Request.java",
Language: v1alpha1.LanguageJavaScript,
}
language := Language(code)
diff --git a/pkg/stub/action/context/build.go b/pkg/stub/action/context/build.go
index e8d26e4..6d06b22 100644
--- a/pkg/stub/action/context/build.go
+++ b/pkg/stub/action/context/build.go
@@ -73,6 +73,7 @@ func (action *integrationContextBuildAction) Handle(context
*v1alpha1.Integratio
target := context.DeepCopy()
target.Status.Image = buildResult.Image
target.Status.Phase = v1alpha1.IntegrationContextPhaseReady
+ target.Spec.Classpath = buildResult.Classpath
if err := sdk.Update(target); err != nil {
return err
}
diff --git a/pkg/stub/action/integration/deploy.go
b/pkg/stub/action/integration/deploy.go
index 8d873ea..c6ba592 100644
--- a/pkg/stub/action/integration/deploy.go
+++ b/pkg/stub/action/integration/deploy.go
@@ -19,8 +19,11 @@ package action
import (
"fmt"
+ "os"
"strings"
+ "github.com/apache/camel-k/version"
+
"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
"github.com/operator-framework/operator-sdk/pkg/sdk"
"github.com/pkg/errors"
@@ -74,6 +77,12 @@ func getConfigMapFor(ctx *v1alpha1.IntegrationContext,
integration *v1alpha1.Int
// combine properties of integration with context, integration
// properties have the priority
properties := CombineConfigurationAsMap("property", ctx, integration)
+ classpath := make([]string, 0, len(ctx.Spec.Classpath))
+
+ //TODO: we need some constants
+ for _, path := range ctx.Spec.Classpath {
+ classpath = append(classpath, strings.Replace(path,
"/tmp/artifacts/m2", "/deployments/m2", 1))
+ }
cm := corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
@@ -102,6 +111,7 @@ func getConfigMapFor(ctx *v1alpha1.IntegrationContext,
integration *v1alpha1.Int
Data: map[string]string{
"integration": integration.Spec.Source.Content,
"properties": PropertiesString(properties),
+ "classpath": strings.Join(classpath,
string(os.PathListSeparator)),
},
}
@@ -142,6 +152,11 @@ func getDeploymentFor(ctx *v1alpha1.IntegrationContext,
integration *v1alpha1.In
// set env vars needed by the runtime
environment["JAVA_MAIN_CLASS"] = "org.apache.camel.k.jvm.Application"
+ environment["JAVA_LIB_DIR"] = "/etc/camel/conf"
+
+ //TODO: remove this hack !!! s2i java (/opt/run-java/run-java.sh) fails
if JAVA_APP_JAR is not set
+ environment["JAVA_APP_JAR"] =
fmt.Sprintf("/deployments/m2/org/apache/camel/k/camel-k-runtime-jvm/%s/camel-k-runtime-jvm-%s.jar",
version.Version, version.Version)
+
environment["CAMEL_K_ROUTES_URI"] = "file:/etc/camel/conf/" + sourceName
environment["CAMEL_K_ROUTES_LANGUAGE"] =
string(integration.Spec.Source.Language)
environment["CAMEL_K_CONF"] = "/etc/camel/conf/application.properties"
@@ -226,6 +241,9 @@ func getDeploymentFor(ctx *v1alpha1.IntegrationContext,
integration *v1alpha1.In
}, {
Key: "properties",
Path: "application.properties",
+ }, {
+ Key: "classpath",
+ Path: "classpath",
},
},
},
diff --git a/pkg/util/maven/maven.go b/pkg/util/maven/maven.go
index 1a919d2..8f67982 100644
--- a/pkg/util/maven/maven.go
+++ b/pkg/util/maven/maven.go
@@ -19,6 +19,7 @@ package maven
import (
"archive/tar"
+ "bufio"
"bytes"
"encoding/xml"
"io"
@@ -38,38 +39,63 @@ const (
artifactDirPrefix = "maven-bin-"
)
+// BuildResult --
+type BuildResult struct {
+ TarFilePath string
+ Classpath []string
+}
+
// Build takes a project description and returns a binary tar with the built
artifacts
-func Build(project Project) (string, error) {
+func Build(project Project) (BuildResult, error) {
+ res := BuildResult{}
buildDir, err := ioutil.TempDir("", buildDirPrefix)
if err != nil {
- return "", errors.Wrap(err, "could not create temporary dir for
maven source files")
+ return res, errors.Wrap(err, "could not create temporary dir
for maven source files")
}
+
defer os.RemoveAll(buildDir)
err = createMavenStructure(buildDir, project)
if err != nil {
- return "", errors.Wrap(err, "could not write maven source
files")
+ return res, errors.Wrap(err, "could not write maven source
files")
}
- err = runMavenBuild(buildDir)
+ err = runMavenBuild(buildDir, &res)
if err != nil {
- return "", err
+ return res, err
}
- tarfile, err := createTar(buildDir, project)
+
+ res.TarFilePath, err = createTar(project, &res)
if err != nil {
- return "", err
+ return res, err
}
- return tarfile, nil
+
+ return res, nil
}
-func runMavenBuild(buildDir string) error {
- copyDepsCmd := exec.Command("mvn", mavenExtraOptions(), "clean",
"dependency:copy-dependencies")
- copyDepsCmd.Dir = buildDir
- copyDepsCmd.Stdout = os.Stdout
- copyDepsCmd.Stderr = os.Stderr
+func runMavenBuild(buildDir string, result *BuildResult) error {
+ // file where the classpath is listed
+ out := path.Join(buildDir, "integration.classpath")
+
+ cmd := exec.Command("mvn", mavenExtraOptions(),
"-Dmdep.outputFile="+out, "dependency:build-classpath")
+ cmd.Dir = buildDir
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+
+ logrus.Infof("determine classpath: mvn: %v", cmd.Args)
+ if err := cmd.Run(); err != nil {
+ return errors.Wrap(err, "failure while determining classpath")
+ }
- logrus.Infof("Copying maven dependencies: mvn %v", copyDepsCmd.Args)
- if err := copyDepsCmd.Run(); err != nil {
- return errors.Wrap(err, "failure while extracting maven
dependencies")
+ lines, err := readLines(out)
+ if err != nil {
+ return err
+ }
+
+ result.Classpath = make([]string, 0)
+ for _, line := range lines {
+ for _, item := range strings.Split(line,
string(os.PathListSeparator)) {
+ result.Classpath = append(result.Classpath, item)
+ }
}
logrus.Info("Maven build completed successfully")
@@ -83,7 +109,7 @@ func mavenExtraOptions() string {
return "-Dcamel.noop=true"
}
-func createTar(buildDir string, project Project) (string, error) {
+func createTar(project Project, result *BuildResult) (string, error) {
artifactDir, err := ioutil.TempDir("", artifactDirPrefix)
if err != nil {
return "", errors.Wrap(err, "could not create temporary dir for
maven artifacts")
@@ -97,22 +123,15 @@ func createTar(buildDir string, project Project) (string,
error) {
defer tarFile.Close()
writer := tar.NewWriter(tarFile)
+ defer writer.Close()
- dependenciesDir := path.Join(buildDir, "target", "dependency")
- dependencies, err := ioutil.ReadDir(dependenciesDir)
- if err != nil {
- return "", err
- }
-
- for _, dep := range dependencies {
- err = appendToTar(path.Join(dependenciesDir, dep.Name()), "",
writer)
+ for _, path := range result.Classpath {
+ err = appendToTarWithPath(path, writer)
if err != nil {
return "", err
}
}
- writer.Close()
-
return tarFileName, nil
}
@@ -146,6 +165,35 @@ func appendToTar(filePath string, tarPath string, writer
*tar.Writer) error {
return nil
}
+func appendToTarWithPath(path string, writer *tar.Writer) error {
+ info, err := os.Stat(path)
+ if err != nil {
+ return err
+ }
+
+ //TODO: we ned some constants
+ relocatedPath := strings.TrimPrefix(path, "/tmp/artifacts")
+
+ writer.WriteHeader(&tar.Header{
+ Name: relocatedPath,
+ Size: info.Size(),
+ Mode: int64(info.Mode()),
+ ModTime: info.ModTime(),
+ })
+
+ file, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ _, err = io.Copy(writer, file)
+ if err != nil {
+ return errors.Wrap(err, "cannot add file to the tar archive")
+ }
+ return nil
+}
+
func createMavenStructure(buildDir string, project Project) error {
pom, err := GeneratePomFileContent(project)
if err != nil {
@@ -182,15 +230,20 @@ func writeFile(buildDir string, relativePath string,
content string) error {
return nil
}
-func envFileContent(env map[string]string) string {
- if env == nil {
- return ""
+//TODO: move to a file utility package
+func readLines(path string) ([]string, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return nil, err
}
- content := ""
- for k, v := range env {
- content = content + "export " + k + "=" + v + "\n"
+ defer file.Close()
+
+ var lines []string
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ lines = append(lines, scanner.Text())
}
- return content
+ return lines, scanner.Err()
}
// GeneratePomFileContent generate a pom.xml file from the given project
definition
diff --git a/pkg/util/maven/types.go b/pkg/util/maven/maven_project.go
similarity index 100%
rename from pkg/util/maven/types.go
rename to pkg/util/maven/maven_project.go
