This is an automated email from the ASF dual-hosted git repository.
nferraro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
commit 2c51dfd0d359885a8b298b35d77e50fbf0fb200f
Author: nferraro <ni.ferr...@gmail.com>
AuthorDate: Fri Sep 21 13:26:20 2018 +0200
Added incremental publisher
---
pkg/build/publish/s2i_incremental_publisher.go | 105 +++++++++++++++++++++++++
pkg/build/publish/s2i_publisher.go | 43 +++++++---
2 files changed, 137 insertions(+), 11 deletions(-)
diff --git a/pkg/build/publish/s2i_incremental_publisher.go
b/pkg/build/publish/s2i_incremental_publisher.go
new file mode 100644
index 0000000..42d7c81
--- /dev/null
+++ b/pkg/build/publish/s2i_incremental_publisher.go
@@ -0,0 +1,105 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package publish
+
+import (
+ "context"
+ "github.com/apache/camel-k/pkg/build"
+)
+
+type s2iIncrementalPublisher struct {
+ s2iPublisher *s2iPublisher
+ lister PublishedImagesLister
+}
+
+// PublishedImage represent a base image that can be used as starting point
+type PublishedImage struct {
+ Image string
+ Classpath []string
+}
+
+// PublishedImagesLister allows to list all images already published
+type PublishedImagesLister interface {
+ ListPublishedImages() ([]PublishedImage, error)
+}
+
+// NewS2IIncrementalPublisher creates a new publisher that is able to do a
Openshift S2I binary builds on top of other builds
+func NewS2IIncrementalPublisher(ctx context.Context, namespace string, lister
PublishedImagesLister) build.Publisher {
+ layeredPublisher := s2iIncrementalPublisher{
+ lister: lister,
+ }
+ layeredPublisher.s2iPublisher = newS2IPublisher(ctx, namespace,
layeredPublisher.selectArtifactsToUpload)
+ return &layeredPublisher
+}
+
+func (p *s2iIncrementalPublisher) Publish(req build.Request, assembled
build.AssembledOutput) <-chan build.PublishedOutput {
+ return p.s2iPublisher.Publish(req, assembled)
+}
+
+func (p *s2iIncrementalPublisher) selectArtifactsToUpload(entries
[]build.ClasspathEntry) (string, []build.ClasspathEntry, error) {
+ images, err := p.lister.ListPublishedImages()
+ if err != nil {
+ return "", nil, err
+ }
+
+ bestImage, commonLibs := p.findBestImage(images, entries)
+ if (bestImage != nil) {
+ selectedClasspath := make([]build.ClasspathEntry, 0)
+ for _, entry := range entries {
+ if _, isCommon := commonLibs[entry.ID]; !isCommon {
+ selectedClasspath = append(selectedClasspath,
entry)
+ }
+ }
+
+ return bestImage.Image, selectedClasspath, nil
+ }
+
+ // return default selection
+ return baseImage, entries, nil
+}
+
+func (p *s2iIncrementalPublisher) findBestImage(images []PublishedImage,
entries []build.ClasspathEntry) (*PublishedImage, map[string]bool) {
+ requiredLibs := make(map[string]bool, len(entries))
+ for _, entry := range entries {
+ requiredLibs[entry.ID] = true
+ }
+
+ var bestImage *PublishedImage
+ bestImageCommonLibs := make(map[string]bool, 0)
+ for _, image := range images {
+ common := make(map[string]bool)
+ for _, id := range image.Classpath {
+ if _, ok := requiredLibs[id]; ok {
+ common[id] = true
+ }
+ }
+ numCommonLibs := len(common)
+ surplus := len(image.Classpath) - numCommonLibs
+ if surplus >= numCommonLibs/3 {
+ // Heuristic approach: if there are too many unrelated
libraries, just use the base image
+ continue
+ }
+
+ if (numCommonLibs > len(bestImageCommonLibs)) {
+ bestImage = &image
+ bestImageCommonLibs = common
+ }
+ }
+
+ return bestImage, bestImageCommonLibs
+}
diff --git a/pkg/build/publish/s2i_publisher.go
b/pkg/build/publish/s2i_publisher.go
index 5548194..ee86a1e 100644
--- a/pkg/build/publish/s2i_publisher.go
+++ b/pkg/build/publish/s2i_publisher.go
@@ -40,11 +40,13 @@ import (
const (
artifactDirPrefix = "s2i-"
+ baseImage = "fabric8/s2i-java:2.3"
)
type s2iPublisher struct {
buffer chan publishOperation
namespace string
+ uploadedArtifactsSelector
}
type publishOperation struct {
@@ -53,11 +55,20 @@ type publishOperation struct {
output chan build.PublishedOutput
}
+type uploadedArtifactsSelector func([]build.ClasspathEntry) (string,
[]build.ClasspathEntry, error)
+
// NewS2IPublisher creates a new publisher doing a Openshift S2I binary build
func NewS2IPublisher(ctx context.Context, namespace string) build.Publisher {
+ identitySelector := func(entries []build.ClasspathEntry) (string,
[]build.ClasspathEntry, error) { return baseImage, entries, nil }
+ return newS2IPublisher(ctx, namespace, identitySelector)
+}
+
+// NewS2IPublisher creates a new publisher doing a Openshift S2I binary build
+func newS2IPublisher(ctx context.Context, namespace string,
uploadedArtifactsSelector uploadedArtifactsSelector) *s2iPublisher {
publisher := s2iPublisher{
- buffer: make(chan publishOperation, 100),
- namespace: namespace,
+ buffer: make(chan publishOperation, 100),
+ namespace: namespace,
+ uploadedArtifactsSelector: uploadedArtifactsSelector,
}
go publisher.publishCycle(ctx)
return &publisher
@@ -98,12 +109,17 @@ func (b *s2iPublisher) publishCycle(ctx context.Context) {
}
func (b *s2iPublisher) execute(request build.Request, assembled
build.AssembledOutput) build.PublishedOutput {
- tarFile, err := b.createTar(assembled)
+ baseImageName, selectedArtifacts, err :=
b.uploadedArtifactsSelector(assembled.Classpath)
if err != nil {
return build.PublishedOutput{Error: err}
}
- image, err := b.publish(tarFile, request)
+ tarFile, err := b.createTar(assembled, selectedArtifacts)
+ if err != nil {
+ return build.PublishedOutput{Error: err}
+ }
+
+ image, err := b.publish(tarFile, baseImageName, request)
if err != nil {
return build.PublishedOutput{Error: err}
}
@@ -111,7 +127,7 @@ func (b *s2iPublisher) execute(request build.Request,
assembled build.AssembledO
return build.PublishedOutput{Image: image}
}
-func (b *s2iPublisher) publish(tarFile string, source build.Request) (string,
error) {
+func (b *s2iPublisher) publish(tarFile string, imageName string, source
build.Request) (string, error) {
bc := buildv1.BuildConfig{
TypeMeta: metav1.TypeMeta{
@@ -131,7 +147,7 @@ func (b *s2iPublisher) publish(tarFile string, source
build.Request) (string, er
SourceStrategy:
&buildv1.SourceBuildStrategy{
From: v1.ObjectReference{
Kind: "DockerImage",
- Name:
"fabric8/s2i-java:2.3",
+ Name: imageName,
},
},
},
@@ -235,7 +251,7 @@ func (b *s2iPublisher) publish(tarFile string, source
build.Request) (string, er
return is.Status.DockerImageRepository + ":" +
source.Identifier.Qualifier, nil
}
-func (b *s2iPublisher) createTar(assembled build.AssembledOutput) (string,
error) {
+func (b *s2iPublisher) createTar(assembled build.AssembledOutput,
selectedArtifacts []build.ClasspathEntry) (string, error) {
artifactDir, err := ioutil.TempDir("", artifactDirPrefix)
if err != nil {
return "", errors.Wrap(err, "could not create temporary dir for
s2i artifacts")
@@ -248,19 +264,24 @@ func (b *s2iPublisher) createTar(assembled
build.AssembledOutput) (string, error
}
defer tarAppender.Close()
- cp := ""
- for _, entry := range assembled.Classpath {
+ tarDir := "dependencies/"
+ for _, entry := range selectedArtifacts {
gav, err := maven.ParseGAV(entry.ID)
if err != nil {
return "", nil
}
- tarPath := path.Join("dependencies/", gav.GroupID)
- fileName, err := tarAppender.AddFile(entry.Location, tarPath)
+ tarPath := path.Join(tarDir, gav.GroupID)
+ _, err = tarAppender.AddFile(entry.Location, tarPath)
if err != nil {
return "", err
}
+ }
+ cp := ""
+ for _, entry := range assembled.Classpath {
+ _, fileName := path.Split(entry.Location)
+ fileName = path.Join(tarDir, fileName)
cp += fileName + "\n"
}
