squakez commented on a change in pull request #2577: URL: https://github.com/apache/camel-k/pull/2577#discussion_r699021504
########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,124 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/tls" + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "my-combined-tls" + keyName = "my-key" + certName = "my-cert" + keyFilePath = "files/key.key" + certFilePath = "files/crt.crt" +) + +func TestRunRouteTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + createSecrets(ns) + + refKey := secretName + "/" + keyName + refCert := secretName + "/" + certName + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/NettyServer.java", + "-t", "route.enabled=true", + "-t", "route.tls-termination=edge", + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + ).Execute()).To(Succeed()) + Eventually(IntegrationPodPhase(ns, "netty-server"), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route https works", func(t *testing.T) { + route := Route(ns, "netty-server") + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request + time.Sleep(3 * time.Second) + response := httpsRequest(t, fmt.Sprintf("https://%s/hello";, route().Spec.Host)) + assert.Equal(t, "Hello World", response) + }) + // Cleanup + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + // }) + }) +} + +func httpsRequest(t *testing.T, url string) string { + transCfg := &http.Transport{ + // ignore self signed SSL certificates + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: transCfg} + response, err := client.Get(url) + defer func() { + if response != nil { + _ = response.Body.Close() + } + }() + assert.Nil(t, err) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(response.Body) + assert.Nil(t, err) + return buf.String() +} + +func createSecrets(ns string) error { Review comment: Yeah, I think it would be cleaner to have a `func NewBinarySecret(ns string, name string, data map[string][]byte) error` method which generically takes in a binary content. Then you use it from your private method with the certificates created for the purpose. ########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,444 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + rand2 "math/rand" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "test-certificate" + servingCertificateSecret = "serving-certificate" + integrationName = "platform-http-server" +) + +type keyCertificatePair struct { + Key []byte + Certificate []byte +} + +var certPem []byte + +// this test uses a route with no TLS +func TestRunRouteNoTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) Review comment: In order reduce the execution time, I think it would be wiser to run the test into the same method. You will only install the operator once and reuse the same kit along all tests. I used the approach in the [user config test](https://github.com/apache/camel-k/blob/main/e2e/common/config/config_test.go). ########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,444 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + rand2 "math/rand" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "test-certificate" + servingCertificateSecret = "serving-certificate" + integrationName = "platform-http-server" +) + +type keyCertificatePair struct { + Key []byte + Certificate []byte +} + +var certPem []byte + +// this test uses a route with no TLS +func TestRunRouteNoTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java").Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route unsecure http works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before doing an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(4 * time.Second) + url := fmt.Sprintf("http://%s/hello?name=Simple";, route().Spec.Host) + response, err := httpRequest(t, url, false) + assert.Nil(t, err) + assert.Equal(t, "Hello Simple", response) + }) + }) +} + +func TestRunRouteTLS_Edge(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + // delete secret when the test finishes + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret/certificates previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + "-t", "route.tls-termination=edge", + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + ).Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Edge https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + url := fmt.Sprintf("https://%s/hello?name=TLS_Edge";, route().Spec.Host) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello TLS_Edge", response) + }) + }) +} + +func TestRunRouteTLS_Passthrough(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=passthrough", + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route passthrough https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Passthrough" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a self-signed certificate to create a TLS route +func TestRunRouteTLS_Reencrypt(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // the destination CA certificate which the route service uses to validate the HTTP endpoint TLS certificate + "-t", "route.tls-destination-ca-certificate-secret=" + refCert, + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Reencrypt https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Reencrypt" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a certificate provided by openshift "service serving certificates" to create a TLS route +func TestRunRouteTLS_ReencryptWithServiceCA(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + // they are used only for the pod exposing the HTTP endpoint, not the router + "--resource", "secret:" + servingCertificateSecret + "@/etc/ssl/" + servingCertificateSecret, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + // these certificates are used only in the pod, not the router + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + servingCertificateSecret + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + servingCertificateSecret + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // these certificates are used only in the router to encrypt the connection from the client to the router + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + time.Sleep(2 * time.Second) + Eventually(Service(ns, integrationName)()).ShouldNot(BeNil()) + annotations := make(map[string]string) + annotations["service.beta.openshift.io/serving-cert-secret-name"] = servingCertificateSecret + err = addAnnotation(ns, integrationName, annotations) + assert.Nil(t, err) + + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + t.Run("Route reencrypt (with openshift service CA) https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_ReencryptWithServiceCAWithServiceCA" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +func httpRequest(t *testing.T, url string, tlsEnabled bool) (string, error) { + var client http.Client + if tlsEnabled { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certPem) + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config { + RootCAs: certPool, + }, + } + client = http.Client{Transport: transCfg} + } else { + client = http.Client{} + } + response, err := client.Get(url) + defer func() { + if response != nil { + _ = response.Body.Close() + } + }() + if err != nil { + return "", err + } + assert.Nil(t, err) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(response.Body) + if err != nil { + return "", err + } + assert.Nil(t, err) + return buf.String(), nil +} + +func createSecret(ns string) (corev1.Secret, error) { + keyCertPair := generateSampleKeyAndCertificate(ns) + sec := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: keyCertPair.Key, + corev1.TLSCertKey: keyCertPair.Certificate, + }, + } + return sec, TestClient().Create(TestContext, &sec) +} + +func generateSampleKeyAndCertificate(ns string) keyCertificatePair { + // Generate the TLS certificate + serialNumber := big.NewInt(rand2.Int63()) + dnsHostname := integrationName + "-" + ns + "." + domainName() + x509Certificate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Camel K test"}, + }, + IsCA: true, + DNSNames: []string{dnsHostname}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // generate the private key + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Printf("Error generating private key: %s\n", err) + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + // encode for storing into secret + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, &x509Certificate, &x509Certificate, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + fmt.Printf("Error generating certificate: %s\n", err) + } + + // encode for storing into secret + certPem = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return keyCertificatePair { + Key: privateKeyPem, + Certificate: certPem, + } +} + +func domainName() string { + consoleroute := Route("openshift-console", "console")() Review comment: Any reason why to use `openshift-console`? Could we use something not bound to any product? ########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,444 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + rand2 "math/rand" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "test-certificate" + servingCertificateSecret = "serving-certificate" + integrationName = "platform-http-server" +) + +type keyCertificatePair struct { + Key []byte + Certificate []byte +} + +var certPem []byte + +// this test uses a route with no TLS +func TestRunRouteNoTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java").Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route unsecure http works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before doing an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(4 * time.Second) + url := fmt.Sprintf("http://%s/hello?name=Simple";, route().Spec.Host) + response, err := httpRequest(t, url, false) + assert.Nil(t, err) + assert.Equal(t, "Hello Simple", response) + }) + }) +} + +func TestRunRouteTLS_Edge(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + // delete secret when the test finishes + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret/certificates previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + "-t", "route.tls-termination=edge", + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + ).Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Edge https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + url := fmt.Sprintf("https://%s/hello?name=TLS_Edge";, route().Spec.Host) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello TLS_Edge", response) + }) + }) +} + +func TestRunRouteTLS_Passthrough(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=passthrough", + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route passthrough https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Passthrough" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a self-signed certificate to create a TLS route +func TestRunRouteTLS_Reencrypt(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // the destination CA certificate which the route service uses to validate the HTTP endpoint TLS certificate + "-t", "route.tls-destination-ca-certificate-secret=" + refCert, + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Reencrypt https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Reencrypt" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a certificate provided by openshift "service serving certificates" to create a TLS route +func TestRunRouteTLS_ReencryptWithServiceCA(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + // they are used only for the pod exposing the HTTP endpoint, not the router + "--resource", "secret:" + servingCertificateSecret + "@/etc/ssl/" + servingCertificateSecret, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + // these certificates are used only in the pod, not the router + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + servingCertificateSecret + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + servingCertificateSecret + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // these certificates are used only in the router to encrypt the connection from the client to the router + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + time.Sleep(2 * time.Second) + Eventually(Service(ns, integrationName)()).ShouldNot(BeNil()) + annotations := make(map[string]string) + annotations["service.beta.openshift.io/serving-cert-secret-name"] = servingCertificateSecret + err = addAnnotation(ns, integrationName, annotations) + assert.Nil(t, err) + + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + t.Run("Route reencrypt (with openshift service CA) https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_ReencryptWithServiceCAWithServiceCA" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +func httpRequest(t *testing.T, url string, tlsEnabled bool) (string, error) { + var client http.Client + if tlsEnabled { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certPem) + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config { + RootCAs: certPool, + }, + } + client = http.Client{Transport: transCfg} + } else { + client = http.Client{} + } + response, err := client.Get(url) + defer func() { + if response != nil { + _ = response.Body.Close() + } + }() + if err != nil { + return "", err + } + assert.Nil(t, err) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(response.Body) + if err != nil { + return "", err + } + assert.Nil(t, err) + return buf.String(), nil +} + +func createSecret(ns string) (corev1.Secret, error) { + keyCertPair := generateSampleKeyAndCertificate(ns) + sec := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: keyCertPair.Key, + corev1.TLSCertKey: keyCertPair.Certificate, + }, + } + return sec, TestClient().Create(TestContext, &sec) +} + +func generateSampleKeyAndCertificate(ns string) keyCertificatePair { + // Generate the TLS certificate + serialNumber := big.NewInt(rand2.Int63()) + dnsHostname := integrationName + "-" + ns + "." + domainName() + x509Certificate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Camel K test"}, + }, + IsCA: true, + DNSNames: []string{dnsHostname}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // generate the private key + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Printf("Error generating private key: %s\n", err) + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + // encode for storing into secret + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, &x509Certificate, &x509Certificate, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + fmt.Printf("Error generating certificate: %s\n", err) + } + + // encode for storing into secret + certPem = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return keyCertificatePair { + Key: privateKeyPem, + Certificate: certPem, + } +} + +func domainName() string { + consoleroute := Route("openshift-console", "console")() + return strings.ReplaceAll(consoleroute.Spec.Host, "console-openshift-console.", "") +} + +func getService(ns string, name string) (*corev1.Service, error) { + svc := corev1.Service{} + key := ctrl.ObjectKey{ + Namespace: ns, + Name: name, + } + err := TestClient().Get(TestContext, key, &svc) + return &svc, err +} + +func addAnnotation(ns string, name string, annotations map[string]string) error { Review comment: Same comment as for `addService` method ########## File path: pkg/trait/route.go ########## @@ -174,19 +196,83 @@ func (t *routeTrait) Apply(e *Environment) error { return nil } -func (t *routeTrait) getTLSConfig() *routev1.TLSConfig { +func (t *routeTrait) getTLSConfig() (*routev1.TLSConfig, error) { + // a certificate is a multiline text, but to set it as value in a single line in CLI, the user must escape the new line character as \\n + // but in the TLS configuration, the certificates should be a multiline string + // then we need to replace the incoming escaped new lines \\n for a real new line \n + key := strings.ReplaceAll(t.TLSKey, "\\n", "\n") Review comment: Probably I did not express my concern properly. I was thinking this part would be better read with an approach like the following: ``` var key string if t.TLSKey != "" { if t.TLSKeySecret != "" { // error, both key and keysecret provided } key = // take value from t.TLSKey } else if t.TLSKeySecret != "" { key = // take value from t.TLSKeySecret } else { // error, missing key/key secret } ``` ########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,444 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + rand2 "math/rand" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "test-certificate" + servingCertificateSecret = "serving-certificate" + integrationName = "platform-http-server" +) + +type keyCertificatePair struct { + Key []byte + Certificate []byte +} + +var certPem []byte + +// this test uses a route with no TLS +func TestRunRouteNoTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java").Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route unsecure http works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before doing an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(4 * time.Second) + url := fmt.Sprintf("http://%s/hello?name=Simple";, route().Spec.Host) + response, err := httpRequest(t, url, false) + assert.Nil(t, err) + assert.Equal(t, "Hello Simple", response) + }) + }) +} + +func TestRunRouteTLS_Edge(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + // delete secret when the test finishes + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret/certificates previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + "-t", "route.tls-termination=edge", + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + ).Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Edge https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + url := fmt.Sprintf("https://%s/hello?name=TLS_Edge";, route().Spec.Host) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello TLS_Edge", response) + }) + }) +} + +func TestRunRouteTLS_Passthrough(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=passthrough", + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route passthrough https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Passthrough" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a self-signed certificate to create a TLS route +func TestRunRouteTLS_Reencrypt(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // the destination CA certificate which the route service uses to validate the HTTP endpoint TLS certificate + "-t", "route.tls-destination-ca-certificate-secret=" + refCert, + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Reencrypt https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Reencrypt" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a certificate provided by openshift "service serving certificates" to create a TLS route +func TestRunRouteTLS_ReencryptWithServiceCA(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + // they are used only for the pod exposing the HTTP endpoint, not the router + "--resource", "secret:" + servingCertificateSecret + "@/etc/ssl/" + servingCertificateSecret, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + // these certificates are used only in the pod, not the router + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + servingCertificateSecret + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + servingCertificateSecret + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // these certificates are used only in the router to encrypt the connection from the client to the router + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + time.Sleep(2 * time.Second) + Eventually(Service(ns, integrationName)()).ShouldNot(BeNil()) + annotations := make(map[string]string) + annotations["service.beta.openshift.io/serving-cert-secret-name"] = servingCertificateSecret + err = addAnnotation(ns, integrationName, annotations) + assert.Nil(t, err) + + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + t.Run("Route reencrypt (with openshift service CA) https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_ReencryptWithServiceCAWithServiceCA" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +func httpRequest(t *testing.T, url string, tlsEnabled bool) (string, error) { + var client http.Client + if tlsEnabled { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certPem) + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config { + RootCAs: certPool, + }, + } + client = http.Client{Transport: transCfg} + } else { + client = http.Client{} + } + response, err := client.Get(url) + defer func() { + if response != nil { + _ = response.Body.Close() + } + }() + if err != nil { + return "", err + } + assert.Nil(t, err) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(response.Body) + if err != nil { + return "", err + } + assert.Nil(t, err) + return buf.String(), nil +} + +func createSecret(ns string) (corev1.Secret, error) { + keyCertPair := generateSampleKeyAndCertificate(ns) + sec := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: keyCertPair.Key, + corev1.TLSCertKey: keyCertPair.Certificate, + }, + } + return sec, TestClient().Create(TestContext, &sec) +} + +func generateSampleKeyAndCertificate(ns string) keyCertificatePair { + // Generate the TLS certificate + serialNumber := big.NewInt(rand2.Int63()) + dnsHostname := integrationName + "-" + ns + "." + domainName() + x509Certificate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Camel K test"}, + }, + IsCA: true, + DNSNames: []string{dnsHostname}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // generate the private key + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Printf("Error generating private key: %s\n", err) + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + // encode for storing into secret + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, &x509Certificate, &x509Certificate, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + fmt.Printf("Error generating certificate: %s\n", err) + } + + // encode for storing into secret + certPem = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return keyCertificatePair { + Key: privateKeyPem, + Certificate: certPem, + } +} + +func domainName() string { + consoleroute := Route("openshift-console", "console")() + return strings.ReplaceAll(consoleroute.Spec.Host, "console-openshift-console.", "") +} + +func getService(ns string, name string) (*corev1.Service, error) { Review comment: If it's a generic method, and is not already present in test_support.go, we better move it there and have it available for anybody willing to use it. ########## File path: e2e/common/traits/route_test.go ########## @@ -0,0 +1,444 @@ +// +build integration + +// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "knative" + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package traits + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + rand2 "math/rand" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/apache/camel-k/e2e/support" + "github.com/apache/camel-k/pkg/util/openshift" +) + +const( + secretName = "test-certificate" + servingCertificateSecret = "serving-certificate" + integrationName = "platform-http-server" +) + +type keyCertificatePair struct { + Key []byte + Certificate []byte +} + +var certPem []byte + +// this test uses a route with no TLS +func TestRunRouteNoTLS(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java").Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route unsecure http works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before doing an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(4 * time.Second) + url := fmt.Sprintf("http://%s/hello?name=Simple";, route().Spec.Host) + response, err := httpRequest(t, url, false) + assert.Nil(t, err) + assert.Equal(t, "Hello Simple", response) + }) + }) +} + +func TestRunRouteTLS_Edge(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + // delete secret when the test finishes + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret/certificates previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + "-t", "route.tls-termination=edge", + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + ).Execute()).To(Succeed()) + // Cleanup all integrations when the test finishes + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Edge https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + url := fmt.Sprintf("https://%s/hello?name=TLS_Edge";, route().Spec.Host) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello TLS_Edge", response) + }) + }) +} + +func TestRunRouteTLS_Passthrough(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=passthrough", + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route passthrough https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Passthrough" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a self-signed certificate to create a TLS route +func TestRunRouteTLS_Reencrypt(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls with certificates + // this secret is used to setupt the route TLS object + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + "--resource", "secret:" + secretName + "@/etc/ssl/" + secretName, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + secretName + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + secretName + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // the destination CA certificate which the route service uses to validate the HTTP endpoint TLS certificate + "-t", "route.tls-destination-ca-certificate-secret=" + refCert, + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + + t.Run("Route Reencrypt https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_Reencrypt" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +// This test uses a certificate provided by openshift "service serving certificates" to create a TLS route +func TestRunRouteTLS_ReencryptWithServiceCA(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + ocp, err := openshift.IsOpenShift(TestClient()) + if !ocp { + t.Skip("This test requires route object which is available on OpenShift only.") + return + } + assert.Nil(t, err) + + // create a test secret of type tls + secret, err := createSecret(ns) + assert.Nil(t, err) + + defer func() { + Expect(TestClient().Delete(TestContext, &secret)).To(Succeed()) + }() + + // they refer to the secret previsouly created + refKey := secretName + "/" + corev1.TLSPrivateKeyKey + refCert := secretName + "/" + corev1.TLSCertKey + Expect(Kamel("install", "-n", ns).Execute()).To(Succeed()) + Expect(Kamel("run", "-n", ns, "files/PlatformHttpServer.java", + // the --resource mounts the certificates inside secret as files in the integration pod + // they are used only for the pod exposing the HTTP endpoint, not the router + "--resource", "secret:" + servingCertificateSecret + "@/etc/ssl/" + servingCertificateSecret, + // quarkus platform-http uses these two properties to setup the HTTP endpoint with TLS support + // these certificates are used only in the pod, not the router + "-p", "quarkus.http.ssl.certificate.file=/etc/ssl/" + servingCertificateSecret + "/tls.crt", + "-p", "quarkus.http.ssl.certificate.key-file=/etc/ssl/" + servingCertificateSecret + "/tls.key", + "-t", "route.tls-termination=reencrypt", + // these certificates are used only in the router to encrypt the connection from the client to the router + "-t", "route.tls-certificate-secret=" + refCert, + "-t", "route.tls-key-secret=" + refKey, + "-t", "container.port=8443", + ).Execute()).To(Succeed()) + // Cleanup all integrations + defer func() { + Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil()) + }() + time.Sleep(2 * time.Second) + Eventually(Service(ns, integrationName)()).ShouldNot(BeNil()) + annotations := make(map[string]string) + annotations["service.beta.openshift.io/serving-cert-secret-name"] = servingCertificateSecret + err = addAnnotation(ns, integrationName, annotations) + assert.Nil(t, err) + + Eventually(IntegrationPodPhase(ns, integrationName), TestTimeoutMedium).Should(Equal(corev1.PodRunning)) + t.Run("Route reencrypt (with openshift service CA) https works", func(t *testing.T) { + route := Route(ns, integrationName) + Eventually(route, TestTimeoutMedium).ShouldNot(BeNil()) + // must wait a little time after route is created, before an http request, + // otherwise the route is unavailable and the http request will fail + time.Sleep(6 * time.Second) + code := "TLS_ReencryptWithServiceCAWithServiceCA" + url := fmt.Sprintf("https://%s/hello?name=%s";, route().Spec.Host, code) + response, err := httpRequest(t, url, true) + assert.Nil(t, err) + assert.Equal(t, "Hello " + code, response) + }) + }) +} + +func httpRequest(t *testing.T, url string, tlsEnabled bool) (string, error) { + var client http.Client + if tlsEnabled { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(certPem) + transCfg := &http.Transport{ + TLSClientConfig: &tls.Config { + RootCAs: certPool, + }, + } + client = http.Client{Transport: transCfg} + } else { + client = http.Client{} + } + response, err := client.Get(url) + defer func() { + if response != nil { + _ = response.Body.Close() + } + }() + if err != nil { + return "", err + } + assert.Nil(t, err) + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(response.Body) + if err != nil { + return "", err + } + assert.Nil(t, err) + return buf.String(), nil +} + +func createSecret(ns string) (corev1.Secret, error) { + keyCertPair := generateSampleKeyAndCertificate(ns) + sec := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: keyCertPair.Key, + corev1.TLSCertKey: keyCertPair.Certificate, + }, + } + return sec, TestClient().Create(TestContext, &sec) +} + +func generateSampleKeyAndCertificate(ns string) keyCertificatePair { + // Generate the TLS certificate + serialNumber := big.NewInt(rand2.Int63()) + dnsHostname := integrationName + "-" + ns + "." + domainName() + x509Certificate := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Camel K test"}, + }, + IsCA: true, + DNSNames: []string{dnsHostname}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + // generate the private key + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + fmt.Printf("Error generating private key: %s\n", err) + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + // encode for storing into secret + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, &x509Certificate, &x509Certificate, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + fmt.Printf("Error generating certificate: %s\n", err) + } + + // encode for storing into secret + certPem = pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return keyCertificatePair { + Key: privateKeyPem, + Certificate: certPem, + } +} + +func domainName() string { + consoleroute := Route("openshift-console", "console")() + return strings.ReplaceAll(consoleroute.Spec.Host, "console-openshift-console.", "") +} + +func getService(ns string, name string) (*corev1.Service, error) { + svc := corev1.Service{} + key := ctrl.ObjectKey{ + Namespace: ns, + Name: name, + } + err := TestClient().Get(TestContext, key, &svc) + return &svc, err +} + +func addAnnotation(ns string, name string, annotations map[string]string) error { Review comment: Same comment as for `getService` method -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@camel.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
