ignite-734: finished implementation, provided tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/fcb8d0d1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/fcb8d0d1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/fcb8d0d1 Branch: refs/heads/ignite-757 Commit: fcb8d0d13f3d5db92c7a10d342ade879fdeb6364 Parents: b98da84 Author: Denis Magda <dma...@gridgain.com> Authored: Wed Apr 22 18:03:03 2015 +0300 Committer: Denis Magda <dma...@gridgain.com> Committed: Wed Apr 22 18:03:03 2015 +0300 ---------------------------------------------------------------------- modules/gce/README.txt | 32 ++ modules/gce/licenses/apache-2.0.txt | 202 ++++++++++ modules/gce/pom.xml | 35 ++ .../gce/TcpDiscoveryGoogleStorageIpFinder.java | 368 +++++++++++++++++++ .../tcp/ipfinder/gce/package-info.java | 22 ++ .../google/TcpDiscoveryGoogleCloudIpFinder.java | 364 ------------------ .../tcp/ipfinder/google/package-info.java | 22 -- ...pDiscoveryGoogleStorageIpFinderSelfTest.java | 72 ++++ .../tcp/ipfinder/gce/package-info.java | 22 ++ .../ignite/testsuites/IgniteGCETestSuite.java | 71 ++++ 10 files changed, 824 insertions(+), 386 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/README.txt ---------------------------------------------------------------------- diff --git a/modules/gce/README.txt b/modules/gce/README.txt new file mode 100644 index 0000000..22c0cb9 --- /dev/null +++ b/modules/gce/README.txt @@ -0,0 +1,32 @@ +Apache Ignite GCE Module +------------------------ + +Apache Ignite GCE module provides Google Cloud Storage based implementations of IP finder for TCP discovery. + +To enable GCE module when starting a standalone node, move 'optional/ignite-gce' folder to +'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will +be added to classpath in this case. + +Importing GCE Module In Maven Project +------------------------------------- + +If you are using Maven to manage dependencies of your project, you can add GCE module +dependency like this (replace '${ignite.version}' with actual Ignite version you are +interested in): + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + ... + <dependencies> + ... + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-gce</artifactId> + <version>${ignite.version}</version> + </dependency> + ... + </dependencies> + ... +</project> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/licenses/apache-2.0.txt ---------------------------------------------------------------------- diff --git a/modules/gce/licenses/apache-2.0.txt b/modules/gce/licenses/apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/modules/gce/licenses/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/pom.xml ---------------------------------------------------------------------- diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml index d4452fc..b12bb18 100644 --- a/modules/gce/pom.xml +++ b/modules/gce/pom.xml @@ -54,6 +54,41 @@ <version>v1-rev32-1.20.0</version> </dependency> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-core</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinder.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinder.java b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinder.java new file mode 100644 index 0000000..a84b2fa --- /dev/null +++ b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinder.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp.ipfinder.gce; + +import java.io.*; +import java.net.*; +import java.security.*; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; + +import com.google.api.client.googleapis.javanet.*; +import com.google.api.client.googleapis.json.*; +import com.google.api.client.http.*; +import com.google.api.client.http.javanet.*; +import com.google.api.services.storage.*; +import com.google.api.services.storage.model.*; +import com.google.api.client.googleapis.auth.oauth2.*; +import com.google.api.client.json.jackson2.*; + +import org.apache.ignite.*; +import org.apache.ignite.internal.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.resources.*; +import org.apache.ignite.spi.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; + +/** + * Google Cloud Storage based IP finder. + * <p> + * For information about Cloud Storage visit <a href="https://cloud.google.com/storage/";>cloud.google.com</a>. + * <h1 class="header">Configuration</h1> + * <h2 class="header">Mandatory</h2> + * <ul> + * <li>Service Account Id (see {@link #setServiceAccountId(String)})</li> + * <li>Service Account P12 key file path (see {@link #setServiceAccountP12FilePath(String)})</li> + * <li>Google Platform project name (see {@link #setProjectName(String)})</li> + * <li>Google Storage bucket name (see {@link #setBucketName(String)})</li> + * </ul> + * <h2 class="header">Optional</h2> + * <ul> + * <li>Shared flag (see {@link #setShared(boolean)})</li> + * </ul> + * <p> + * The finder will create a bucket with the provided name. The bucket will contain entries named + * like the following: {@code 192.168.1.136#1001}. + * <p> + * Note that storing data in Google Cloud Storage service will result in charges to your Google Cloud Platform account. + * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local + * or home network tests. + * <p> + * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. + */ +public class TcpDiscoveryGoogleStorageIpFinder extends TcpDiscoveryIpFinderAdapter { + /* Default object's content. */ + private final static ByteArrayInputStream OBJECT_CONTENT = new ByteArrayInputStream(new byte[0]); + + /** Grid logger. */ + @LoggerResource + private IgniteLogger log; + + /* Google Cloud Platform's project name.*/ + private String projectName; + + /* Google Storage bucket name. */ + private String bucketName; + + /* Service account p12 private key file name. */ + private String serviceAccountP12FilePath; + + /* Service account id. */ + private String serviceAccountId; + + /* Google storage. */ + private Storage storage; + + /* Init routine guard. */ + private final AtomicBoolean initGuard = new AtomicBoolean(); + + /* Init routine latch. */ + private final CountDownLatch initLatch = new CountDownLatch(1); + + public TcpDiscoveryGoogleStorageIpFinder() { + setShared(true); + } + + /** {@inheritDoc} */ + @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { + init(); + + Collection<InetSocketAddress> addrs = new LinkedList<>(); + + try { + Storage.Objects.List listObjects = storage.objects().list(bucketName); + + com.google.api.services.storage.model.Objects objects; + + do { + objects = listObjects.execute(); + + if (objects == null || objects.getItems() == null) + break; + + for (StorageObject object : objects.getItems()) + addrs.add(addrFromString(object.getName())); + + listObjects.setPageToken(objects.getNextPageToken()); + } while (null != objects.getNextPageToken()); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); + } + + return addrs; + } + + /** {@inheritDoc} */ + @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + init(); + + for (InetSocketAddress addr : addrs) { + String key = keyFromAddr(addr); + + StorageObject object = new StorageObject(); + + object.setBucket(bucketName); + object.setName(key); + + InputStreamContent content = new InputStreamContent("application/octet-stream", OBJECT_CONTENT); + content.setLength(OBJECT_CONTENT.available()); + + try { + Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, object, content); + + insertObject.execute(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + + ", entry=" + key + ']', e); + } + } + } + + /** {@inheritDoc} */ + @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + init(); + + for (InetSocketAddress addr : addrs) { + String key = keyFromAddr(addr); + + try { + Storage.Objects.Delete deleteObject = storage.objects().delete(bucketName, key); + + deleteObject.execute(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to delete entry [bucketName=" + bucketName + + ", entry=" + key + ']', e); + } + } + } + + /** + * Sets Google Cloud Platforms project name. + * Usually this is an auto generated project number (ex. 208709979073) that can be found in "Overview" section + * of Google Developer Console. + * <p> + * For details refer to Google Cloud Platform API reference. + * + * @param projectName Project name. + */ + @IgniteSpiConfiguration(optional = false) + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + /** + * Sets Google Cloud Storage bucket name. + * If the bucket doesn't exist Ignite will automatically create it. However the name must be unique across whole + * Google Cloud Storage and Service Account Id (see {@link #setServiceAccountId(String)}) must be authorized to + * perform this operation. + * + * @param bucketName Bucket name. + */ + @IgniteSpiConfiguration(optional = false) + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + + /** + * Sets a full path to the private key in PKCS12 format of the Service Account. + * <p> + * For more information refer to + * <a href="https://cloud.google.com/storage/docs/authentication#service_accounts";>Service Account Authentication</a>. + * + * @param p12FileName Private key file full path. + */ + @IgniteSpiConfiguration(optional = false) + public void setServiceAccountP12FilePath(String p12FileName) { + this.serviceAccountP12FilePath = p12FileName; + } + + /** + * Sets the service account ID (typically an e-mail address). + * <p> + * For more information refer to + * <a href="https://cloud.google.com/storage/docs/authentication#service_accounts";>Service Account Authentication</a>. + * + * @param id + */ + @IgniteSpiConfiguration(optional = false) + public void setServiceAccountId(String id) { + this.serviceAccountId = id; + } + + /** + * Google Cloud Storage initialization. + * + * @throws IgniteSpiException In case of error. + */ + private void init() throws IgniteSpiException { + if (initGuard.compareAndSet(false, true)) { + if (serviceAccountId == null || serviceAccountP12FilePath == null || projectName == null || + bucketName == null) + throw new IgniteSpiException("One or more of the required parameters is not set [serviceAccountId=" + + serviceAccountId + ", serviceAccountP12FilePath=" + serviceAccountP12FilePath + ", projectName=" + + projectName + ", bucketName=" + bucketName + "]"); + + try { + NetHttpTransport httpTransport; + + try { + httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + } + catch (GeneralSecurityException | IOException e) { + throw new IgniteSpiException(e); + } + + GoogleCredential credential; + + try { + credential = new GoogleCredential.Builder().setTransport(httpTransport) + .setJsonFactory(JacksonFactory.getDefaultInstance()).setServiceAccountId(serviceAccountId) + .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountP12FilePath)) + .setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).build(); + + } + catch (Exception e) { + throw new IgniteSpiException("Failed to authenticate on Google Cloud Platform", e); + } + + try { + storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), credential) + .setApplicationName(projectName).build(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to open a storage for given project name: " + projectName, e); + } + + boolean createBucket = false; + + try { + Storage.Buckets.Get getBucket = storage.buckets().get(bucketName); + getBucket.setProjection("full"); + + getBucket.execute(); + } + catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == 404) { + U.warn(log, "Bucket doesn't exist, will create it [bucketName=" + bucketName + "]"); + createBucket = true; + } + else + throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); + } + + if (createBucket) { + Bucket newBucket = new Bucket(); + newBucket.setName(bucketName); + + try { + Storage.Buckets.Insert insertBucket = storage.buckets().insert(projectName, newBucket); + insertBucket.setProjection("full"); + insertBucket.setPredefinedDefaultObjectAcl("projectPrivate"); + + insertBucket.execute(); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to create the bucket: " + bucketName, e); + } + } + } + finally { + initLatch.countDown(); + } + } + else { + try { + U.await(initLatch); + } + catch (IgniteInterruptedCheckedException e) { + throw new IgniteSpiException("Thread has been interrupted.", e); + } + + if (storage == null) + throw new IgniteSpiException("IpFinder has not been initialized properly"); + } + } + + /** + * Constructs bucket's key from an address. + * + * @param addr Node address. + * @return Bucket key. + */ + private String keyFromAddr(InetSocketAddress addr) { + return addr.getAddress().getHostAddress() + "#" + addr.getPort(); + } + + /** + * Constructs a node address from bucket's key. + * + * @param key Bucket key. + * @return Node address. + * @throws IgniteSpiException In case of error. + */ + private InetSocketAddress addrFromString(String key) throws IgniteSpiException { + String[] res = key.split("#"); + + if (res.length != 2) + throw new IgniteSpiException("Invalid address string: " + key); + + + int port; + try { + port = Integer.parseInt(res[1]); + } + catch (NumberFormatException e) { + throw new IgniteSpiException("Invalid port number: " + res[1]); + } + + return new InetSocketAddress(res[0], port); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java new file mode 100644 index 0000000..d46ceb9 --- /dev/null +++ b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Contains Google Cloud Storage IP finder. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.gce; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java deleted file mode 100644 index 4f7529f..0000000 --- a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/TcpDiscoveryGoogleCloudIpFinder.java +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.discovery.tcp.ipfinder.google; - -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.http.InputStreamContent; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; -import com.google.api.services.storage.model.*; -import com.google.common.collect.ImmutableMap; -import org.apache.ignite.IgniteLogger; -import org.apache.ignite.internal.IgniteInterruptedCheckedException; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.resources.LoggerResource; -import org.apache.ignite.spi.*; -import org.apache.ignite.spi.discovery.tcp.ipfinder.*; - -import com.google.api.client.googleapis.auth.oauth2.*; -import com.google.api.client.json.jackson2.*; - -import java.io.*; -import java.net.*; -import java.security.GeneralSecurityException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Google Cloud Storage based IP finder. - * <p> - * For information about Cloud Storage visit <a href="https://cloud.google.com/storage/";>cloud.google.com</a>. - * <h1 class="header">Configuration</h1> - * <h2 class="header">Mandatory</h2> - * <ul> - * <li>Service Account Id (see {@link #setServiceAccountId(String)})</li> - * <li>Service Account P12 key file path (see {@link #setServiceAccountP12FilePath(String)})</li> - * <li>Google Platform project name (see {@link #setProjectName(String)})</li> - * <li>Google Storage bucket name (see {@link #setBucketName(String)})</li> - * </ul> - * <h2 class="header">Optional</h2> - * <ul> - * <li>Shared flag (see {@link #setShared(boolean)})</li> - * </ul> - * <p> - * The finder will create a bucket with the provided name. The bucket will contain entries named - * like the following: {@code 192.168.1.136#1001}. - * <p> - * Note that storing data in Google Cloud Storage service will result in charges to your Google Cloud Platform account. - * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local - * or home network tests. - * <p> - * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. - */ -public class TcpDiscoveryGoogleCloudIpFinder extends TcpDiscoveryIpFinderAdapter { - /* Default object's content. */ - private final static ByteArrayInputStream OBJECT_CONTENT = new ByteArrayInputStream(new byte[0]); - - /** Grid logger. */ - @LoggerResource - private IgniteLogger log; - - /* Google Cloud Platform's project name.*/ - private String projectName; - - /* Google Storage bucket name. */ - private String bucketName; - - /* Service account p12 private key file name. */ - private String serviceAccountP12FilePath; - - /* Service account id. */ - private String serviceAccountId; - - /* Google storage. */ - private Storage storage; - - /* Init routine guard. */ - private final AtomicBoolean initGuard = new AtomicBoolean(); - - /* Init routine latch. */ - private final CountDownLatch initLatch = new CountDownLatch(1); - - public TcpDiscoveryGoogleCloudIpFinder() { - setShared(true); - } - - /** {@inheritDoc} */ - @Override public Collection<InetSocketAddress> getRegisteredAddresses() throws IgniteSpiException { - init(); - - Collection<InetSocketAddress> addrs = new LinkedList<>(); - - try { - Storage.Objects.List listObjects = storage.objects().list(bucketName); - - com.google.api.services.storage.model.Objects objects; - - do { - objects = listObjects.execute(); - - if (objects == null || objects.getItems() == null) - break; - - for (StorageObject object : objects.getItems()) - addrs.add(addrFromString(object.getName())); - - listObjects.setPageToken(objects.getNextPageToken()); - } while (null != objects.getNextPageToken()); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to get content from the bucket: " + bucketName, e); - } - - return addrs; - } - - /** {@inheritDoc} */ - @Override public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { - assert !F.isEmpty(addrs); - - init(); - - for (InetSocketAddress addr : addrs) { - String key = keyFromAddr(addr); - - StorageObject object = new StorageObject(); - - object.setBucket(bucketName); - object.setName(key); - - InputStreamContent content = new InputStreamContent("application/octet-stream", OBJECT_CONTENT); - content.setLength(OBJECT_CONTENT.available()); - - try { - Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, object, content); - - insertObject.execute(); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + - ", entry=" + key + ']', e); - } - } - } - - /** {@inheritDoc} */ - @Override public void unregisterAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException { - assert !F.isEmpty(addrs); - - init(); - - for (InetSocketAddress addr : addrs) { - String key = keyFromAddr(addr); - - try { - Storage.Objects.Delete deleteObject = storage.objects().delete(bucketName, key); - - deleteObject.execute(); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to delete entry [bucketName=" + bucketName + - ", entry=" + key + ']', e); - } - } - } - - /** - * Sets Google Cloud Platforms project name. - * Usually this is an auto generated project number (ex. 208709979073) that can be found in "Overview" section - * of Google Developer Console. - * <p> - * For details refer to Google Cloud Platform API reference. - * - * @param projectName Project name. - */ - @IgniteSpiConfiguration(optional = false) - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - /** - * Sets Google Cloud Storage bucket name. - * If the bucket doesn't exist Ignite will automatically create it. However the name must be unique across whole - * Google Cloud Storage and Service Account Id (see {@link #setServiceAccountId(String)}) must be authorized to - * perform this operation. - * - * @param bucketName Bucket name. - */ - @IgniteSpiConfiguration(optional = false) - public void setBucketName(String bucketName) { - this.bucketName = bucketName; - } - - - /** - * Sets a full path to the private key in PKCS12 format of the Service Account. - * <p> - * For more information refer to - * <a href="https://cloud.google.com/storage/docs/authentication#service_accounts";>Service Account Authentication</a>. - * - * @param p12FileName Private key file full path. - */ - @IgniteSpiConfiguration(optional = false) - public void setServiceAccountP12FilePath(String p12FileName) { - this.serviceAccountP12FilePath = p12FileName; - } - - /** - * Sets the service account ID (typically an e-mail address). - * <p> - * For more information refer to - * <a href="https://cloud.google.com/storage/docs/authentication#service_accounts";>Service Account Authentication</a>. - * - * @param id - */ - @IgniteSpiConfiguration(optional = false) - public void setServiceAccountId(String id) { - this.serviceAccountId = id; - } - - /** - * Google Cloud Storage initialization. - * - * @throws IgniteSpiException In case of error. - */ - private void init() throws IgniteSpiException { - if (initGuard.compareAndSet(false, true)) { - try { - NetHttpTransport httpTransport; - - try { - httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - } - catch (GeneralSecurityException | IOException e) { - throw new IgniteSpiException(e); - } - - GoogleCredential credential; - - try { - credential = new GoogleCredential.Builder().setTransport(httpTransport) - .setJsonFactory(JacksonFactory.getDefaultInstance()).setServiceAccountId(serviceAccountId) - .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountP12FilePath)) - .setServiceAccountScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).build(); - - } - catch (Exception e) { - throw new IgniteSpiException("Failed to authenticate on Google Cloud Platform", e); - } - - try { - storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(), credential) - .setApplicationName(projectName).build(); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to open a storage for given project name: " + projectName, e); - } - - boolean createBucket = false; - - try { - Storage.Buckets.Get getBucket = storage.buckets().get(bucketName); - getBucket.setProjection("full"); - - getBucket.execute(); - } - catch (GoogleJsonResponseException e) { - if (e.getStatusCode() == 404) { - U.warn(log, "Bucket doesn't exist, will create it [bucketName=" + bucketName + "]"); - createBucket = true; - } - else - throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to open the bucket: " + bucketName, e); - } - - if (createBucket) { - Bucket newBucket = new Bucket(); - newBucket.setName(bucketName); - - try { - Storage.Buckets.Insert insertBucket = storage.buckets().insert(projectName, newBucket); - insertBucket.setProjection("full"); - insertBucket.setPredefinedDefaultObjectAcl("projectPrivate"); - - insertBucket.execute(); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to create the bucket: " + bucketName, e); - } - } - } - finally { - initLatch.countDown(); - } - } - else { - try { - U.await(initLatch); - } - catch (IgniteInterruptedCheckedException e) { - throw new IgniteSpiException("Thread has been interrupted.", e); - } - - if (storage == null) - throw new IgniteSpiException("IpFinder has not been initialized properly"); - } - } - - /** - * Constructs bucket's key from an address. - * - * @param addr Node address. - * @return Bucket key. - */ - private String keyFromAddr(InetSocketAddress addr) { - return addr.getAddress().getHostAddress() + "#" + addr.getPort(); - } - - /** - * Constructs a node address from bucket's key. - * - * @param key Bucket key. - * @return Node address. - * @throws IgniteSpiException In case of error. - */ - private InetSocketAddress addrFromString(String key) throws IgniteSpiException { - String[] res = key.split("#"); - - if (res.length != 2) - throw new IgniteSpiException("Invalid address string: " + key); - - - int port; - try { - port = Integer.parseInt(res[1]); - } - catch (NumberFormatException e) { - throw new IgniteSpiException("Invalid port number: " + res[1]); - } - - return new InetSocketAddress(res[0], port); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/package-info.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/package-info.java b/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/package-info.java deleted file mode 100644 index fcf00b4..0000000 --- a/modules/gce/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/google/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * <!-- Package description. --> - * Contains Google Cloud Storage IP finder. - */ -package org.apache.ignite.spi.discovery.tcp.ipfinder.google; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinderSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinderSelfTest.java b/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinderSelfTest.java new file mode 100644 index 0000000..b769efe --- /dev/null +++ b/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/TcpDiscoveryGoogleStorageIpFinderSelfTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp.ipfinder.gce; + +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.testsuites.*; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Collection; + +/** + * Google Cloud Storage based IP finder tests. + */ +public class TcpDiscoveryGoogleStorageIpFinderSelfTest + extends TcpDiscoveryIpFinderAbstractSelfTest<TcpDiscoveryGoogleStorageIpFinder> { + /** + * Constructor. + * + * @throws Exception If any error occurs. + */ + public TcpDiscoveryGoogleStorageIpFinderSelfTest() throws Exception { + // No-op. + } + + @Override protected TcpDiscoveryGoogleStorageIpFinder ipFinder() throws Exception { + TcpDiscoveryGoogleStorageIpFinder finder = new TcpDiscoveryGoogleStorageIpFinder(); + + injectLogger(finder); + + assert finder.isShared() : "Ip finder must be shared by default."; + + finder.setServiceAccountId(IgniteGCETestSuite.getServiceAccountId()); + finder.setServiceAccountP12FilePath(IgniteGCETestSuite.getP12FilePath()); + finder.setProjectName(IgniteGCETestSuite.getProjectName()); + + // Bucket name must be unique across the whole GCE platform. + finder.setBucketName("ip-finder-test-bucket-" + InetAddress.getLocalHost().getAddress()[3]); + + for (int i = 0; i < 5; i++) { + Collection<InetSocketAddress> addrs = finder.getRegisteredAddresses(); + + if (!addrs.isEmpty()) + finder.unregisterAddresses(addrs); + else + return finder; + + U.sleep(1000); + } + + if (!finder.getRegisteredAddresses().isEmpty()) + throw new Exception("Failed to initialize IP finder."); + + return finder; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java b/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java new file mode 100644 index 0000000..4d33114 --- /dev/null +++ b/modules/gce/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/gce/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Contains Google Cloud Storage IP finder internal tests. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.gce; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/fcb8d0d1/modules/gce/src/test/java/org/apache/ignite/testsuites/IgniteGCETestSuite.java ---------------------------------------------------------------------- diff --git a/modules/gce/src/test/java/org/apache/ignite/testsuites/IgniteGCETestSuite.java b/modules/gce/src/test/java/org/apache/ignite/testsuites/IgniteGCETestSuite.java new file mode 100644 index 0000000..ff66c20 --- /dev/null +++ b/modules/gce/src/test/java/org/apache/ignite/testsuites/IgniteGCETestSuite.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.gce.*; + +/** + * Google Compute Engine integration tests. + */ +public class IgniteGCETestSuite extends TestSuite { + /** + * @return Test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new TestSuite("Google Compute Engine Integration Test Suite"); + + suite.addTest(new TestSuite(TcpDiscoveryGoogleStorageIpFinderSelfTest.class)); + + return suite; + } + + /** + * @return Service Account Id. + */ + public static String getServiceAccountId() { + String id = System.getenv("test.gce.account.id"); + + assert id != null : "Environment variable 'test.gce.account.id' is not set"; + + return id; + } + + /** + * @return Service Account p12 file path. + */ + public static String getP12FilePath() { + String path = System.getenv("test.gce.p12.path"); + + assert path != null : "Environment variable 'test.gce.p12.path' is not set"; + + return path; + } + + /** + * @return GCE project name. + */ + public static String getProjectName() { + String name = System.getenv("test.gce.project.name"); + + assert name != null : "Environment variable 'test.gce.project.name' is not set"; + + return name; + } +}
