http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java deleted file mode 100644 index dc9044e..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridUriDeploymentFtpScanner.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners.ftp; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.spi.deployment.uri.scanners.*; - -import java.io.*; -import java.net.*; -import java.util.*; - -/** - * FTP scanner scans directory for new files. Scanned directory defined in URI. - * Scanner doesn't search files in subfolders. - */ -public class GridUriDeploymentFtpScanner extends GridUriDeploymentScanner { - /** */ - private static final long UNKNOWN_FILE_TSTAMP = -1; - - /** */ - private final GridUriDeploymentFtpConfiguration cfg; - - /** Cache of found files to check if any of it has been updated. */ - private Map<GridUriDeploymentFtpFile, Long> cache = new HashMap<>(); - - /** - * @param gridName Grid instance name. - * @param uri FTP URI. - * @param deployDir FTP directory. - * @param freq Scanner frequency. - * @param filter Scanner filter. - * @param lsnr Deployment listener. - * @param log Logger to use. - */ - public GridUriDeploymentFtpScanner( - String gridName, - URI uri, - File deployDir, - long freq, - FilenameFilter filter, - GridUriDeploymentScannerListener lsnr, - IgniteLogger log) { - super(gridName, uri, deployDir, freq, filter, lsnr, log); - - cfg = initializeFtpConfiguration(uri); - } - - /** - * @param uri FTP URI. - * @return FTP configuration. - */ - private GridUriDeploymentFtpConfiguration initializeFtpConfiguration(URI uri) { - assert "ftp".equals(uri.getScheme()); - - GridUriDeploymentFtpConfiguration cfg = new GridUriDeploymentFtpConfiguration(); - - String userInfo = uri.getUserInfo(); - String username = null; - String pswd = null; - - if (userInfo != null) { - String[] arr = userInfo.split(";"); - - if (arr != null && arr.length > 0) - for (String el : arr) - if (el.startsWith("freq=")) { - // No-op. - } - else if (el.indexOf(':') != -1) { - int idx = el.indexOf(':'); - - username = el.substring(0, idx); - pswd = el.substring(idx + 1); - } - else - username = el; - } - - // Username and password must be defined in URI. - if (username == null) - throw new IgniteException("Username has not been provided."); - - if (pswd == null) - throw new IgniteException("Password has not been provided."); - - cfg.setHost(uri.getHost()); - cfg.setPort(uri.getPort()); - cfg.setUsername(username); - cfg.setPassword(pswd); - cfg.setDirectory(uri.getPath()); - - return cfg; - } - - /** {@inheritDoc} */ - @Override protected void process() { - Collection<GridUriDeploymentFtpFile> foundFiles = U.newHashSet(cache.size()); - - long start = U.currentTimeMillis(); - - processFtp(foundFiles); - - if (getLogger().isDebugEnabled()) - getLogger().debug("FTP scanner time in milliseconds: " + (U.currentTimeMillis() - start)); - - if (!isFirstScan()) { - Collection<GridUriDeploymentFtpFile> delFiles = new HashSet<>(cache.keySet()); - - delFiles.removeAll(foundFiles); - - if (!delFiles.isEmpty()) { - List<String> uris = new ArrayList<>(); - - for (GridUriDeploymentFtpFile file : delFiles) { - Long tstamp = cache.get(file); - - // Ignore files in cache w/o timestamp. - if (tstamp != null && tstamp != UNKNOWN_FILE_TSTAMP) - uris.add(getFileUri(file.getName())); - } - - cache.keySet().removeAll(delFiles); - - getListener().onDeletedFiles(uris); - } - } - } - - /** - * @param files File to process. - */ - @SuppressWarnings({"UnusedCatchParameter"}) - private void processFtp(Collection<GridUriDeploymentFtpFile> files) { - GridUriDeploymentFtpClient ftp = new GridUriDeploymentFtpClient(cfg, getLogger()); - - try { - ftp.connect(); - - for (GridUriDeploymentFtpFile file : ftp.getFiles()) { - String fileName = file.getName(); - - if (getFilter().accept(null, fileName.toLowerCase()) && file.isFile()) { - files.add(file); - - Long lastModified = cache.get(file); - - Calendar fileTstamp = file.getTimestamp(); - - if (fileTstamp == null) { - if (lastModified == null) { - // Add new file in cache to avoid print warning every time. - cache.put(file, UNKNOWN_FILE_TSTAMP); - - U.warn(getLogger(), "File with unknown timestamp will be ignored " + - "(check FTP server configuration): " + file); - } - } - // If file is new or has been modified. - else if (lastModified == null || lastModified != fileTstamp.getTimeInMillis()) { - cache.put(file, fileTstamp.getTimeInMillis()); - - if (getLogger().isDebugEnabled()) - getLogger().debug("Discovered deployment file or directory: " + file); - - try { - File diskFile = createTempFile(fileName, getDeployDirectory()); - - ftp.downloadToFile(file, diskFile); - - String fileUri = getFileUri(fileName); - - // Delete file when JVM stopped. - diskFile.deleteOnExit(); - - // Deployment SPI apply. - // NOTE: If SPI listener blocks then FTP connection may be closed by timeout. - getListener().onNewOrUpdatedFile(diskFile, fileUri, fileTstamp.getTimeInMillis()); - } - catch (IOException e) { - U.error(getLogger(), "Failed to download file from FTP server: " + fileName, e); - } - } - } - } - } - catch (GridUriDeploymentFtpException e) { - if (!isCancelled()) { - String maskedUri = getUri() != null ? U.hidePassword(getUri().toString()) : null; - - if (e.hasCause(ConnectException.class)) - LT.warn(getLogger(), e, "Failed to connect to FTP server (connection refused): " + maskedUri); - - else if (e.hasCause(UnknownHostException.class)) - LT.warn(getLogger(), e, "Failed to connect to FTP server (host is unknown): " + maskedUri); - - else - U.error(getLogger(), "Failed to get files from FTP server: " + maskedUri, e); - } - } - finally { - try { - ftp.close(); - } - catch (GridUriDeploymentFtpException e) { - if (!isCancelled()) - U.error(getLogger(), "Failed to close FTP client.", e); - } - } - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridUriDeploymentFtpScanner.class, this, - "uri", getUri() != null ? U.hidePassword(getUri().toString()) : null, - "freq", getFrequency(), - "deployDir", getDeployDirectory()); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java deleted file mode 100644 index ca29005..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * <!-- Package description. --> - * Contains FTP scanner for URI deployer SPI. - */ -package org.apache.ignite.spi.deployment.uri.scanners.ftp; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java deleted file mode 100644 index 49ac343..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners.http; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.tostring.*; -import org.apache.ignite.internal.util.typedef.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.spi.*; -import org.apache.ignite.spi.deployment.uri.scanners.*; -import org.jetbrains.annotations.*; -import org.w3c.dom.*; -import org.w3c.tidy.*; - -import javax.net.ssl.*; -import java.io.*; -import java.net.*; -import java.security.*; -import java.security.cert.*; -import java.util.*; - -/** - * URI deployment HTTP scanner. - */ -public class GridUriDeploymentHttpScanner extends GridUriDeploymentScanner { - /** Secure socket protocol to use. */ - private static final String PROTOCOL = "TLS"; - - /** */ - @GridToStringExclude - private URL scanDir; - - /** Cache of found files to check if any of it has been updated. */ - private Map<String, Long> tstampCache = new HashMap<>(); - - /** */ - @GridToStringExclude - private final Tidy tidy; - - /** Outgoing data SSL socket factory. */ - private SSLSocketFactory sockFactory; - - /** - * @param gridName Grid instance name. - * @param uri HTTP URI. - * @param deployDir Deployment directory. - * @param freq Scanner frequency. - * @param filter Filename filter. - * @param lsnr Deployment listener. - * @param log Logger to use. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error. - */ - public GridUriDeploymentHttpScanner( - String gridName, - URI uri, - File deployDir, - long freq, - FilenameFilter filter, - GridUriDeploymentScannerListener lsnr, - IgniteLogger log) throws IgniteSpiException { - super(gridName, uri, deployDir, freq, filter, lsnr, log); - - initialize(uri); - - tidy = new Tidy(); - - tidy.setQuiet(true); - tidy.setOnlyErrors(true); - tidy.setShowWarnings(false); - tidy.setInputEncoding("UTF8"); - tidy.setOutputEncoding("UTF8"); - } - - /** - * @param uri HTTP URI. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error. - */ - private void initialize(URI uri) throws IgniteSpiException { - assert "http".equals(uri.getScheme()) || "https".equals(uri.getScheme()); - - try { - scanDir = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath()); - } - catch (MalformedURLException e) { - scanDir = null; - - throw new IgniteSpiException("Wrong value for scanned HTTP directory with URI: " + uri, e); - } - - try { - if ("https".equals(uri.getScheme())) { - // Set up socket factory to do authentication. - SSLContext ctx = SSLContext.getInstance(PROTOCOL); - - ctx.init(null, getTrustManagers(), null); - - sockFactory = ctx.getSocketFactory(); - } - } - catch (NoSuchAlgorithmException e) { - throw new IgniteSpiException("Failed to initialize SSL context. URI: " + uri, e); - } - catch (KeyManagementException e) { - throw new IgniteSpiException("Failed to initialize SSL context. URI:" + uri, e); - } - } - - /** {@inheritDoc} */ - @Override protected void process() { - Collection<String> foundFiles = U.newHashSet(tstampCache.size()); - - long start = U.currentTimeMillis(); - - processHttp(foundFiles); - - if (getLogger().isDebugEnabled()) - getLogger().debug("HTTP scanner time in ms: " + (U.currentTimeMillis() - start)); - - if (!isFirstScan()) { - Collection<String> deletedFiles = new HashSet<>(tstampCache.keySet()); - - deletedFiles.removeAll(foundFiles); - - if (!deletedFiles.isEmpty()) { - List<String> uris = new ArrayList<>(); - - for (String file : deletedFiles) { - uris.add(getFileUri(getFileName(file))); - } - - tstampCache.keySet().removeAll(deletedFiles); - - getListener().onDeletedFiles(uris); - } - } - } - - /** - * @param files Files to process. - */ - private void processHttp(Collection<String> files) { - Set<String> urls = getUrls(scanDir); - - for (String url : urls) { - String fileName = getFileName(url); - - if (getFilter().accept(null, fileName)) { - files.add(url); - - Long lastModified = tstampCache.get(url); - - InputStream in = null; - OutputStream out = null; - - File file = null; - - try { - URLConnection conn = new URL(url).openConnection(); - - if (conn instanceof HttpsURLConnection) { - HttpsURLConnection httpsConn = (HttpsURLConnection)conn; - - httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); - - assert sockFactory != null; - - // Initialize socket factory. - httpsConn.setSSLSocketFactory(sockFactory); - } - - if (lastModified != null) - conn.setIfModifiedSince(lastModified); - - in = conn.getInputStream(); - - long rcvLastModified = conn.getLastModified(); - - if (in == null || lastModified != null && (lastModified == rcvLastModified || - conn instanceof HttpURLConnection && - ((HttpURLConnection)conn).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED)) - continue; - - tstampCache.put(url, rcvLastModified); - - lastModified = rcvLastModified; - - if (getLogger().isDebugEnabled()) { - getLogger().debug("Discovered deployment file or directory: " + - U.hidePassword(url)); - } - - file = createTempFile(fileName, getDeployDirectory()); - - // Delete file when JVM stopped. - file.deleteOnExit(); - - out = new FileOutputStream(file); - - U.copy(in, out); - } - catch (IOException e) { - if (!isCancelled()) { - if (X.hasCause(e, ConnectException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " + - U.hidePassword(url)); - } - else if (X.hasCause(e, UnknownHostException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " + - U.hidePassword(url)); - } - else - U.error(getLogger(), "Failed to save file: " + fileName, e); - } - } - finally { - U.closeQuiet(in); - U.closeQuiet(out); - } - - if (file != null && file.exists() && file.length() > 0) - getListener().onNewOrUpdatedFile(file, getFileUri(fileName), lastModified); - } - } - } - - /** - * @param node XML element node. - * @param res Set of URLs in string format to populate. - * @param baseUrl Base URL. - */ - @SuppressWarnings( {"UnusedCatchParameter", "UnnecessaryFullyQualifiedName"}) - private void findReferences(org.w3c.dom.Node node, Set<String> res, URL baseUrl) { - if (node instanceof Element && "a".equals(node.getNodeName().toLowerCase())) { - Element element = (Element)node; - - String href = element.getAttribute("href"); - - if (href != null && !href.isEmpty()) { - URL url = null; - - try { - url = new URL(href); - } - catch (MalformedURLException e) { - try { - url = new URL(baseUrl.getProtocol(), baseUrl.getHost(), baseUrl.getPort(), - href.charAt(0) == '/' ? href : baseUrl.getFile() + '/' + href); - } - catch (MalformedURLException e1) { - U.error(getLogger(), "Skipping bad URL: " + url, e1); - } - } - - if (url != null) - res.add(url.toString()); - } - } - - NodeList childNodes = node.getChildNodes(); - - for (int i = 0; i < childNodes.getLength(); i++) - findReferences(childNodes.item(i), res, baseUrl); - } - - /** - * @param url Base URL. - * @return Set of referenced URLs in string format. - */ - private Set<String> getUrls(URL url) { - assert url != null; - - InputStream in = null; - - Set<String> urls = new HashSet<>(); - - Document dom = null; - - try { - URLConnection conn = url.openConnection(); - - if (conn instanceof HttpsURLConnection) { - HttpsURLConnection httpsConn = (HttpsURLConnection)conn; - - httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); - - assert sockFactory != null; - - // Initialize socket factory. - httpsConn.setSSLSocketFactory(sockFactory); - } - - in = conn.getInputStream(); - - if (in == null) - throw new IOException("Failed to open connection: " + U.hidePassword(url.toString())); - - dom = tidy.parseDOM(in, null); - } - catch (IOException e) { - if (!isCancelled()) { - if (X.hasCause(e, ConnectException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " + - U.hidePassword(url.toString())); - } - else if (X.hasCause(e, UnknownHostException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " + - U.hidePassword(url.toString())); - } - else - U.error(getLogger(), "Failed to get HTML page: " + U.hidePassword(url.toString()), e); - } - } - finally{ - U.closeQuiet(in); - } - - if (dom != null) - findReferences(dom, urls, url); - - return urls; - } - - /** - * @param url Base URL string format. - * @return File name extracted from {@code url} string format. - */ - private String getFileName(String url) { - assert url != null; - - return url.substring(url.lastIndexOf('/') + 1); - } - - /** - * Construct array with one trust manager which don't reject input certificates. - * - * @return Array with one X509TrustManager implementation of trust manager. - */ - private TrustManager[] getTrustManagers() { - return new TrustManager[]{ - new X509TrustManager() { - /** {@inheritDoc} */ - @Nullable @Override public X509Certificate[] getAcceptedIssuers() { return null; } - - /** {@inheritDoc} */ - @Override public void checkClientTrusted(X509Certificate[] certs, String authType) { - StringBuilder buf = new StringBuilder(); - - buf.append("Trust manager handle client certificates [authType="); - buf.append(authType); - buf.append(", certificates="); - - for (X509Certificate cert : certs) { - buf.append("{type="); - buf.append(cert.getType()); - buf.append(", principalName="); - buf.append(cert.getSubjectX500Principal().getName()); - buf.append('}'); - } - - buf.append(']'); - - if (getLogger().isDebugEnabled()) - getLogger().debug(buf.toString()); - } - - /** {@inheritDoc} */ - @Override public void checkServerTrusted(X509Certificate[] certs, String authType) { - StringBuilder buf = new StringBuilder(); - - buf.append("Trust manager handle server certificates [authType="); - buf.append(authType); - buf.append(", certificates="); - - for (X509Certificate cert : certs) { - buf.append("{type="); - buf.append(cert.getType()); - buf.append(", principalName="); - buf.append(cert.getSubjectX500Principal().getName()); - buf.append('}'); - } - - buf.append(']'); - - if (getLogger().isDebugEnabled()) - getLogger().debug(buf.toString()); - } - } - }; - } - - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridUriDeploymentHttpScanner.class, this, - "scanDir", scanDir != null ? U.hidePassword(scanDir.toString()) : null); - } - - /** - * Verifier always return successful result for any host. - */ - private static class DeploymentHostnameVerifier implements HostnameVerifier { - /** {@inheritDoc} */ - @Override public boolean verify(String hostname, SSLSession ses) { - // Remote host trusted by default. - return true; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java new file mode 100644 index 0000000..002b2e1 --- /dev/null +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/UriDeploymentHttpScanner.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.deployment.uri.scanners.http; + +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.spi.*; +import org.apache.ignite.spi.deployment.uri.scanners.*; +import org.jetbrains.annotations.*; +import org.w3c.dom.*; +import org.w3c.tidy.*; + +import javax.net.ssl.*; +import java.io.*; +import java.net.*; +import java.security.*; +import java.security.cert.*; +import java.util.*; +import java.util.concurrent.*; + +/** + * URI deployment HTTP scanner. + */ +public class UriDeploymentHttpScanner implements UriDeploymentScanner { + /** Default scan frequency. */ + private static final int DFLT_SCAN_FREQ = 300000; + + /** Secure socket protocol to use. */ + private static final String PROTOCOL = "TLS"; + + /** Per-URI contexts. */ + private final ConcurrentHashMap<URI, URIContext> uriCtxs = new ConcurrentHashMap<>(); + + /** {@inheritDoc} */ + @Override public boolean acceptsURI(URI uri) { + String proto = uri.getScheme().toLowerCase(); + + return "http".equals(proto) || "https".equals(proto); + } + + /** {@inheritDoc} */ + @Override public void scan(UriDeploymentScannerContext scanCtx) { + URI uri = scanCtx.getUri(); + + URIContext uriCtx = uriCtxs.get(uri); + + if (uriCtx == null) { + uriCtx = createUriContext(uri, scanCtx); + + URIContext oldUriCtx = uriCtxs.putIfAbsent(uri, uriCtx); + + if (oldUriCtx != null) + uriCtx = oldUriCtx; + } + + uriCtx.scan(scanCtx); + } + + /** + * Create context for the given URI. + * + * @param uri URI. + * @param scanCtx Scanner context. + * @return URI context. + */ + private URIContext createUriContext(URI uri, final UriDeploymentScannerContext scanCtx) { + assert "http".equals(uri.getScheme()) || "https".equals(uri.getScheme()); + + URL scanDir; + + try { + scanDir = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath()); + } + catch (MalformedURLException e) { + throw new IgniteSpiException("Wrong value for scanned HTTP directory with URI: " + uri, e); + } + + SSLSocketFactory sockFactory = null; + + try { + if ("https".equals(uri.getScheme())) { + // Set up socket factory to do authentication. + SSLContext ctx = SSLContext.getInstance(PROTOCOL); + + ctx.init(null, getTrustManagers(scanCtx), null); + + sockFactory = ctx.getSocketFactory(); + } + } + catch (NoSuchAlgorithmException e) { + throw new IgniteSpiException("Failed to initialize SSL context. URI: " + uri, e); + } + catch (KeyManagementException e) { + throw new IgniteSpiException("Failed to initialize SSL context. URI:" + uri, e); + } + + return new URIContext(scanDir, sockFactory); + } + + /** {@inheritDoc} */ + @Override public long getDefaultScanFrequency() { + return DFLT_SCAN_FREQ; + } + + /** + * Construct array with one trust manager which don't reject input certificates. + * + * @param scanCtx context. + * @return Array with one X509TrustManager implementation of trust manager. + */ + private static TrustManager[] getTrustManagers(final UriDeploymentScannerContext scanCtx) { + return new TrustManager[]{ + new X509TrustManager() { + /** {@inheritDoc} */ + @Nullable + @Override public X509Certificate[] getAcceptedIssuers() { return null; } + + /** {@inheritDoc} */ + @Override public void checkClientTrusted(X509Certificate[] certs, String authType) { + StringBuilder buf = new StringBuilder(); + + buf.append("Trust manager handle client certificates [authType="); + buf.append(authType); + buf.append(", certificates="); + + for (X509Certificate cert : certs) { + buf.append("{type="); + buf.append(cert.getType()); + buf.append(", principalName="); + buf.append(cert.getSubjectX500Principal().getName()); + buf.append('}'); + } + + buf.append(']'); + + if (scanCtx.getLogger().isDebugEnabled()) + scanCtx.getLogger().debug(buf.toString()); + } + + /** {@inheritDoc} */ + @Override public void checkServerTrusted(X509Certificate[] certs, String authType) { + StringBuilder buf = new StringBuilder(); + + buf.append("Trust manager handle server certificates [authType="); + buf.append(authType); + buf.append(", certificates="); + + for (X509Certificate cert : certs) { + buf.append("{type="); + buf.append(cert.getType()); + buf.append(", principalName="); + buf.append(cert.getSubjectX500Principal().getName()); + buf.append('}'); + } + + buf.append(']'); + + if (scanCtx.getLogger().isDebugEnabled()) + scanCtx.getLogger().debug(buf.toString()); + } + } + }; + } + + /** + * URI context. + */ + private class URIContext { + /** */ + private final URL scanDir; + + /** Outgoing data SSL socket factory. */ + private final SSLSocketFactory sockFactory; + + /** */ + private final Tidy tidy; + + /** Cache of found files to check if any of it has been updated. */ + private final Map<String, Long> tstampCache = new HashMap<>(); + + /** + * Constructor. + * + * @param scanDir Scan directory. + * @param sockFactory Socket factory. + */ + public URIContext(URL scanDir, SSLSocketFactory sockFactory) { + this.scanDir = scanDir; + this.sockFactory = sockFactory; + + tidy = new Tidy(); + + tidy.setQuiet(true); + tidy.setOnlyErrors(true); + tidy.setShowWarnings(false); + tidy.setInputEncoding("UTF8"); + tidy.setOutputEncoding("UTF8"); + } + + /** + * Perform scan. + * + * @param scanCtx Scan context. + */ + private void scan(UriDeploymentScannerContext scanCtx) { + Collection<String> foundFiles = U.newHashSet(tstampCache.size()); + + long start = U.currentTimeMillis(); + + processHttp(foundFiles, scanCtx); + + if (scanCtx.getLogger().isDebugEnabled()) + scanCtx.getLogger().debug("HTTP scanner time in ms: " + (U.currentTimeMillis() - start)); + + if (!scanCtx.isFirstScan()) { + Collection<String> deletedFiles = new HashSet<>(tstampCache.keySet()); + + deletedFiles.removeAll(foundFiles); + + if (!deletedFiles.isEmpty()) { + List<String> uris = new ArrayList<>(); + + for (String file : deletedFiles) + uris.add(getFileUri(fileName(file), scanCtx)); + + tstampCache.keySet().removeAll(deletedFiles); + + scanCtx.getListener().onDeletedFiles(uris); + } + } + } + + /** + * @param files Files to process. + * @param scanCtx Scan context. + */ + @SuppressWarnings("unchecked") + private void processHttp(Collection<String> files, UriDeploymentScannerContext scanCtx) { + Set<String> urls = getUrls(scanDir, scanCtx); + + for (String url : urls) { + String fileName = fileName(url); + + if (scanCtx.getFilter().accept(null, fileName)) { + files.add(url); + + Long lastModified = tstampCache.get(url); + + InputStream in = null; + OutputStream out = null; + + File file = null; + + try { + URLConnection conn = new URL(url).openConnection(); + + if (conn instanceof HttpsURLConnection) { + HttpsURLConnection httpsConn = (HttpsURLConnection)conn; + + httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); + + assert sockFactory != null; + + // Initialize socket factory. + httpsConn.setSSLSocketFactory(sockFactory); + } + + if (lastModified != null) + conn.setIfModifiedSince(lastModified); + + in = conn.getInputStream(); + + long rcvLastModified = conn.getLastModified(); + + if (in == null || lastModified != null && (lastModified == rcvLastModified || + conn instanceof HttpURLConnection && + ((HttpURLConnection)conn).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED)) + continue; + + tstampCache.put(url, rcvLastModified); + + lastModified = rcvLastModified; + + if (scanCtx.getLogger().isDebugEnabled()) { + scanCtx.getLogger().debug("Discovered deployment file or directory: " + + U.hidePassword(url)); + } + + file = scanCtx.createTempFile(fileName, scanCtx.getDeployDirectory()); + + // Delete file when JVM stopped. + file.deleteOnExit(); + + out = new FileOutputStream(file); + + U.copy(in, out); + } + catch (IOException e) { + if (!scanCtx.isCancelled()) { + if (X.hasCause(e, ConnectException.class)) { + LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " + + "(connection refused): " + U.hidePassword(url)); + } + else if (X.hasCause(e, UnknownHostException.class)) { + LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server " + + "(host is unknown): " + U.hidePassword(url)); + } + else + U.error(scanCtx.getLogger(), "Failed to save file: " + fileName, e); + } + } + finally { + U.closeQuiet(in); + U.closeQuiet(out); + } + + if (file != null && file.exists() && file.length() > 0) + scanCtx.getListener().onNewOrUpdatedFile(file, getFileUri(fileName, scanCtx), lastModified); + } + } + } + + /** + * @param url Base URL. + * @param scanCtx Scan context. + * @return Set of referenced URLs in string format. + */ + @SuppressWarnings("unchecked") + private Set<String> getUrls(URL url, UriDeploymentScannerContext scanCtx) { + assert url != null; + + InputStream in = null; + + Set<String> urls = new HashSet<>(); + + Document dom = null; + + try { + URLConnection conn = url.openConnection(); + + if (conn instanceof HttpsURLConnection) { + HttpsURLConnection httpsConn = (HttpsURLConnection)conn; + + httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); + + assert sockFactory != null; + + // Initialize socket factory. + httpsConn.setSSLSocketFactory(sockFactory); + } + + in = conn.getInputStream(); + + if (in == null) + throw new IOException("Failed to open connection: " + U.hidePassword(url.toString())); + + dom = tidy.parseDOM(in, null); + } + catch (IOException e) { + if (!scanCtx.isCancelled()) { + if (X.hasCause(e, ConnectException.class)) { + LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (connection refused): " + + U.hidePassword(url.toString())); + } + else if (X.hasCause(e, UnknownHostException.class)) { + LT.warn(scanCtx.getLogger(), e, "Failed to connect to HTTP server (host is unknown): " + + U.hidePassword(url.toString())); + } + else + U.error(scanCtx.getLogger(), "Failed to get HTML page: " + U.hidePassword(url.toString()), e); + } + } + finally{ + U.closeQuiet(in); + } + + if (dom != null) + findReferences(dom, urls, url, scanCtx); + + return urls; + } + + /** + * @param node XML element node. + * @param res Set of URLs in string format to populate. + * @param baseUrl Base URL. + * @param scanCtx Scan context. + */ + @SuppressWarnings( {"UnusedCatchParameter", "UnnecessaryFullyQualifiedName"}) + private void findReferences(org.w3c.dom.Node node, Set<String> res, URL baseUrl, + UriDeploymentScannerContext scanCtx) { + if (node instanceof Element && "a".equals(node.getNodeName().toLowerCase())) { + Element element = (Element)node; + + String href = element.getAttribute("href"); + + if (href != null && !href.isEmpty()) { + URL url = null; + + try { + url = new URL(href); + } + catch (MalformedURLException e) { + try { + url = new URL(baseUrl.getProtocol(), baseUrl.getHost(), baseUrl.getPort(), + href.charAt(0) == '/' ? href : baseUrl.getFile() + '/' + href); + } + catch (MalformedURLException e1) { + U.error(scanCtx.getLogger(), "Skipping bad URL: " + href, e1); + } + } + + if (url != null) + res.add(url.toString()); + } + } + + NodeList childNodes = node.getChildNodes(); + + for (int i = 0; i < childNodes.getLength(); i++) + findReferences(childNodes.item(i), res, baseUrl, scanCtx); + } + + /** + * @param url Base URL string format. + * @return File name extracted from {@code url} string format. + */ + private String fileName(String url) { + assert url != null; + + return url.substring(url.lastIndexOf('/') + 1); + } + + /** + * Gets file URI for the given file name. It extends any given name with {@code URI}. + * + * @param name File name. + * @param scanCtx Scan context. + * @return URI for the given file name. + */ + private String getFileUri(String name, UriDeploymentScannerContext scanCtx) { + assert name != null; + + String fileUri = scanCtx.getUri().toString(); + + fileUri = fileUri.length() > 0 && fileUri.charAt(fileUri.length() - 1) == '/' ? fileUri + name : + fileUri + '/' + name; + + return fileUri; + } + } + + /** + * Verifier always return successful result for any host. + */ + private static class DeploymentHostnameVerifier implements HostnameVerifier { + /** {@inheritDoc} */ + @Override public boolean verify(String hostname, SSLSession ses) { + // Remote host trusted by default. + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java index 21e5eb5..32fdadc 100644 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java +++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersErrorThrottlingTest.java @@ -44,9 +44,6 @@ public class GridUriDeploymentMultiScannersErrorThrottlingTest extends GridUriDe public List<String> getUriList() { List<String> uriList = new ArrayList<>(); - uriList.add("ftp://anonymous:111111;freq=5...@unknown.host:21/pub/gg-test";); - uriList.add("ftp://anonymous:111111;freq=5000@localhost:21/pub/gg-test";); - uriList.add("http://freq=5000@localhost/tasks";); uriList.add("http://freq=5...@unknownhost.host/tasks";); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java index af54e06..1beee49 100644 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java +++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/GridUriDeploymentMultiScannersSelfTest.java @@ -42,8 +42,7 @@ public class GridUriDeploymentMultiScannersSelfTest extends GridUriDeploymentAbs public List<String> getUriList() { List<String> uriList = new ArrayList<>(); - // Fake URIs. - uriList.add(GridTestProperties.getProperty("deploy.uri.ftp")); + // Fake URI. uriList.add(GridTestProperties.getProperty("deploy.uri.http")); // One real URI. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java index 2d6d681..0ac7c98 100644 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java +++ b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridFileDeploymentUndeploySelfTest.java @@ -70,7 +70,7 @@ public class GridFileDeploymentUndeploySelfTest extends GridSpiAbstractTest<UriD assert newGarFile.exists(); - Thread.sleep(UriDeploymentSpi.DFLT_DISK_SCAN_FREQUENCY + 3000); + Thread.sleep(UriDeploymentFileScanner.DFLT_SCAN_FREQ + 3000); assert getSpi().findResource("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3") != null : "Failed to find resource for added GAR file."; @@ -82,7 +82,7 @@ public class GridFileDeploymentUndeploySelfTest extends GridSpiAbstractTest<UriD assert !newGarFile.exists(); - Thread.sleep(UriDeploymentSpi.DFLT_DISK_SCAN_FREQUENCY + 3000); + Thread.sleep(UriDeploymentFileScanner.DFLT_SCAN_FREQ + 3000); assert getSpi().findResource("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3") == null; assert getSpi().findResource("GridUriDeploymentTestWithNameTask3") == null; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java deleted file mode 100644 index 5b73889..0000000 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/GridFtpDeploymentTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners.ftp; - -import org.apache.ignite.spi.deployment.uri.*; -import org.apache.ignite.testframework.config.*; -import org.apache.ignite.testframework.junits.spi.*; - -import java.util.*; - -/** - * Test for FTP deployment. - */ -@GridSpiTest(spi = UriDeploymentSpi.class, group = "Deployment SPI") -public class GridFtpDeploymentTest extends GridUriDeploymentAbstractSelfTest { - /** - * @throws Exception If failed. - */ - public void testDeployment() throws Exception { - checkTask("org.apache.ignite.spi.deployment.uri.tasks.GridUriDeploymentTestTask3"); - checkTask("GridUriDeploymentTestWithNameTask3"); - } - - /** - * @return List of deployment sources. - */ - @GridSpiTestConfig - public List<String> getUriList() { - return Collections.singletonList(GridTestProperties.getProperty("deploy.uri.ftp")); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0391d27c/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java b/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java deleted file mode 100644 index 50fd545..0000000 --- a/modules/urideploy/src/test/java/org/apache/ignite/spi/deployment/uri/scanners/ftp/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * <!-- Package description. --> - * Contains internal tests or test related classes and interfaces. - */ -package org.apache.ignite.spi.deployment.uri.scanners.ftp; \ No newline at end of file
