# IGNITE-661: Implemented.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/0755231c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/0755231c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/0755231c Branch: refs/heads/ignite-661 Commit: 0755231c19bb2416169300e4d1ad6165cf92cff4 Parents: 47c4b17 Author: vozerov-gridgain <voze...@gridgain.com> Authored: Thu Apr 9 13:27:57 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Thu Apr 9 13:27:57 2015 +0300 ---------------------------------------------------------------------- .../spi/deployment/uri/UriDeploymentSpi.java | 109 ++--- .../uri/scanners/GridUriDeploymentScanner.java | 286 ----------- .../uri/scanners/UriDeploymentScanner.java | 47 ++ .../scanners/UriDeploymentScannerContext.java | 92 ++++ .../scanners/UriDeploymentScannerManager.java | 221 +++++++++ .../file/GridUriDeploymentFileScanner.java | 311 ------------ .../scanners/file/UriDeploymentFileScanner.java | 327 +++++++++++++ .../http/GridUriDeploymentHttpScanner.java | 423 ---------------- .../scanners/http/UriDeploymentHttpScanner.java | 478 +++++++++++++++++++ .../GridFileDeploymentUndeploySelfTest.java | 4 +- 10 files changed, 1222 insertions(+), 1076 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java index a301a39..cb089c3 100644 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/UriDeploymentSpi.java @@ -161,12 +161,6 @@ import java.util.Map.*; * <th>Optional</th> * <th>Default</th> * </tr> - * <tr> - * <td>freq</td> - * <td>File directory scan frequency in milliseconds.</td> - * <td>Yes</td> - * <td>{@code 5000} ms specified in {@link #DFLT_DISK_SCAN_FREQUENCY DFLT_DISK_SCAN_FREQUENCY}.</td> - * </tr> * </table> * <h2 class="header">File URI Example</h2> * The following example will scan {@code 'c:/Program files/ignite/deployment'} @@ -192,12 +186,6 @@ import java.util.Map.*; * <th>Optional</th> * <th>Default</th> * </tr> - * <tr> - * <td>freq</td> - * <td>File directory scan frequency in milliseconds.</td> - * <td>Yes</td> - * <td>{@code 5000} ms specified in {@link #DFLT_DISK_SCAN_FREQUENCY DFLT_DISK_SCAN_FREQUENCY}.</td> - * </tr> * </table> * <h2 class="header">Classes URI Example</h2> * The following example will scan {@code 'c:/Program files/ignite/deployment'} @@ -280,12 +268,6 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, */ public static final String DFLT_DEPLOY_DIR = "deployment/file"; - /** Default scan frequency for {@code file://} and {@code classes://} protocols (value is {@code 5000}). */ - public static final int DFLT_DISK_SCAN_FREQUENCY = 5000; - - /** Default scan frequency for {@code http://} protocol (value is {@code 300000}). */ - public static final int DFLT_HTTP_SCAN_FREQUENCY = 300000; - /** Default task description file path and name (value is {@code META-INF/ignite.xml}). */ public static final String XML_DESCRIPTOR_PATH = "META-INF/ignite.xml"; @@ -312,15 +294,14 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, /** */ @SuppressWarnings({"CollectionDeclaredAsConcreteClass"}) - private final LinkedList<GridUriDeploymentUnitDescriptor> unitLoaders = - new LinkedList<>(); + private final LinkedList<GridUriDeploymentUnitDescriptor> unitLoaders = new LinkedList<>(); /** */ @SuppressWarnings({"TypeMayBeWeakened"}) private final LastTimeUnitDescriptorComparator unitComp = new LastTimeUnitDescriptorComparator(); - /** List of scanners. Every URI has it's own scanner. */ - private final Collection<GridUriDeploymentScanner> scanners = new ArrayList<>(); + /** List of scanner managers. Every URI has it's own manager. */ + private final Collection<UriDeploymentScannerManager> mgrs = new ArrayList<>(); /** Whether URIs should be encoded or not. */ private boolean encodeUri = true; @@ -342,6 +323,9 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, @SuppressWarnings("UnusedDeclaration") private boolean delayOnNewOrUpdatedFile; + /** Configured scanners. */ + private UriDeploymentScanner[] scanners; + /** * Sets absolute path to temporary directory which will be used by * deployment SPI to keep all deployed classes in. @@ -421,17 +405,36 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, this.lsnr = lsnr; } + /** + * Gets scanners. + * + * @return Scanners. + */ + public UriDeploymentScanner[] getScanners() { + return scanners; + } + + /** + * Sets scanners. + * + * @param scanners Scanners. + */ + @IgniteSpiConfiguration(optional = true) + public void setScanners(UriDeploymentScanner... scanners) { + this.scanners = scanners; + } + /** {@inheritDoc} */ @Override public void spiStop() throws IgniteSpiException { - for (GridUriDeploymentScanner scanner : scanners) - scanner.cancel(); + for (UriDeploymentScannerManager mgr : mgrs) + mgr.cancel(); - for (GridUriDeploymentScanner scanner : scanners) - scanner.join(); + for (UriDeploymentScannerManager mgr : mgrs) + mgr.join(); // Clear inner collections. uriEncodedList.clear(); - scanners.clear(); + mgrs.clear(); List<ClassLoader> tmpClsLdrs; @@ -545,9 +548,15 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, } }; - for (URI uri : uriEncodedList) { - String proto = uri.getScheme(); + // Set default scanners if none are configured. + if (scanners == null) { + scanners = new UriDeploymentScanner[2]; + + scanners[0] = new UriDeploymentFileScanner(); + scanners[1] = new UriDeploymentHttpScanner(); + } + for (URI uri : uriEncodedList) { File file = new File(deployTmpDirPath); long freq = -1; @@ -559,29 +568,23 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, U.error(log, "Error parsing parameter value for frequency.", e); } - assert proto != null; - - GridUriDeploymentScanner scanner; + UriDeploymentScannerManager mgr = null; - switch (proto) { - case "file": - scanner = new GridUriDeploymentFileScanner(gridName, uri, file, freq > 0 ? freq : - DFLT_DISK_SCAN_FREQUENCY, filter, lsnr, log); - break; + for (UriDeploymentScanner scanner : scanners) { + if (scanner.acceptsURI(uri)) { + mgr = new UriDeploymentScannerManager(gridName, uri, file, freq > 0 ? freq : + scanner.getDefaultScanFrequency(), filter, lsnr, log, scanner); - case "http": - case "https": - scanner = new GridUriDeploymentHttpScanner(gridName, uri, file, freq > 0 ? freq : - DFLT_HTTP_SCAN_FREQUENCY, filter, lsnr, log); break; - - default: - throw new IgniteSpiException("Unsupported protocol: " + proto); + } } - scanners.add(scanner); + if (mgr == null) + throw new IgniteSpiException("Unsupported URI (please configure appropriate scanner): " + uri); + + mgrs.add(mgr); - scanner.start(); + mgr.start(); } // Ack parameters. @@ -589,7 +592,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, log.debug(configInfo("tmpDirPath", tmpDirPath)); log.debug(configInfo("uriList", uriList)); log.debug(configInfo("encodeUri", encodeUri)); - log.debug(configInfo("scanners", scanners)); + log.debug(configInfo("scanners", mgrs)); } // Ack ok start. @@ -615,7 +618,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, if (userInfo != null) { String[] arr = userInfo.split(";"); - if (arr != null && arr.length > 0) + if (arr.length > 0) for (String el : arr) if (el.startsWith("freq=")) return Long.parseLong(el.substring(5)); @@ -629,7 +632,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, @Override public DeploymentResource findResource(String rsrcName) { assert rsrcName != null; - // Wait until all scanners finish their first scanning. + // Wait until all scanner managers finish their first scanning. try { synchronized (mux) { while (!isFirstScanFinished(firstScanCntr)) @@ -637,7 +640,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, } } catch (InterruptedException e) { - U.error(log, "Failed to wait while all scanners finish their first scanning.", e); + U.error(log, "Failed to wait while all scanner managers finish their first scanning.", e); Thread.currentThread().interrupt(); @@ -945,8 +948,6 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, for (String uri : uriList) { assertParameter(uri != null, "uriList.get(X) != null"); - assert uri != null; - String encUri = encodeUri(uri.replaceAll("\\\\", "/")); URI uriObj; @@ -969,7 +970,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, } /** - * Add configuration for file scanner {@link GridUriDeploymentFileScanner}. + * Add configuration for file scanner. * * @throws org.apache.ignite.spi.IgniteSpiException Thrown if default URI syntax is incorrect. */ @@ -1075,7 +1076,7 @@ public class UriDeploymentSpi extends IgniteSpiAdapter implements DeploymentSpi, GridUriDeploymentUnitDescriptor desc = iter.next(); assert !newDesc.getClassLoader().equals(desc.getClassLoader()) : - "Uri scanners always create new class loader for every GAR file: " + newDesc; + "URI scanners always create new class loader for every GAR file: " + newDesc; // Only for GAR files. Undeploy all for overwritten GAR files. if (desc.getType() == GridUriDeploymentUnitDescriptor.Type.FILE && http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/GridUriDeploymentScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/GridUriDeploymentScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/GridUriDeploymentScanner.java deleted file mode 100644 index e20d156..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/GridUriDeploymentScanner.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.tostring.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.spi.*; - -import java.io.*; -import java.net.*; - -/** - * Base deployment scanner implementation. It simplifies scanner implementation - * by providing loggers, executors and file names parsing methods. - */ -public abstract class GridUriDeploymentScanner { - /** Grid name. */ - private final String gridName; - - /** URI that scanner should looks after. */ - @GridToStringExclude - private final URI uri; - - /** Temporary deployment directory. */ - private final File deployDir; - - /** Scan frequency. */ - private final long freq; - - /** Found files filter. */ - private final FilenameFilter filter; - - /** Scanner listener which should be notified about changes. */ - private final GridUriDeploymentScannerListener lsnr; - - /** Logger. */ - private final IgniteLogger log; - - /** Scanner implementation. */ - private IgniteSpiThread scanner; - - /** Whether first scan completed or not. */ - private boolean firstScan = true; - - /** - * Scans URI for new, updated or deleted files. - */ - protected abstract void process(); - - /** - * Creates new scanner. - * - * @param gridName Grid name. - * @param uri URI which scanner should looks after. - * @param deployDir Temporary deployment directory. - * @param freq Scan frequency. - * @param filter Found files filter. - * @param lsnr Scanner listener which should be notifier about changes. - * @param log Logger. - */ - protected GridUriDeploymentScanner( - String gridName, - URI uri, - File deployDir, - long freq, - FilenameFilter filter, - GridUriDeploymentScannerListener lsnr, - IgniteLogger log) { - assert uri != null; - assert freq > 0; - assert deployDir != null; - assert filter != null; - assert log != null; - assert lsnr != null; - - this.gridName = gridName; - this.uri = uri; - this.deployDir = deployDir; - this.freq = freq; - this.filter = filter; - this.log = log.getLogger(getClass()); - this.lsnr = lsnr; - } - - /** - * Starts scanner. - */ - public void start() { - scanner = new IgniteSpiThread(gridName, "grid-uri-scanner", log) { - /** {@inheritDoc} */ - @SuppressWarnings({"BusyWait"}) - @Override protected void body() throws InterruptedException { - try { - while (!isInterrupted()) { - try { - process(); - } - finally { - // Do it in finally to avoid any hanging. - if (firstScan) { - firstScan = false; - - lsnr.onFirstScanFinished(); - } - } - - Thread.sleep(freq); - } - } - finally { - // Double check. If we were cancelled before anything has been scanned. - if (firstScan) { - firstScan = false; - - lsnr.onFirstScanFinished(); - } - } - } - }; - - scanner.start(); - - if (log.isDebugEnabled()) - log.debug("Grid URI deployment scanner started: " + this); - } - - /** - * Cancels scanner execution. - */ - public void cancel() { - U.interrupt(scanner); - } - - /** - * Joins scanner thread. - */ - public void join() { - U.join(scanner, log); - - if (log.isDebugEnabled()) - log.debug("Grid URI deployment scanner stopped: " + this); - } - - /** - * Tests whether scanner was cancelled before or not. - * - * @return {@code true} if scanner was cancelled and {@code false} - * otherwise. - */ - protected boolean isCancelled() { - assert scanner != null; - - return scanner.isInterrupted(); - } - - /** - * Creates temp file in temp directory. - * - * @param fileName File name. - * @param tmpDir dir to creating file. - * @return created file. - * @throws IOException if error occur. - */ - protected File createTempFile(String fileName, File tmpDir) throws IOException { - assert fileName != null; - - int idx = fileName.lastIndexOf('.'); - - if (idx == -1) - idx = fileName.length(); - - String prefix = fileName.substring(0, idx); - if (idx < 3) { // Prefix must be at least 3 characters long. See File.createTempFile(...). - prefix += "___"; - } - - String suffix = fileName.substring(idx); - - return File.createTempFile(prefix, suffix, tmpDir); - } - - /** - * Gets file URI for the given file name. It extends any given name with {@link #uri}. - * - * @param name File name. - * @return URI for the given file name. - */ - protected String getFileUri(String name) { - assert name != null; - - String fileUri = uri.toString(); - - fileUri = fileUri.length() > 0 && fileUri.charAt(fileUri.length() - 1) == '/' ? fileUri + name : - fileUri + '/' + name; - - return fileUri; - } - - /** - * Tests whether first scan completed or not. - * - * @return {@code true} if first scan has been already completed and - * {@code false} otherwise. - */ - protected boolean isFirstScan() { - return firstScan; - } - - /** - * Gets deployment URI. - * - * @return Deployment URI. - */ - protected final URI getUri() { - return uri; - } - - /** - * Gets deployment frequency. - * - * @return Deployment frequency. - */ - protected final long getFrequency() { - return freq; - } - - /** - * Gets temporary deployment directory. - * - * @return Temporary deployment directory. - */ - protected final File getDeployDirectory() { - return deployDir; - } - - /** - * Gets filter for found files. Before {@link #lsnr} is notified about - * changes with certain file last should be accepted by filter. - * - * @return New, updated or deleted file filter. - */ - protected final FilenameFilter getFilter() { - return filter; - } - - /** - * Gets deployment listener. - * - * @return Listener which should be notified about all deployment events - * by scanner. - */ - protected final GridUriDeploymentScannerListener getListener() { - return lsnr; - } - - /** - * Gets scanner logger. - * - * @return Logger. - */ - protected final IgniteLogger getLogger() { - return log; - } - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridUriDeploymentScanner.class, this, - "uri", uri != null ? U.hidePassword(uri.toString()) : null); - } -} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScanner.java new file mode 100644 index 0000000..e33351c --- /dev/null +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScanner.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.deployment.uri.scanners; + +import java.net.*; + +/** + * URI deployment scanner. + */ +public interface UriDeploymentScanner { + /** + * Check whether scanner is able to process the given URI. + * + * @param uri URI. + * @return {@code true} if scanner is able to process the URI. + */ + boolean acceptsURI(URI uri); + + /** + * Scan the given URI. + * + * @param scanCtx Scan context. + */ + void scan(UriDeploymentScannerContext scanCtx); + + /** + * Gets default scan frequency in milliseconds. + * + * @return Default scan frequency. + */ + long getDefaultScanFrequency(); +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerContext.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerContext.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerContext.java new file mode 100644 index 0000000..9867f0f --- /dev/null +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerContext.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.deployment.uri.scanners; + +import org.apache.ignite.*; +import org.apache.ignite.spi.deployment.uri.scanners.*; + +import java.io.*; +import java.net.*; + +/** + * Deployment scanner context. + */ +public interface UriDeploymentScannerContext { + /** + * Creates temp file in temp directory. + * + * @param fileName File name. + * @param tmpDir dir to creating file. + * @return created file. + * @throws IOException if error occur. + */ + File createTempFile(String fileName, File tmpDir) throws IOException; + + /** + * Gets temporary deployment directory. + * + * @return Temporary deployment directory. + */ + File getDeployDirectory(); + + /** + * Gets filter for found files. Before listener is notified about + * changes with certain file last should be accepted by filter. + * + * @return New, updated or deleted file filter. + */ + FilenameFilter getFilter(); + + /** + * Gets deployment listener. + * + * @return Listener which should be notified about all deployment events + * by scanner. + */ + GridUriDeploymentScannerListener getListener(); + + /** + * Gets scanner logger. + * + * @return Logger. + */ + IgniteLogger getLogger(); + + /** + * Gets deployment URI. + * + * @return Deployment URI. + */ + URI getUri(); + + /** + * Tests whether scanner was cancelled before or not. + * + * @return {@code true} if scanner was cancelled and {@code false} + * otherwise. + */ + boolean isCancelled(); + + /** + * Tests whether first scan completed or not. + * + * @return {@code true} if first scan has been already completed and + * {@code false} otherwise. + */ + boolean isFirstScan(); +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerManager.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerManager.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerManager.java new file mode 100644 index 0000000..0d959d9 --- /dev/null +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/UriDeploymentScannerManager.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.deployment.uri.scanners; + +import org.apache.ignite.*; +import org.apache.ignite.internal.util.tostring.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.spi.*; + +import java.io.*; +import java.net.*; + +/** + * URI deployment scanner manager. + */ +public class UriDeploymentScannerManager implements UriDeploymentScannerContext { + /** Grid name. */ + private final String gridName; + + /** URI that scanner should looks after. */ + @GridToStringExclude + private final URI uri; + + /** Temporary deployment directory. */ + private final File deployDir; + + /** Scan frequency. */ + private final long freq; + + /** Found files filter. */ + private final FilenameFilter filter; + + /** Scanner listener which should be notified about changes. */ + private final GridUriDeploymentScannerListener lsnr; + + /** Logger. */ + private final IgniteLogger log; + + /** Underlying scanner. */ + private final UriDeploymentScanner scanner; + + /** Scanner implementation. */ + private IgniteSpiThread scannerThread; + + /** Whether first scan completed or not. */ + private boolean firstScan = true; + + /** + * Creates new scanner. + * + * @param gridName Grid name. + * @param uri URI which scanner should looks after. + * @param deployDir Temporary deployment directory. + * @param freq Scan frequency. + * @param filter Found files filter. + * @param lsnr Scanner listener which should be notifier about changes. + * @param log Logger. + * @param scanner Scanner. + */ + public UriDeploymentScannerManager( + String gridName, + URI uri, + File deployDir, + long freq, + FilenameFilter filter, + GridUriDeploymentScannerListener lsnr, + IgniteLogger log, + UriDeploymentScanner scanner) { + assert uri != null; + assert freq > 0; + assert deployDir != null; + assert filter != null; + assert log != null; + assert lsnr != null; + assert scanner != null; + + this.gridName = gridName; + this.uri = uri; + this.deployDir = deployDir; + this.freq = freq; + this.filter = filter; + this.log = log.getLogger(getClass()); + this.lsnr = lsnr; + this.scanner = scanner; + } + + /** + * Starts scanner. + */ + public void start() { + scannerThread = new IgniteSpiThread(gridName, "grid-uri-scanner", log) { + /** {@inheritDoc} */ + @SuppressWarnings({"BusyWait"}) + @Override protected void body() throws InterruptedException { + try { + while (!isInterrupted()) { + try { + scanner.scan(UriDeploymentScannerManager.this); + } + finally { + // Do it in finally to avoid any hanging. + if (firstScan) { + firstScan = false; + + lsnr.onFirstScanFinished(); + } + } + + Thread.sleep(freq); + } + } + finally { + // Double check. If we were cancelled before anything has been scanned. + if (firstScan) { + firstScan = false; + + lsnr.onFirstScanFinished(); + } + } + } + }; + + scannerThread.start(); + + if (log.isDebugEnabled()) + log.debug("Grid URI deployment scanner started: " + this); + } + + /** + * Cancels scanner execution. + */ + public void cancel() { + U.interrupt(scannerThread); + } + + /** + * Joins scanner thread. + */ + public void join() { + U.join(scannerThread, log); + + if (log.isDebugEnabled()) + log.debug("Grid URI deployment scanner stopped: " + this); + } + + /** {@inheritDoc} */ + public boolean isCancelled() { + assert scannerThread != null; + + return scannerThread.isInterrupted(); + } + + /** {@inheritDoc} */ + public File createTempFile(String fileName, File tmpDir) throws IOException { + assert fileName != null; + + int idx = fileName.lastIndexOf('.'); + + if (idx == -1) + idx = fileName.length(); + + String prefix = fileName.substring(0, idx); + if (idx < 3) { // Prefix must be at least 3 characters long. See File.createTempFile(...). + prefix += "___"; + } + + String suffix = fileName.substring(idx); + + return File.createTempFile(prefix, suffix, tmpDir); + } + + /** {@inheritDoc} */ + public boolean isFirstScan() { + return firstScan; + } + + /** {@inheritDoc} */ + public URI getUri() { + return uri; + } + + /** {@inheritDoc} */ + public File getDeployDirectory() { + return deployDir; + } + + /** {@inheritDoc} */ + public FilenameFilter getFilter() { + return filter; + } + + /** {@inheritDoc} */ + public GridUriDeploymentScannerListener getListener() { + return lsnr; + } + + /** {@inheritDoc} */ + public IgniteLogger getLogger() { + return log; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(UriDeploymentScannerManager.class, this, "uri", U.hidePassword(uri.toString())); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridUriDeploymentFileScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridUriDeploymentFileScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridUriDeploymentFileScanner.java deleted file mode 100644 index fc81c13..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/GridUriDeploymentFileScanner.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners.file; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.lang.*; -import org.apache.ignite.internal.util.typedef.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.spi.*; -import org.apache.ignite.spi.deployment.uri.scanners.*; - -import java.io.*; -import java.net.*; -import java.util.*; - -/** - * Scanner that processes all URIs with "file" scheme. Usually URI point to - * certain directory or file and scanner is in charge of watching all changes - * (file deletion, creation and so on) and sending notification to the listener - * about every change. - */ -public class GridUriDeploymentFileScanner extends GridUriDeploymentScanner { - /** Scanning directory or file. */ - private File scanDir; - - /** Cache of found GAR-files or GAR-directories to check if any of it has been updated. */ - private Map<File, Long> tstampCache = new HashMap<>(); - - /** Cache of found files in GAR-folder to check if any of it has been updated. */ - private Map<File, Map<File, Long>> garDirFilesTstampCache = new HashMap<>(); - - /** */ - private FileFilter garFilter; - - /** */ - private FileFilter garDirFilesFilter; - - /** - * Creates new instance of scanner with given name. - * - * @param gridName Grid name. - * @param uri URI which scanner should look after. - * @param deployDir Temporary deployment directory. - * @param freq Scan frequency. - * @param filter Found files filter. - * @param lsnr Scanner listener which should be notifier about changes. - * @param log Logger. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown if URI is {@code null} or is not a - * directory. - */ - public GridUriDeploymentFileScanner( - String gridName, - URI uri, - File deployDir, - long freq, - FilenameFilter filter, - GridUriDeploymentScannerListener lsnr, - IgniteLogger log) throws IgniteSpiException { - super(gridName, uri, deployDir, freq, filter, lsnr, log); - - initialize(uri); - } - - /** - * Initializes scanner by parsing given URI and extracting scanning - * directory path and creating file filters. - * - * @param uri Scanning URI with "file" scheme. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown if URI is {@code null} or is not a - * directory. - */ - private void initialize(URI uri) throws IgniteSpiException { - assert "file".equals(getUri().getScheme()); - - String scanDirPath = uri.getPath(); - - if (scanDirPath != null) - scanDir = new File(scanDirPath); - - if (scanDir == null || !scanDir.isDirectory()) { - scanDir = null; - - throw new IgniteSpiException("URI is either not provided or is not a directory: " + - U.hidePassword(uri.toString())); - } - - garFilter = new FileFilter() { - /** {@inheritDoc} */ - @Override public boolean accept(File pathname) { - return getFilter().accept(null, pathname.getName()); - } - }; - - garDirFilesFilter = new FileFilter() { - /** {@inheritDoc} */ - @Override public boolean accept(File pathname) { - // Allow all files in GAR-directory. - return pathname.isFile(); - } - }; - } - - /** - * Handles changes in scanning directory by tracking files modification date. - * Checks files modification date against those one that was collected before - * and notifies listener about every changed or deleted file. - */ - @Override protected void process() { - final Set<File> foundFiles = isFirstScan() ? new HashSet<File>() : U.<File>newHashSet(tstampCache.size()); - - GridDeploymentFileHandler hnd = new GridDeploymentFileHandler() { - /** {@inheritDoc} */ - @Override public void handle(File file) { - foundFiles.add(file); - - handleFile(file); - } - }; - - // Scan directory for deploy units. - GridDeploymentFolderScannerHelper.scanFolder(scanDir, garFilter, hnd); - - // Print warning if no GAR-units found first time. - if (isFirstScan() && foundFiles.isEmpty()) - U.warn(getLogger(), "No GAR-units found in: " + U.hidePassword(getUri().toString())); - - if (!isFirstScan()) { - Collection<File> deletedFiles = new HashSet<>(tstampCache.keySet()); - - deletedFiles.removeAll(foundFiles); - - if (!deletedFiles.isEmpty()) { - List<String> uris = new ArrayList<>(); - - for (File file : deletedFiles) { - uris.add(getFileUri(file.getAbsolutePath())); - } - - // Clear cache. - tstampCache.keySet().removeAll(deletedFiles); - - garDirFilesTstampCache.keySet().removeAll(deletedFiles); - - getListener().onDeletedFiles(uris); - } - } - } - - /** - * Tests whether given directory or file was changed since last check and if so - * copies all directory sub-folders and files or file itself to the deployment - * directory and than notifies listener about new or updated files. - * - * @param file Scanning directory or file. - */ - private void handleFile(File file) { - boolean changed; - - Long lastMod; - - if (file.isDirectory()) { - GridTuple<Long> dirLastModified = F.t(file.lastModified()); - - changed = checkGarDirectoryChanged(file, dirLastModified); - - lastMod = dirLastModified.get(); - } - else { - lastMod = tstampCache.get(file); - - changed = lastMod == null || lastMod != file.lastModified(); - - lastMod = file.lastModified(); - } - - // If file is new or has been modified. - if (changed) { - tstampCache.put(file, lastMod); - - if (getLogger().isDebugEnabled()) - getLogger().debug("Discovered deployment file or directory: " + file); - - String fileName = file.getName(); - - try { - File cpFile = createTempFile(fileName, getDeployDirectory()); - - // Delete file when JVM stopped. - cpFile.deleteOnExit(); - - if (file.isDirectory()) { - cpFile = new File(cpFile.getParent(), "dir_" + cpFile.getName()); - - // Delete directory when JVM stopped. - cpFile.deleteOnExit(); - } - - // Copy file to deploy directory. - U.copy(file, cpFile, true); - - String fileUri = getFileUri(file.getAbsolutePath()); - - getListener().onNewOrUpdatedFile(cpFile, fileUri, lastMod); - } - catch (IOException e) { - U.error(getLogger(), "Error saving file: " + fileName, e); - } - } - } - - /** - * Tests whether certain directory was changed since given modification date. - * It scans all directory files one by one and compares their modification - * dates with those ones that was collected before. - * <p> - * If at least one file was changed (has modification date after given one) - * whole directory is considered as modified. - * - * @param dir Scanning directory. - * @param lastModified Last calculated Directory modification date. - * @return {@code true} if directory was changed since last check and - * {@code false} otherwise. - */ - private boolean checkGarDirectoryChanged(File dir, final GridTuple<Long> lastModified) { - final Map<File, Long> clssTstampCache; - - boolean firstScan = false; - - if (!garDirFilesTstampCache.containsKey(dir)) { - firstScan = true; - - garDirFilesTstampCache.put(dir, clssTstampCache = new HashMap<>()); - } - else - clssTstampCache = garDirFilesTstampCache.get(dir); - - assert clssTstampCache != null; - - final GridTuple<Boolean> changed = F.t(false); - - final Set<File> foundFiles = firstScan ? new HashSet<File>() : U.<File>newHashSet(clssTstampCache.size()); - - GridDeploymentFileHandler hnd = new GridDeploymentFileHandler() { - @Override public void handle(File file) { - foundFiles.add(file); - - Long fileLastModified = clssTstampCache.get(file); - - if (fileLastModified == null || fileLastModified != file.lastModified()) { - clssTstampCache.put(file, fileLastModified = file.lastModified()); - - changed.set(true); - } - - // Calculate last modified file in folder. - if (fileLastModified > lastModified.get()) - lastModified.set(fileLastModified); - } - }; - - // Scan GAR-directory for changes. - GridDeploymentFolderScannerHelper.scanFolder(dir, garDirFilesFilter, hnd); - - // Clear cache for deleted files. - if (!firstScan && clssTstampCache.keySet().retainAll(foundFiles)) - changed.set(true); - - return changed.get(); - } - - /** - * Converts given file name to the URI with "file" scheme. - * - * @param name File name to be converted. - * @return File name with "file://" prefix. - */ - @Override protected String getFileUri(String name) { - assert name != null; - - name = name.replace("\\","/"); - - return "file://" + (name.charAt(0) == '/' ? "" : '/') + name; - } - - /** {@inheritDoc} */ - @Override public String toString() { - StringBuilder buf = new StringBuilder(); - - buf.append(getClass().getName()).append(" ["); - buf.append("scanDir=").append(scanDir); - buf.append(']'); - - return buf.toString(); - } -} - http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/UriDeploymentFileScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/UriDeploymentFileScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/UriDeploymentFileScanner.java new file mode 100644 index 0000000..2cbe38b --- /dev/null +++ b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/file/UriDeploymentFileScanner.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.deployment.uri.scanners.file; + +import org.apache.ignite.internal.util.lang.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.spi.*; +import org.apache.ignite.spi.deployment.uri.scanners.*; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.concurrent.*; + +/** + * URI deployment file scanner. + */ +public class UriDeploymentFileScanner implements UriDeploymentScanner { + /** Default scan frequency. */ + public static final int DFLT_SCAN_FREQ = 5000; + + /** Per-URI contexts. */ + private final ConcurrentHashMap<URI, URIContext> uriCtxs = new ConcurrentHashMap<>(); + + /** {@inheritDoc} */ + @Override public boolean acceptsURI(URI uri) { + String proto = uri.getScheme().toLowerCase(); + + return "file".equals(proto); + } + + /** {@inheritDoc} */ + @Override public void scan(UriDeploymentScannerContext scanCtx) { + URI uri = scanCtx.getUri(); + + URIContext uriCtx = uriCtxs.get(uri); + + if (uriCtx == null) { + uriCtx = createUriContext(uri, scanCtx); + + URIContext oldUriCtx = uriCtxs.putIfAbsent(uri, uriCtx); + + if (oldUriCtx != null) + uriCtx = oldUriCtx; + } + + uriCtx.scan(scanCtx); + } + + /** {@inheritDoc} */ + @Override public long getDefaultScanFrequency() { + return DFLT_SCAN_FREQ; + } + + /** + * Create context for the given URI. + * + * @param uri URI. + * @param scanCtx Scanner context. + * @return URI context. + */ + private URIContext createUriContext(URI uri, final UriDeploymentScannerContext scanCtx) { + String scanDirPath = uri.getPath(); + + File scanDir = null; + + if (scanDirPath != null) + scanDir = new File(scanDirPath); + + if (scanDir == null || !scanDir.isDirectory()) + throw new IgniteSpiException("URI is either not provided or is not a directory: " + + U.hidePassword(uri.toString())); + + FileFilter garFilter = new FileFilter() { + /** {@inheritDoc} */ + @Override public boolean accept(File pathname) { + return scanCtx.getFilter().accept(null, pathname.getName()); + } + }; + + FileFilter garDirFilesFilter = new FileFilter() { + /** {@inheritDoc} */ + @Override public boolean accept(File pathname) { + // Allow all files in GAR-directory. + return pathname.isFile(); + } + }; + + return new URIContext(scanDir, garFilter, garDirFilesFilter); + } + + /** + * Converts given file name to the URI with "file" scheme. + * + * @param name File name to be converted. + * @return File name with "file://" prefix. + */ + private static String getFileUri(String name) { + assert name != null; + + name = name.replace("\\","/"); + + return "file://" + (name.charAt(0) == '/' ? "" : '/') + name; + } + + /** + * Context for the given URI. + */ + private static class URIContext { + /** Scanning directory or file. */ + private final File scanDir; + + /** GAR filter. */ + private final FileFilter garFilter; + + /** GAR directory files filter. */ + private final FileFilter garDirFilesFilter; + + /** Cache of found GAR-files or GAR-directories to check if any of it has been updated. */ + private final Map<File, Long> tstampCache = new HashMap<>(); + + /** Cache of found files in GAR-folder to check if any of it has been updated. */ + private final Map<File, Map<File, Long>> garDirFilesTstampCache = new HashMap<>(); + + /** + * Constructor. + * + * @param scanDir Scan directory. + * @param garFilter Gar filter. + * @param garDirFilesFilter GAR directory files filter. + */ + private URIContext(File scanDir, FileFilter garFilter, FileFilter garDirFilesFilter) { + this.scanDir = scanDir; + this.garFilter = garFilter; + this.garDirFilesFilter = garDirFilesFilter; + } + + /** + * Perform scan. + * + * @param scanCtx Scan context. + */ + private void scan(final UriDeploymentScannerContext scanCtx) { + final Set<File> foundFiles = scanCtx.isFirstScan() ? + new HashSet<File>() : U.<File>newHashSet(tstampCache.size()); + + GridDeploymentFileHandler hnd = new GridDeploymentFileHandler() { + /** {@inheritDoc} */ + @Override public void handle(File file) { + foundFiles.add(file); + + handleFile(file, scanCtx); + } + }; + + // Scan directory for deploy units. + GridDeploymentFolderScannerHelper.scanFolder(scanDir, garFilter, hnd); + + // Print warning if no GAR-units found first time. + if (scanCtx.isFirstScan() && foundFiles.isEmpty()) + U.warn(scanCtx.getLogger(), "No GAR-units found in: " + U.hidePassword(scanCtx.getUri().toString())); + + if (!scanCtx.isFirstScan()) { + Collection<File> deletedFiles = new HashSet<>(tstampCache.keySet()); + + deletedFiles.removeAll(foundFiles); + + if (!deletedFiles.isEmpty()) { + List<String> uris = new ArrayList<>(); + + for (File file : deletedFiles) { + uris.add(getFileUri(file.getAbsolutePath())); + } + + // Clear cache. + tstampCache.keySet().removeAll(deletedFiles); + + garDirFilesTstampCache.keySet().removeAll(deletedFiles); + + scanCtx.getListener().onDeletedFiles(uris); + } + } + } + + /** + * Tests whether given directory or file was changed since last check and if so + * copies all directory sub-folders and files or file itself to the deployment + * directory and than notifies listener about new or updated files. + * + * @param file Scanning directory or file. + * @param ctx Scanner context. + */ + private void handleFile(File file, UriDeploymentScannerContext ctx) { + boolean changed; + + Long lastMod; + + if (file.isDirectory()) { + GridTuple<Long> dirLastModified = F.t(file.lastModified()); + + changed = checkGarDirectoryChanged(file, dirLastModified); + + lastMod = dirLastModified.get(); + } + else { + lastMod = tstampCache.get(file); + + changed = lastMod == null || lastMod != file.lastModified(); + + lastMod = file.lastModified(); + } + + // If file is new or has been modified. + if (changed) { + tstampCache.put(file, lastMod); + + if (ctx.getLogger().isDebugEnabled()) + ctx.getLogger().debug("Discovered deployment file or directory: " + file); + + String fileName = file.getName(); + + try { + File cpFile = ctx.createTempFile(fileName, ctx.getDeployDirectory()); + + // Delete file when JVM stopped. + cpFile.deleteOnExit(); + + if (file.isDirectory()) { + cpFile = new File(cpFile.getParent(), "dir_" + cpFile.getName()); + + // Delete directory when JVM stopped. + cpFile.deleteOnExit(); + } + + // Copy file to deploy directory. + U.copy(file, cpFile, true); + + String fileUri = getFileUri(file.getAbsolutePath()); + + assert lastMod != null; + + ctx.getListener().onNewOrUpdatedFile(cpFile, fileUri, lastMod); + } + catch (IOException e) { + U.error(ctx.getLogger(), "Error saving file: " + fileName, e); + } + } + } + + /** + * Tests whether certain directory was changed since given modification date. + * It scans all directory files one by one and compares their modification + * dates with those ones that was collected before. + * <p> + * If at least one file was changed (has modification date after given one) + * whole directory is considered as modified. + * + * @param dir Scanning directory. + * @param lastModified Last calculated Directory modification date. + * @return {@code true} if directory was changed since last check and + * {@code false} otherwise. + */ + @SuppressWarnings("ConstantConditions") + private boolean checkGarDirectoryChanged(File dir, final GridTuple<Long> lastModified) { + final Map<File, Long> clssTstampCache; + + boolean firstScan = false; + + if (!garDirFilesTstampCache.containsKey(dir)) { + firstScan = true; + + garDirFilesTstampCache.put(dir, clssTstampCache = new HashMap<>()); + } + else + clssTstampCache = garDirFilesTstampCache.get(dir); + + assert clssTstampCache != null; + + final GridTuple<Boolean> changed = F.t(false); + + final Set<File> foundFiles = firstScan ? new HashSet<File>() : U.<File>newHashSet(clssTstampCache.size()); + + GridDeploymentFileHandler hnd = new GridDeploymentFileHandler() { + @Override public void handle(File file) { + foundFiles.add(file); + + Long fileLastModified = clssTstampCache.get(file); + + if (fileLastModified == null || fileLastModified != file.lastModified()) { + clssTstampCache.put(file, fileLastModified = file.lastModified()); + + changed.set(true); + } + + // Calculate last modified file in folder. + if (fileLastModified > lastModified.get()) + lastModified.set(fileLastModified); + } + }; + + // Scan GAR-directory for changes. + GridDeploymentFolderScannerHelper.scanFolder(dir, garDirFilesFilter, hnd); + + // Clear cache for deleted files. + if (!firstScan && clssTstampCache.keySet().retainAll(foundFiles)) + changed.set(true); + + return changed.get(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0755231c/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java ---------------------------------------------------------------------- diff --git a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java b/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java deleted file mode 100644 index 49ac343..0000000 --- a/modules/urideploy/src/main/java/org/apache/ignite/spi/deployment/uri/scanners/http/GridUriDeploymentHttpScanner.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.deployment.uri.scanners.http; - -import org.apache.ignite.*; -import org.apache.ignite.internal.util.tostring.*; -import org.apache.ignite.internal.util.typedef.*; -import org.apache.ignite.internal.util.typedef.internal.*; -import org.apache.ignite.spi.*; -import org.apache.ignite.spi.deployment.uri.scanners.*; -import org.jetbrains.annotations.*; -import org.w3c.dom.*; -import org.w3c.tidy.*; - -import javax.net.ssl.*; -import java.io.*; -import java.net.*; -import java.security.*; -import java.security.cert.*; -import java.util.*; - -/** - * URI deployment HTTP scanner. - */ -public class GridUriDeploymentHttpScanner extends GridUriDeploymentScanner { - /** Secure socket protocol to use. */ - private static final String PROTOCOL = "TLS"; - - /** */ - @GridToStringExclude - private URL scanDir; - - /** Cache of found files to check if any of it has been updated. */ - private Map<String, Long> tstampCache = new HashMap<>(); - - /** */ - @GridToStringExclude - private final Tidy tidy; - - /** Outgoing data SSL socket factory. */ - private SSLSocketFactory sockFactory; - - /** - * @param gridName Grid instance name. - * @param uri HTTP URI. - * @param deployDir Deployment directory. - * @param freq Scanner frequency. - * @param filter Filename filter. - * @param lsnr Deployment listener. - * @param log Logger to use. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error. - */ - public GridUriDeploymentHttpScanner( - String gridName, - URI uri, - File deployDir, - long freq, - FilenameFilter filter, - GridUriDeploymentScannerListener lsnr, - IgniteLogger log) throws IgniteSpiException { - super(gridName, uri, deployDir, freq, filter, lsnr, log); - - initialize(uri); - - tidy = new Tidy(); - - tidy.setQuiet(true); - tidy.setOnlyErrors(true); - tidy.setShowWarnings(false); - tidy.setInputEncoding("UTF8"); - tidy.setOutputEncoding("UTF8"); - } - - /** - * @param uri HTTP URI. - * @throws org.apache.ignite.spi.IgniteSpiException Thrown in case of any error. - */ - private void initialize(URI uri) throws IgniteSpiException { - assert "http".equals(uri.getScheme()) || "https".equals(uri.getScheme()); - - try { - scanDir = new URL(uri.getScheme(), uri.getHost(), uri.getPort(), uri.getPath()); - } - catch (MalformedURLException e) { - scanDir = null; - - throw new IgniteSpiException("Wrong value for scanned HTTP directory with URI: " + uri, e); - } - - try { - if ("https".equals(uri.getScheme())) { - // Set up socket factory to do authentication. - SSLContext ctx = SSLContext.getInstance(PROTOCOL); - - ctx.init(null, getTrustManagers(), null); - - sockFactory = ctx.getSocketFactory(); - } - } - catch (NoSuchAlgorithmException e) { - throw new IgniteSpiException("Failed to initialize SSL context. URI: " + uri, e); - } - catch (KeyManagementException e) { - throw new IgniteSpiException("Failed to initialize SSL context. URI:" + uri, e); - } - } - - /** {@inheritDoc} */ - @Override protected void process() { - Collection<String> foundFiles = U.newHashSet(tstampCache.size()); - - long start = U.currentTimeMillis(); - - processHttp(foundFiles); - - if (getLogger().isDebugEnabled()) - getLogger().debug("HTTP scanner time in ms: " + (U.currentTimeMillis() - start)); - - if (!isFirstScan()) { - Collection<String> deletedFiles = new HashSet<>(tstampCache.keySet()); - - deletedFiles.removeAll(foundFiles); - - if (!deletedFiles.isEmpty()) { - List<String> uris = new ArrayList<>(); - - for (String file : deletedFiles) { - uris.add(getFileUri(getFileName(file))); - } - - tstampCache.keySet().removeAll(deletedFiles); - - getListener().onDeletedFiles(uris); - } - } - } - - /** - * @param files Files to process. - */ - private void processHttp(Collection<String> files) { - Set<String> urls = getUrls(scanDir); - - for (String url : urls) { - String fileName = getFileName(url); - - if (getFilter().accept(null, fileName)) { - files.add(url); - - Long lastModified = tstampCache.get(url); - - InputStream in = null; - OutputStream out = null; - - File file = null; - - try { - URLConnection conn = new URL(url).openConnection(); - - if (conn instanceof HttpsURLConnection) { - HttpsURLConnection httpsConn = (HttpsURLConnection)conn; - - httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); - - assert sockFactory != null; - - // Initialize socket factory. - httpsConn.setSSLSocketFactory(sockFactory); - } - - if (lastModified != null) - conn.setIfModifiedSince(lastModified); - - in = conn.getInputStream(); - - long rcvLastModified = conn.getLastModified(); - - if (in == null || lastModified != null && (lastModified == rcvLastModified || - conn instanceof HttpURLConnection && - ((HttpURLConnection)conn).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED)) - continue; - - tstampCache.put(url, rcvLastModified); - - lastModified = rcvLastModified; - - if (getLogger().isDebugEnabled()) { - getLogger().debug("Discovered deployment file or directory: " + - U.hidePassword(url)); - } - - file = createTempFile(fileName, getDeployDirectory()); - - // Delete file when JVM stopped. - file.deleteOnExit(); - - out = new FileOutputStream(file); - - U.copy(in, out); - } - catch (IOException e) { - if (!isCancelled()) { - if (X.hasCause(e, ConnectException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " + - U.hidePassword(url)); - } - else if (X.hasCause(e, UnknownHostException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " + - U.hidePassword(url)); - } - else - U.error(getLogger(), "Failed to save file: " + fileName, e); - } - } - finally { - U.closeQuiet(in); - U.closeQuiet(out); - } - - if (file != null && file.exists() && file.length() > 0) - getListener().onNewOrUpdatedFile(file, getFileUri(fileName), lastModified); - } - } - } - - /** - * @param node XML element node. - * @param res Set of URLs in string format to populate. - * @param baseUrl Base URL. - */ - @SuppressWarnings( {"UnusedCatchParameter", "UnnecessaryFullyQualifiedName"}) - private void findReferences(org.w3c.dom.Node node, Set<String> res, URL baseUrl) { - if (node instanceof Element && "a".equals(node.getNodeName().toLowerCase())) { - Element element = (Element)node; - - String href = element.getAttribute("href"); - - if (href != null && !href.isEmpty()) { - URL url = null; - - try { - url = new URL(href); - } - catch (MalformedURLException e) { - try { - url = new URL(baseUrl.getProtocol(), baseUrl.getHost(), baseUrl.getPort(), - href.charAt(0) == '/' ? href : baseUrl.getFile() + '/' + href); - } - catch (MalformedURLException e1) { - U.error(getLogger(), "Skipping bad URL: " + url, e1); - } - } - - if (url != null) - res.add(url.toString()); - } - } - - NodeList childNodes = node.getChildNodes(); - - for (int i = 0; i < childNodes.getLength(); i++) - findReferences(childNodes.item(i), res, baseUrl); - } - - /** - * @param url Base URL. - * @return Set of referenced URLs in string format. - */ - private Set<String> getUrls(URL url) { - assert url != null; - - InputStream in = null; - - Set<String> urls = new HashSet<>(); - - Document dom = null; - - try { - URLConnection conn = url.openConnection(); - - if (conn instanceof HttpsURLConnection) { - HttpsURLConnection httpsConn = (HttpsURLConnection)conn; - - httpsConn.setHostnameVerifier(new DeploymentHostnameVerifier()); - - assert sockFactory != null; - - // Initialize socket factory. - httpsConn.setSSLSocketFactory(sockFactory); - } - - in = conn.getInputStream(); - - if (in == null) - throw new IOException("Failed to open connection: " + U.hidePassword(url.toString())); - - dom = tidy.parseDOM(in, null); - } - catch (IOException e) { - if (!isCancelled()) { - if (X.hasCause(e, ConnectException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (connection refused): " + - U.hidePassword(url.toString())); - } - else if (X.hasCause(e, UnknownHostException.class)) { - LT.warn(getLogger(), e, "Failed to connect to HTTP server (host is unknown): " + - U.hidePassword(url.toString())); - } - else - U.error(getLogger(), "Failed to get HTML page: " + U.hidePassword(url.toString()), e); - } - } - finally{ - U.closeQuiet(in); - } - - if (dom != null) - findReferences(dom, urls, url); - - return urls; - } - - /** - * @param url Base URL string format. - * @return File name extracted from {@code url} string format. - */ - private String getFileName(String url) { - assert url != null; - - return url.substring(url.lastIndexOf('/') + 1); - } - - /** - * Construct array with one trust manager which don't reject input certificates. - * - * @return Array with one X509TrustManager implementation of trust manager. - */ - private TrustManager[] getTrustManagers() { - return new TrustManager[]{ - new X509TrustManager() { - /** {@inheritDoc} */ - @Nullable @Override public X509Certificate[] getAcceptedIssuers() { return null; } - - /** {@inheritDoc} */ - @Override public void checkClientTrusted(X509Certificate[] certs, String authType) { - StringBuilder buf = new StringBuilder(); - - buf.append("Trust manager handle client certificates [authType="); - buf.append(authType); - buf.append(", certificates="); - - for (X509Certificate cert : certs) { - buf.append("{type="); - buf.append(cert.getType()); - buf.append(", principalName="); - buf.append(cert.getSubjectX500Principal().getName()); - buf.append('}'); - } - - buf.append(']'); - - if (getLogger().isDebugEnabled()) - getLogger().debug(buf.toString()); - } - - /** {@inheritDoc} */ - @Override public void checkServerTrusted(X509Certificate[] certs, String authType) { - StringBuilder buf = new StringBuilder(); - - buf.append("Trust manager handle server certificates [authType="); - buf.append(authType); - buf.append(", certificates="); - - for (X509Certificate cert : certs) { - buf.append("{type="); - buf.append(cert.getType()); - buf.append(", principalName="); - buf.append(cert.getSubjectX500Principal().getName()); - buf.append('}'); - } - - buf.append(']'); - - if (getLogger().isDebugEnabled()) - getLogger().debug(buf.toString()); - } - } - }; - } - - - /** {@inheritDoc} */ - @Override public String toString() { - return S.toString(GridUriDeploymentHttpScanner.class, this, - "scanDir", scanDir != null ? U.hidePassword(scanDir.toString()) : null); - } - - /** - * Verifier always return successful result for any host. - */ - private static class DeploymentHostnameVerifier implements HostnameVerifier { - /** {@inheritDoc} */ - @Override public boolean verify(String hostname, SSLSession ses) { - // Remote host trusted by default. - return true; - } - } -}
