Repository: camel Updated Branches: refs/heads/camel-2.14.x 638355862 -> 25d14cf78 refs/heads/master cb5237aba -> 3c53313d0
CAMEL-8266 Allows ignoring directories for SFTP/FTP having no read permission using ignoreFileNotFoundOrPermissionError Project: http://git-wip-us.apache.org/repos/asf/camel/repo Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/3c53313d Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/3c53313d Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/3c53313d Branch: refs/heads/master Commit: 3c53313d0fcb2fd31110cd47f3981ea98bf3081f Parents: cb5237a Author: Shreyas Purohit <shreyas.puro...@gmail.com> Authored: Sat Jan 24 20:31:14 2015 -0800 Committer: Shreyas Purohit <shreyas.puro...@gmail.com> Committed: Sat Jan 24 20:33:22 2015 -0800 ---------------------------------------------------------------------- .../component/file/remote/FtpConsumer.java | 20 +++- .../file/remote/RemoteFileConsumer.java | 50 +++++++- .../component/file/remote/SftpConsumer.java | 2 +- .../remote/RemoteFileIgnoreDoPollErrorTest.java | 118 +++++++++++++++++++ 4 files changed, 181 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/camel/blob/3c53313d/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java ---------------------------------------------------------------------- diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java index fff8e93..ba5d32c 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.Processor; import org.apache.camel.component.file.GenericFile; +import org.apache.camel.component.file.GenericFileOperationFailedException; import org.apache.camel.util.FileUtil; import org.apache.camel.util.ObjectHelper; import org.apache.camel.util.URISupport; @@ -59,7 +60,7 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { } protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<FTPFile>> fileList, int depth) { - boolean answer = doPollDirectory(absolutePath, dirName, fileList, depth); + boolean answer = doSafePollSubDirectory(absolutePath, dirName, fileList, depth); // change back to parent directory when finished polling sub directory if (isStepwise()) { operations.changeToParentDirectory(); @@ -167,10 +168,19 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> { @Override protected boolean ignoreCannotRetrieveFile(String name, Exchange exchange, Exception cause) { if (getEndpoint().getConfiguration().isIgnoreFileNotFoundOrPermissionError()) { - // error code 550 is file not found - int code = exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, 0, int.class); - if (code == 550) { - return true; + if (exchange != null) { + // error code 550 is file not found + int code = exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, 0, int.class); + if (code == 550) { + return true; + } + } + if (cause != null && cause instanceof GenericFileOperationFailedException) { + GenericFileOperationFailedException generic = ObjectHelper.getException(GenericFileOperationFailedException.class, cause); + //exchange is null and cause has the reason for failure to read directories + if (generic.getCode() == 550) { + return true; + } } } return super.ignoreCannotRetrieveFile(name, exchange, cause); http://git-wip-us.apache.org/repos/asf/camel/blob/3c53313d/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java ---------------------------------------------------------------------- diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java index de242da..bb418e2 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java @@ -17,9 +17,11 @@ package org.apache.camel.component.file.remote; import java.io.IOException; +import java.util.List; import org.apache.camel.Exchange; import org.apache.camel.Processor; +import org.apache.camel.component.file.GenericFile; import org.apache.camel.component.file.GenericFileConsumer; import org.apache.camel.component.file.GenericFileOperationFailedException; @@ -70,13 +72,13 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T> { if (!loggedInWarning) { log.warn(message); loggedInWarning = true; - } + } return false; } else { // need to log the failed log again loggedInWarning = false; } - + return true; } @@ -98,7 +100,7 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T> { exchange.setProperty(Exchange.UNIT_OF_WORK_PROCESS_SYNC, Boolean.TRUE); return super.processExchange(exchange); } - + @Override protected boolean isRetrieveFile() { return getEndpoint().isDownload(); @@ -165,4 +167,46 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T> { return ((RemoteFileEndpoint<?>) endpoint).remoteServerInformation(); } + /** + * Executes doPollDirectory and on exception checks if it can be ignored by calling ignoreCannotRetrieveFile . + * + * @param absolutePath The path of the directory to poll + * @param dirName The name of the directory to poll + * @param fileList current list of files gathered + * @param depth the current depth of the directory + * @return whether or not to continue polling, <tt>false</tt> means the maxMessagesPerPoll limit has been hit + * @throws GenericFileOperationFailedException if the exception during doPollDirectory can not be ignored + */ + protected boolean doSafePollSubDirectory(String absolutePath, String dirName, List<GenericFile<T>> fileList, int depth) { + try { + log.trace("Polling sub directory: {} from: {}", absolutePath, endpoint); + //Try to poll the directory + return doPollDirectory(absolutePath, dirName, fileList, depth); + } catch (Exception e) { + log.debug("Caught exception " + e.getMessage()); + if (ignoreCannotRetrieveFile(absolutePath, null, e)) { + log.trace("Ignoring file error " + e.getMessage() + " for " + absolutePath); + //indicate no files in this directory to poll, continue with fileList + return true; + } else { + log.trace("Not ignoring file error " + e.getMessage() + " for " + absolutePath); + if (e instanceof GenericFileOperationFailedException) { + throw (GenericFileOperationFailedException) e; + } else { + throw new GenericFileOperationFailedException("Cannot poll sub-directory: " + absolutePath + " from: " + endpoint, e); + } + } + } + } + + /** + * Poll directory given by dirName or absolutePath + * + * @param absolutePath The path of the directory to poll + * @param dirName The name of the directory to poll + * @param fileList current list of files gathered + * @param depth the current depth of the directory + * @return whether or not to continue polling, <tt>false</tt> means the maxMessagesPerPoll limit has been hit + */ + protected abstract boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<T>> fileList, int depth); } http://git-wip-us.apache.org/repos/asf/camel/blob/3c53313d/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java ---------------------------------------------------------------------- diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java index 0d0906b..0f2f536 100644 --- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java +++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java @@ -58,7 +58,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry> { } protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<ChannelSftp.LsEntry>> fileList, int depth) { - boolean answer = doPollDirectory(absolutePath, dirName, fileList, depth); + boolean answer = doSafePollSubDirectory(absolutePath, dirName, fileList, depth); // change back to parent directory when finished polling sub directory if (isStepwise()) { operations.changeToParentDirectory(); http://git-wip-us.apache.org/repos/asf/camel/blob/3c53313d/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java ---------------------------------------------------------------------- diff --git a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java new file mode 100644 index 0000000..3ca70b6 --- /dev/null +++ b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.file.remote; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.camel.Exchange; +import org.apache.camel.Processor; +import org.apache.camel.component.file.GenericFile; +import org.apache.camel.component.file.GenericFileOperationFailedException; +import org.apache.camel.component.file.GenericFileProducer; +import org.junit.Assert; +import org.junit.Test; + +public class RemoteFileIgnoreDoPollErrorTest { + private final RemoteFileEndpoint<Object> remoteFileEndpoint = new RemoteFileEndpoint<Object>() { + @Override + protected RemoteFileConsumer<Object> buildConsumer(Processor processor) { + return null; + } + + @Override + protected GenericFileProducer<Object> buildProducer() { + return null; + } + + @Override + public RemoteFileOperations<Object> createRemoteFileOperations() throws Exception { + return null; + } + + @Override + public String getScheme() { + return null; + } + }; + + @Test + public void testReadDirErrorIsHandled() throws Exception { + RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("true", true); + boolean result = consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(), 0); + Assert.assertTrue(result); + } + + @Test + public void testReadDirErrorIsHandledWithNoMorePoll() throws Exception { + RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("false", true); + boolean result = consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(), 0); + Assert.assertFalse(result); + } + + @Test + public void testReadDirErrorNotHandled() throws Exception { + RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("IllegalStateException", false); + try { + consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(), 0); + Assert.fail("Must throw wrapped IllegalStateException in GenericFileOperationFailedException"); + } catch (GenericFileOperationFailedException e) { + Assert.assertTrue(e.getCause() instanceof IllegalStateException); + } + } + + @Test + public void testReadDirErrorNotHandledForGenericFileOperationException() throws Exception { + RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("GenericFileOperationFailedException", false); + try { + consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(), 0); + Assert.fail("Must throw GenericFileOperationFailedException"); + } catch (GenericFileOperationFailedException e) { + Assert.assertNull(e.getCause()); + } + } + + private RemoteFileConsumer<Object> getRemoteFileConsumer(final String doPollResult, final boolean ignoreCannotRetrieveFile) { + return new RemoteFileConsumer<Object>(remoteFileEndpoint, null, null) { + @Override + protected boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<Object>> genericFiles, int depth) { + switch (doPollResult) { + case "IllegalStateException": + throw new IllegalStateException("Problem"); + case "GenericFileOperationFailedException": + throw new GenericFileOperationFailedException("Perm error"); + case "true": + return true; + default: + return false; + } + } + @Override + protected boolean pollDirectory(String fileName, List<GenericFile<Object>> genericFiles, int depth) { + return false; + } + @Override + protected boolean isMatched(GenericFile<Object> file, String doneFileName, List<Object> files) { + return false; + } + @Override + protected boolean ignoreCannotRetrieveFile(String name, Exchange exchange, Exception cause) { + return ignoreCannotRetrieveFile; + } + }; + } +}
