Repository: camel Updated Branches: refs/heads/master f1b05572e -> 6a807d605
Add unit test to support folders in zip files. Add implementation to support folders in zip files. Add backwards compatibility and separate entries for folders. Add options to camel-core for ZipFileDataFormat and TarFileDataFormat. Add unit test to support folders in tar files. Add implementation to support folders in tar files with backwards compatibility. Fix checkstyle errors. Project: http://git-wip-us.apache.org/repos/asf/camel/repo Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/38ff6658 Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/38ff6658 Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/38ff6658 Branch: refs/heads/master Commit: 38ff6658f0a3c0b3fa69a8a1923f708ce4ef97d1 Parents: f1b0557 Author: Vreijsen <joery_vreij...@hotmail.com> Authored: Tue Oct 10 14:46:20 2017 +0200 Committer: Claus Ibsen <davscl...@apache.org> Committed: Mon Oct 16 11:04:32 2017 +0200 ---------------------------------------------------------------------- .../model/dataformat/TarFileDataFormat.java | 17 ++++++ .../model/dataformat/ZipFileDataFormat.java | 17 ++++++ .../dataformat/tarfile/TarFileDataFormat.java | 54 +++++++++++++++++--- .../tarfile/TarFileDataFormatTest.java | 31 +++++++++++ .../camel/dataformat/tarfile/TarUtils.java | 21 ++++++++ .../dataformat/zipfile/ZipFileDataFormat.java | 51 +++++++++++++++--- .../zipfile/ZipFileDataFormatTest.java | 36 +++++++++++++ 7 files changed, 214 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/camel-core/src/main/java/org/apache/camel/model/dataformat/TarFileDataFormat.java ---------------------------------------------------------------------- diff --git a/camel-core/src/main/java/org/apache/camel/model/dataformat/TarFileDataFormat.java b/camel-core/src/main/java/org/apache/camel/model/dataformat/TarFileDataFormat.java index 26153f8..b9b5a0e 100644 --- a/camel-core/src/main/java/org/apache/camel/model/dataformat/TarFileDataFormat.java +++ b/camel-core/src/main/java/org/apache/camel/model/dataformat/TarFileDataFormat.java @@ -37,6 +37,8 @@ public class TarFileDataFormat extends DataFormatDefinition { private Boolean usingIterator; @XmlAttribute private Boolean allowEmptyDirectory; + @XmlAttribute + private Boolean preservePathElements; public TarFileDataFormat() { super("tarfile"); @@ -47,6 +49,9 @@ public class TarFileDataFormat extends DataFormatDefinition { if (usingIterator != null) { setProperty(camelContext, dataFormat, "usingIterator", usingIterator); } + if (preservePathElements != null) { + setProperty(camelContext, dataFormat, "preservePathElements", preservePathElements); + } } public Boolean getUsingIterator() { @@ -57,6 +62,10 @@ public class TarFileDataFormat extends DataFormatDefinition { return allowEmptyDirectory; } + public Boolean getPreservePathElements() { + return preservePathElements; + } + /** * If the tar file has more then one entry, the setting this option to true, allows to work with the splitter EIP, * to split the data using an iterator in a streaming mode. @@ -72,4 +81,12 @@ public class TarFileDataFormat extends DataFormatDefinition { public void setAllowEmptyDirectory(Boolean allowEmptyDirectory) { this.allowEmptyDirectory = allowEmptyDirectory; } + + /** + * If the file name contains path elements, setting this option to true, allows the path to be maintained + * in the tar file. + */ + public void setPreservePathElements(Boolean preservePathElements) { + this.preservePathElements = preservePathElements; + } } http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/camel-core/src/main/java/org/apache/camel/model/dataformat/ZipFileDataFormat.java ---------------------------------------------------------------------- diff --git a/camel-core/src/main/java/org/apache/camel/model/dataformat/ZipFileDataFormat.java b/camel-core/src/main/java/org/apache/camel/model/dataformat/ZipFileDataFormat.java index ff2dd60..4d98796 100644 --- a/camel-core/src/main/java/org/apache/camel/model/dataformat/ZipFileDataFormat.java +++ b/camel-core/src/main/java/org/apache/camel/model/dataformat/ZipFileDataFormat.java @@ -38,6 +38,8 @@ public class ZipFileDataFormat extends DataFormatDefinition { private Boolean usingIterator; @XmlAttribute private Boolean allowEmptyDirectory; + @XmlAttribute + private Boolean preservePathElements; public ZipFileDataFormat() { super("zipfile"); @@ -51,6 +53,9 @@ public class ZipFileDataFormat extends DataFormatDefinition { if (allowEmptyDirectory != null) { setProperty(camelContext, dataFormat, "allowEmptyDirectory", allowEmptyDirectory); } + if (preservePathElements != null) { + setProperty(camelContext, dataFormat, "preservePathElements", preservePathElements); + } } public Boolean getUsingIterator() { @@ -61,6 +66,10 @@ public class ZipFileDataFormat extends DataFormatDefinition { return allowEmptyDirectory; } + public Boolean getPreservePathElements() { + return preservePathElements; + } + /** * If the zip file has more then one entry, the setting this option to true, allows to work with the splitter EIP, * to split the data using an iterator in a streaming mode. @@ -77,4 +86,12 @@ public class ZipFileDataFormat extends DataFormatDefinition { this.allowEmptyDirectory = allowEmptyDirectory; } + /** + * If the file name contains path elements, setting this option to true, allows the path to be maintained + * in the zip file. + */ + public void setPreservePathElements(Boolean preservePathElements) { + this.preservePathElements = preservePathElements; + } + } http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/components/camel-tarfile/src/main/java/org/apache/camel/dataformat/tarfile/TarFileDataFormat.java ---------------------------------------------------------------------- diff --git a/components/camel-tarfile/src/main/java/org/apache/camel/dataformat/tarfile/TarFileDataFormat.java b/components/camel-tarfile/src/main/java/org/apache/camel/dataformat/tarfile/TarFileDataFormat.java index b226c6e..35b1a9c 100644 --- a/components/camel-tarfile/src/main/java/org/apache/camel/dataformat/tarfile/TarFileDataFormat.java +++ b/components/camel-tarfile/src/main/java/org/apache/camel/dataformat/tarfile/TarFileDataFormat.java @@ -17,9 +17,12 @@ package org.apache.camel.dataformat.tarfile; import java.io.BufferedInputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Iterator; import org.apache.camel.Exchange; import org.apache.camel.converter.stream.OutputStreamBuilder; @@ -43,6 +46,7 @@ import static org.apache.camel.Exchange.FILE_NAME; public class TarFileDataFormat extends ServiceSupport implements DataFormat, DataFormatName { private boolean usingIterator; private boolean allowEmptyDirectory; + private boolean preservePathElements; @Override public String getDataFormatName() { @@ -51,13 +55,14 @@ public class TarFileDataFormat extends ServiceSupport implements DataFormat, Dat @Override public void marshal(final Exchange exchange, final Object graph, final OutputStream stream) throws Exception { - String filename = exchange.getIn().getHeader(FILE_NAME, String.class); + String filename; + String filepath = exchange.getIn().getHeader(FILE_NAME, String.class); Long filelength = exchange.getIn().getHeader(FILE_LENGTH, Long.class); - if (filename == null) { + if (filepath == null) { // generate the file name as the camel file component would do - filename = StringHelper.sanitize(exchange.getIn().getMessageId()); + filename = filepath = StringHelper.sanitize(exchange.getIn().getMessageId()); } else { - filename = Paths.get(filename).getFileName().toString(); // remove any path elements + filename = Paths.get(filepath).getFileName().toString(); // remove any path elements } TarArchiveOutputStream tos = new TarArchiveOutputStream(stream); @@ -69,9 +74,11 @@ public class TarFileDataFormat extends ServiceSupport implements DataFormat, Dat filelength = (long) is.available(); } - TarArchiveEntry entry = new TarArchiveEntry(filename); - entry.setSize(filelength); - tos.putArchiveEntry(entry); + if (preservePathElements) { + createTarEntries(tos, filepath, filelength); + } else { + createTarEntries(tos, filename, filelength); + } try { IOHelper.copy(is, tos); @@ -115,6 +122,31 @@ public class TarFileDataFormat extends ServiceSupport implements DataFormat, Dat } } + private void createTarEntries(TarArchiveOutputStream tos, String filepath, Long filelength) throws IOException { + Iterator<Path> elements = Paths.get(filepath).iterator(); + StringBuilder sb = new StringBuilder(); + + while (elements.hasNext()) { + Path path = elements.next(); + String element = path.toString(); + Long length = filelength; + + // If there are more elements to come this element is a directory + // The "/" at the end tells the TarEntry it is a folder + if (elements.hasNext()) { + element += "/"; + length = 0L; + } + + // Each entry needs the complete path, including previous created folders. + TarArchiveEntry entry = new TarArchiveEntry(sb + element); + entry.setSize(length); + tos.putArchiveEntry(entry); + + sb.append(element); + } + } + public boolean isUsingIterator() { return usingIterator; } @@ -131,6 +163,14 @@ public class TarFileDataFormat extends ServiceSupport implements DataFormat, Dat this.allowEmptyDirectory = allowEmptyDirectory; } + public boolean isPreservePathElements() { + return preservePathElements; + } + + public void setPreservePathElements(boolean preservePathElements) { + this.preservePathElements = preservePathElements; + } + @Override protected void doStart() throws Exception { // noop http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarFileDataFormatTest.java ---------------------------------------------------------------------- diff --git a/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarFileDataFormatTest.java b/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarFileDataFormatTest.java index 7f8b792..b74cfb6 100644 --- a/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarFileDataFormatTest.java +++ b/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarFileDataFormatTest.java @@ -43,6 +43,7 @@ import static org.apache.camel.Exchange.FILE_NAME; import static org.apache.camel.dataformat.tarfile.TarUtils.TEXT; import static org.apache.camel.dataformat.tarfile.TarUtils.getBytes; import static org.apache.camel.dataformat.tarfile.TarUtils.getTaredText; +import static org.apache.camel.dataformat.tarfile.TarUtils.getTaredTextInFolder; /** * Unit tests for {@link TarFileDataFormat}. @@ -81,6 +82,36 @@ public class TarFileDataFormatTest extends CamelTestSupport { } @Test + public void testTarWithPathElements() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:tar"); + mock.expectedMessageCount(1); + mock.expectedHeaderReceived(FILE_NAME, "poem.txt.tar"); + + template.sendBodyAndHeader("direct:tar", TEXT, FILE_NAME, "poems/poem.txt"); + + assertMockEndpointsSatisfied(); + + Exchange exchange = mock.getReceivedExchanges().get(0); + assertTrue(ObjectHelper.equalByteArray(getTaredText("poem.txt"), (byte[]) exchange.getIn().getBody())); + } + + @Test + public void testTarWithPreservedPathElements() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:tar"); + mock.expectedMessageCount(1); + mock.expectedHeaderReceived(FILE_NAME, "poem.txt.tar"); + + tar.setPreservePathElements(true); + + template.sendBodyAndHeader("direct:tar", TEXT, FILE_NAME, "poems/poem.txt"); + + assertMockEndpointsSatisfied(); + + Exchange exchange = mock.getReceivedExchanges().get(0); + assertTrue(ObjectHelper.equalByteArray(getTaredTextInFolder("poems/", "poems/poem.txt"), (byte[]) exchange.getIn().getBody())); + } + + @Test public void testUntar() throws Exception { getMockEndpoint("mock:untar").expectedBodiesReceived(TEXT); getMockEndpoint("mock:untar").expectedHeaderReceived(FILE_NAME, "file"); http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarUtils.java ---------------------------------------------------------------------- diff --git a/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarUtils.java b/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarUtils.java index df07907..4acf633 100644 --- a/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarUtils.java +++ b/components/camel-tarfile/src/test/java/org/apache/camel/dataformat/tarfile/TarUtils.java @@ -59,6 +59,27 @@ final class TarUtils { return baos.toByteArray(); } + static byte[] getTaredTextInFolder(String folder, String file) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(TEXT.getBytes("UTF-8")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + TarArchiveOutputStream tos = new TarArchiveOutputStream(baos); + try { + TarArchiveEntry folderEntry = new TarArchiveEntry(folder); + folderEntry.setSize(0L); + tos.putArchiveEntry(folderEntry); + + TarArchiveEntry fileEntry = new TarArchiveEntry(file); + fileEntry.setSize(bais.available()); + tos.putArchiveEntry(fileEntry); + + IOHelper.copy(bais, tos); + } finally { + tos.closeArchiveEntry(); + IOHelper.close(bais, tos); + } + return baos.toByteArray(); + } + static byte[] getBytes(File file) throws IOException { FileInputStream fis = new FileInputStream(file); ByteArrayOutputStream baos = new ByteArrayOutputStream(); http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/components/camel-zipfile/src/main/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormat.java ---------------------------------------------------------------------- diff --git a/components/camel-zipfile/src/main/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormat.java b/components/camel-zipfile/src/main/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormat.java index 171858e..28ce82c 100644 --- a/components/camel-zipfile/src/main/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormat.java +++ b/components/camel-zipfile/src/main/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormat.java @@ -16,9 +16,12 @@ */ package org.apache.camel.dataformat.zipfile; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Iterator; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -40,6 +43,7 @@ import static org.apache.camel.Exchange.FILE_NAME; public class ZipFileDataFormat extends ServiceSupport implements DataFormat, DataFormatName { private boolean usingIterator; private boolean allowEmptyDirectory; + private boolean preservePathElements; @Override public String getDataFormatName() { @@ -48,16 +52,22 @@ public class ZipFileDataFormat extends ServiceSupport implements DataFormat, Dat @Override public void marshal(final Exchange exchange, final Object graph, final OutputStream stream) throws Exception { - String filename = exchange.getIn().getHeader(FILE_NAME, String.class); - if (filename == null) { + String filename; + String filepath = exchange.getIn().getHeader(FILE_NAME, String.class); + if (filepath == null) { // generate the file name as the camel file component would do - filename = StringHelper.sanitize(exchange.getIn().getMessageId()); + filename = filepath = StringHelper.sanitize(exchange.getIn().getMessageId()); } else { - filename = Paths.get(filename).getFileName().toString(); // remove any path elements + filename = Paths.get(filepath).getFileName().toString(); // remove any path elements } ZipOutputStream zos = new ZipOutputStream(stream); - zos.putNextEntry(new ZipEntry(filename)); + + if (preservePathElements) { + createZipEntries(zos, filepath); + } else { + createZipEntries(zos, filename); + } InputStream is = exchange.getContext().getTypeConverter().mandatoryConvertTo(InputStream.class, exchange, graph); @@ -100,6 +110,27 @@ public class ZipFileDataFormat extends ServiceSupport implements DataFormat, Dat } } + private void createZipEntries(ZipOutputStream zos, String filepath) throws IOException { + Iterator<Path> elements = Paths.get(filepath).iterator(); + StringBuilder sb = new StringBuilder(); + + while (elements.hasNext()) { + Path path = elements.next(); + String element = path.toString(); + + // If there are more elements to come this element is a directory + // The "/" at the end tells the ZipEntry it is a folder + if (elements.hasNext()) { + element += "/"; + } + + // Each entry needs the complete path, including previous created folders. + zos.putNextEntry(new ZipEntry(sb + element)); + + sb.append(element); + } + } + public boolean isUsingIterator() { return usingIterator; } @@ -116,6 +147,14 @@ public class ZipFileDataFormat extends ServiceSupport implements DataFormat, Dat this.allowEmptyDirectory = allowEmptyDirectory; } + public boolean isPreservePathElements() { + return preservePathElements; + } + + public void setPreservePathElements(boolean preservePathElements) { + this.preservePathElements = preservePathElements; + } + @Override protected void doStart() throws Exception { // noop @@ -125,4 +164,4 @@ public class ZipFileDataFormat extends ServiceSupport implements DataFormat, Dat protected void doStop() throws Exception { // noop } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/camel/blob/38ff6658/components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormatTest.java ---------------------------------------------------------------------- diff --git a/components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormatTest.java b/components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormatTest.java index 305283e..63b1b03 100644 --- a/components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormatTest.java +++ b/components/camel-zipfile/src/test/java/org/apache/camel/dataformat/zipfile/ZipFileDataFormatTest.java @@ -104,6 +104,28 @@ public class ZipFileDataFormatTest extends CamelTestSupport { } @Test + public void testZipWithPathElements() throws Exception { + getMockEndpoint("mock:zip").expectedBodiesReceived(getZippedText("poem.txt")); + getMockEndpoint("mock:zip").expectedHeaderReceived(FILE_NAME, "poem.txt.zip"); + + template.sendBodyAndHeader("direct:zip", TEXT, FILE_NAME, "poems/poem.txt"); + + assertMockEndpointsSatisfied(); + } + + @Test + public void testZipWithPreservedPathElements() throws Exception { + zip.setPreservePathElements(true); + + getMockEndpoint("mock:zip").expectedBodiesReceived(getZippedTextInFolder("poems/", "poems/poem.txt")); + getMockEndpoint("mock:zip").expectedHeaderReceived(FILE_NAME, "poem.txt.zip"); + + template.sendBodyAndHeader("direct:zip", TEXT, FILE_NAME, "poems/poem.txt"); + + assertMockEndpointsSatisfied(); + } + + @Test public void testUnzip() throws Exception { getMockEndpoint("mock:unzip").expectedBodiesReceived(TEXT); getMockEndpoint("mock:unzip").expectedHeaderReceived(FILE_NAME, "file"); @@ -298,6 +320,20 @@ public class ZipFileDataFormatTest extends CamelTestSupport { return baos.toByteArray(); } + private static byte[] getZippedTextInFolder(String folder, String file) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(TEXT.getBytes("UTF-8")); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(baos); + try { + zos.putNextEntry(new ZipEntry(folder)); + zos.putNextEntry(new ZipEntry(file)); + IOHelper.copy(bais, zos); + } finally { + IOHelper.close(bais, zos); + } + return baos.toByteArray(); + } + private static byte[] getBytes(File file) throws IOException { FileInputStream fis = new FileInputStream(file); ByteArrayOutputStream baos = new ByteArrayOutputStream();
