Alon Bar-Lev has uploaded a new change for review.
Change subject: host-deploy: use cpio instead of tar for bundle transfer
......................................................................
host-deploy: use cpio instead of tar for bundle transfer
for some reason fedora-18/19 got tar out of base system, so we are
forced to find some solution, although manual configuration is required
anyway (repository, networking etc...).
there was attempt to use python tar module, however this module has a
bug when last block is empty, so it is not suitable for usage.
cpio is alternative for fedora, however it is rightfully missing from
other distributions as it is much less used. as we do not support hosts
using different distributions, we are ok now.
in future, we can use python self extract script, work already performed
at: I1e5ddab9ae87979da7d7296c4f3c7ef08e21e903 at the price of
complexity.
Change-Id: I1386e7d688a9ec7e28519fb407478fd17cbab4ca
Signed-off-by: Alon Bar-Lev <alo...@redhat.com>
---
M
backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsDeploy.java
M
backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java
R
backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/CachedCpio.java
A
backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/Cpio.java
D
backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/Tar.java
R
backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/CpioTest.java
6 files changed, 166 insertions(+), 253 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/96/17396/1
diff --git
a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsDeploy.java
b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsDeploy.java
index 36d3684..2cfb0b9 100644
---
a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsDeploy.java
+++
b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/VdsDeploy.java
@@ -35,7 +35,7 @@
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.utils.EngineLocalConfig;
import org.ovirt.engine.core.utils.NetworkUtils;
-import org.ovirt.engine.core.utils.archivers.tar.CachedTar;
+import org.ovirt.engine.core.utils.archivers.CachedCpio;
import org.ovirt.engine.core.utils.crypt.EngineEncryptionUtils;
import org.ovirt.engine.core.utils.hostinstall.OpenSslCAWrapper;
import org.ovirt.engine.core.utils.linq.LinqUtils;
@@ -81,7 +81,7 @@
private static final String BOOTSTRAP_CUSTOM_ENVIRONMENT_PLACE_HOLDER =
"@ENVIRONMENT@";
private static final Log log = LogFactory.getLog(VdsDeploy.class);
- private static volatile CachedTar s_deployPackage;
+ private static volatile CachedCpio s_deployPackage;
private SSHDialog.Control _control;
private Thread _thread;
@@ -886,7 +886,7 @@
);
if (s_deployPackage == null) {
- s_deployPackage = new CachedTar(
+ s_deployPackage = new CachedCpio(
new File(
EngineLocalConfig.getInstance().getCacheDir(),
Config.<String> GetValue(ConfigValues.BootstrapPackageName)
diff --git
a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java
b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java
index e3298f9..3495980 100644
---
a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java
+++
b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java
@@ -1065,11 +1065,9 @@
@TypeConverterAttribute(String.class)
@DefaultValueAttribute(
"umask 0077; " +
- "MYTMP=\"$(mktemp -t ovirt-XXXXXXXXXX)\"; " +
+ "MYTMP=\"$(mktemp --tmpdir -d ovirt-XXXXXXXXXX)\"; " +
"trap \"chmod -R u+rwX \\\"${MYTMP}\\\" > /dev/null 2>&1; rm -fr
\\\"${MYTMP}\\\" > /dev/null 2>&1\" 0; " +
- "rm -fr \"${MYTMP}\" && " +
- "mkdir \"${MYTMP}\" && " +
- "tar --warning=no-timestamp -C \"${MYTMP}\" -x && " +
+ "( cd \"${MYTMP}\" && cpio --extract --quiet --make-directories ) && "
+
"@ENVIRONMENT@ \"${MYTMP}\"/setup DIALOG/dialect=str:machine
DIALOG/customization=bool:True"
)
BootstrapCommand(373),
@@ -1081,7 +1079,7 @@
@DefaultValueAttribute("/usr/share/ovirt-host-deploy/interface-3")
BootstrapPackageDirectory(375),
@TypeConverterAttribute(String.class)
- @DefaultValueAttribute("ovirt-host-deploy.tar")
+ @DefaultValueAttribute("ovirt-host-deploy-3.cpio")
BootstrapPackageName(376),
@Reloadable
@TypeConverterAttribute(String.class)
diff --git
a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/CachedTar.java
b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/CachedCpio.java
similarity index 62%
rename from
backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/CachedTar.java
rename to
backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/CachedCpio.java
index d4c6bdf..cc1bd1d 100644
---
a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/CachedTar.java
+++
b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/CachedCpio.java
@@ -1,4 +1,4 @@
-package org.ovirt.engine.core.utils.archivers.tar;
+package org.ovirt.engine.core.utils.archivers;
import java.io.File;
import java.io.FileOutputStream;
@@ -16,50 +16,39 @@
* Cache tar based on directory structure. If files are changed
* recreate tar file. Test file change once per interval.
*/
-public class CachedTar {
+public class CachedCpio {
- private static final Log log = LogFactory.getLog(CachedTar.class);
+ private static final Log log = LogFactory.getLog(CachedCpio.class);
- private long refreshInterval = 10000;
- private long nextCheckTime = 0;
+ private long _refreshInterval = 10000;
+ private long _nextCheckTime = 0;
- private File archive;
- private File dir;
+ private File _archive;
+ private File _dir;
private void create(long timestamp) throws IOException {
File temp = null;
- OutputStream os = null;
try {
// must create within same directory
// so rename be atomic
temp = File.createTempFile(
- this.archive.getName(),
+ _archive.getName(),
"tmp",
- this.archive.getParentFile()
+ _archive.getParentFile()
);
- os = new FileOutputStream(temp);
- Tar.doTar(os, this.dir);
- }
- catch(IOException e) {
- String message = String.format("Cannot create tarball '%1$s'",
this.archive);
- log.error(message, e);
- throw new IOException(message, e);
- }
- finally {
- try {
- if (os != null) {
- os.close();
- }
+ try (OutputStream out = new FileOutputStream(temp)) {
+ Cpio.doArchive(out, _dir);
}
catch(IOException e) {
- log.error(
- String.format("Cannot close '%1$s'", temp),
+ throw new IOException(
+ String.format(
+ "Cannot create cpio '%1$s'",
+ _archive
+ ),
e
);
}
- }
- try {
if (!temp.setLastModified(timestamp)) {
throw new IOException(
String.format(
@@ -70,12 +59,12 @@
);
}
- if (!temp.renameTo(this.archive)) {
+ if (!temp.renameTo(_archive)) {
throw new IOException(
String.format(
"Cannot rename '%1$s' to '%2$s'",
temp.getCanonicalPath(),
- archive.getCanonicalPath()
+ _archive.getCanonicalPath()
)
);
}
@@ -94,25 +83,25 @@
}
private void ensure() throws IOException {
- if (!this.archive.exists()) {
+ if (!_archive.exists()) {
log.info(
String.format(
- "Tarball '%1$s' is missing, creating",
- this.archive.getAbsolutePath()
+ "cpio '%1$s' is missing, creating",
+ _archive.getAbsolutePath()
)
);
- this.nextCheckTime = System.currentTimeMillis() +
this.refreshInterval;
- create(getTimestampRecursive(this.dir));
+ _nextCheckTime = System.currentTimeMillis() + _refreshInterval;
+ create(getTimestampRecursive(_dir));
}
- else if (this.nextCheckTime <= System.currentTimeMillis()) {
- this.nextCheckTime = System.currentTimeMillis() +
this.refreshInterval;
+ else if (_nextCheckTime <= System.currentTimeMillis()) {
+ _nextCheckTime = System.currentTimeMillis() + _refreshInterval;
- long treeTimestamp = getTimestampRecursive(this.dir);
- if (archive.lastModified() != treeTimestamp) {
+ long treeTimestamp = getTimestampRecursive(_dir);
+ if (_archive.lastModified() != treeTimestamp) {
log.info(
String.format(
- "Tarball '%1$s' is out of date, re-creating",
- this.archive.getAbsolutePath()
+ "cpio '%1$s' is out of date, re-creating",
+ _archive.getAbsolutePath()
)
);
create(treeTimestamp);
@@ -125,11 +114,11 @@
* @param archive name of tar to cache.
* @param dir base directory.
*/
- public CachedTar(File archive, File dir) {
- this.archive = archive;
- this.dir = dir;
+ public CachedCpio(File archive, File dir) {
+ _archive = archive;
+ _dir = dir;
- this.refreshInterval = Config.<Integer>GetValue(
+ _refreshInterval = Config.<Integer>GetValue(
ConfigValues.BootstrapCacheRefreshInterval
);
}
@@ -140,7 +129,7 @@
* @return File name.
*/
public File getFileNoUse() {
- return this.archive;
+ return _archive;
}
/**
@@ -149,7 +138,7 @@
*/
public File getFile() throws IOException {
ensure();
- return this.archive;
+ return _archive;
}
/**
diff --git
a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/Cpio.java
b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/Cpio.java
new file mode 100644
index 0000000..2cf81e5
--- /dev/null
+++
b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/Cpio.java
@@ -0,0 +1,108 @@
+package org.ovirt.engine.core.utils.archivers;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveOutputStream;
+import org.apache.commons.compress.archivers.cpio.CpioConstants;
+
+/**
+ * A simple recursive cpio based on apache archives.
+ */
+public class Cpio {
+
+ private static final Log log = LogFactory.getLog(Cpio.class);
+
+ private static void _recurse(
+ CpioArchiveOutputStream archive,
+ File file,
+ String base
+ ) throws SecurityException, IOException {
+
+ CpioArchiveEntry entry = (CpioArchiveEntry)archive.createArchiveEntry(
+ file,
+ base
+ );
+ if (file.isDirectory()) {
+ entry.setMode(
+ CpioConstants.C_ISDIR |
+ CpioConstants.C_IRUSR |
+ CpioConstants.C_IWUSR |
+ CpioConstants.C_IXUSR |
+ 0
+ );
+ archive.putArchiveEntry(entry);
+ archive.closeArchiveEntry();
+ for (String f : file.list()) {
+ _recurse(
+ archive,
+ new File(file, f),
+ new File(entry.getName(), f).getPath()
+ );
+ }
+ }
+ else if (file.isFile()) {
+ int mode = (
+ CpioConstants.C_ISREG |
+ CpioConstants.C_IRUSR |
+ CpioConstants.C_IWUSR |
+ 0
+ );
+ if (file.canExecute()) {
+ mode |= CpioConstants.C_IXUSR;
+ }
+ entry.setMode(mode);
+ archive.putArchiveEntry(entry);
+ try (
+ InputStream is = new FileInputStream(file)
+ ){
+ byte buffer[] = new byte[8192];
+ int n;
+ while ((n = is.read(buffer)) != -1) {
+ archive.write(buffer, 0, n);
+ }
+ }
+ archive.closeArchiveEntry();
+ }
+ }
+
+ /**
+ * Crete archive.
+ * @param os output stream to write into.
+ * @param base base directory.
+ *
+ * Only regular files and directories are supported.
+ * Files will be owner rw and optional execute bit.
+ */
+ public static void doArchive(
+ OutputStream os,
+ File base
+ ) throws SecurityException, IOException {
+
+ if (!base.exists()) {
+ throw new FileNotFoundException(
+ String.format(
+ "File or directory %1$s not found",
+ base
+ )
+ );
+ }
+
+ try (
+ CpioArchiveOutputStream archive = new CpioArchiveOutputStream(
+ os,
+ CpioConstants.FORMAT_NEW
+ )
+ ) {
+ _recurse(archive, base, "./");
+ }
+ }
+}
diff --git
a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/Tar.java
b/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/Tar.java
deleted file mode 100644
index c427f62..0000000
---
a/backend/manager/modules/utils/src/main/java/org/ovirt/engine/core/utils/archivers/tar/Tar.java
+++ /dev/null
@@ -1,142 +0,0 @@
-package org.ovirt.engine.core.utils.archivers.tar;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-
-/**
- * A simple recursive tar based on javatar.
- */
-public class Tar {
-
- private static final Log log = LogFactory.getLog(Tar.class);
-
- private static void _recurse(
- TarArchiveOutputStream archive,
- File file,
- String base
- ) throws SecurityException, IOException {
-
- TarArchiveEntry entry = (TarArchiveEntry)archive.createArchiveEntry(
- file,
- base
- );
- if (entry.getFile().isDirectory()) {
- entry.setMode(0700);
- archive.putArchiveEntry(entry);
- archive.closeArchiveEntry();
- for (String f : file.list()) {
- _recurse(
- archive,
- new File(entry.getFile(), f),
- new File(entry.getName(), f).getPath()
- );
- }
- }
- else if (entry.getFile().isFile()) {
- if (entry.getFile().canExecute()) {
- entry.setMode(0700);
- }
- else {
- entry.setMode(0600);
- }
- archive.putArchiveEntry(entry);
- InputStream is = null;
- try {
- is = new FileInputStream(entry.getFile());
- byte buffer[] = new byte[8192];
- int n;
- while ((n = is.read(buffer)) != -1) {
- archive.write(buffer, 0, n);
- }
- }
- finally {
- if (is != null) {
- try {
- is.close();
- }
- catch(IOException e) {
- log.error(
- String.format(
- "Cannot close file '%1$s'",
- entry.getFile().getAbsolutePath()
- ),
- e
- );
- }
- }
- }
- archive.closeArchiveEntry();
- }
- }
-
- /**
- * Crete tar.
- * @param os output stream to write into.
- * @param base base directory.
- *
- * Only regular files and directories are supported.
- * Files will be owner rw and optional execute bit.
- */
- public static void doTar(
- OutputStream os,
- File base
- ) throws SecurityException, IOException {
-
- if (!base.exists()) {
- throw new FileNotFoundException(
- String.format(
- "File or directory %1$s not found",
- base
- )
- );
- }
-
- TarArchiveOutputStream archive = null;
- try {
- archive = new TarArchiveOutputStream(os);
- // TODO: use LONGFILE_POSIX in newer version of commons-compress
- archive.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
- _recurse(archive, base, "./");
- }
- finally {
- if (archive != null) {
- try {
- archive.close();
- }
- catch(IOException e) {
- log.error(
- String.format("Cannot close tar stream"),
- e
- );
- }
- }
- }
- }
-
- public static void main(String args[]) throws Exception {
- FileOutputStream os = null;
- try {
- os = new FileOutputStream(args[0]);
- Tar.doTar(os, new File(args[1]));
- }
- finally {
- if (os != null) {
- try {
- os.close();
- }
- catch(IOException e) {}
- }
- }
- }
-}
diff --git
a/backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/tar/TarTest.java
b/backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/CpioTest.java
similarity index 62%
rename from
backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/tar/TarTest.java
rename to
backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/CpioTest.java
index da965c1..627336d 100644
---
a/backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/tar/TarTest.java
+++
b/backend/manager/modules/utils/src/test/java/org/ovirt/engine/core/utils/archivers/CpioTest.java
@@ -1,4 +1,4 @@
-package org.ovirt.engine.core.utils.archivers.tar;
+package org.ovirt.engine.core.utils.archivers;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -19,25 +19,13 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
-public class TarTest {
+public class CpioTest {
private void _writeFile(File file, String content, boolean executable)
throws IOException {
- OutputStream os = null;
- try {
- file.getParentFile().mkdirs();
- os = new FileOutputStream(file);
+ file.getParentFile().mkdirs();
+ try (OutputStream os = new FileOutputStream(file)) {
os.write(content.getBytes("UTF-8"));
file.setExecutable(executable);
- }
- finally {
- if (os != null) {
- try {
- os.close();
- }
- catch (IOException e) {
- // ignore
- }
- }
}
}
@@ -53,34 +41,16 @@
}
else {
MessageDigest fmd = MessageDigest.getInstance(md.getAlgorithm());
- InputStream fis = null;
- InputStream is = null;
- try {
- fis = new FileInputStream(fullFile);
- is = new DigestInputStream(fis, fmd);
+ try (
+ InputStream fis = new FileInputStream(fullFile);
+ InputStream is = new DigestInputStream(fis, fmd)
+ ) {
byte buf[] = new byte[1024];
int n;
while ((n = is.read(buf)) != -1) {
// do nothing
}
md.update(fmd.digest());
- }
- finally {
- if (fis != null) {
- try {
- fis.close();
- }
- catch (IOException e) {
- // ignore
- }
- }
- if (is != null) {
- try {
- is.close();
- } catch (IOException e) {
- // ignore
- }
- }
}
}
}
@@ -89,12 +59,12 @@
public void testSimple() throws Exception {
assumeTrue(SystemUtils.IS_OS_UNIX);
- File tmpTar = null;
+ File tmpCpio = null;
File tmpDir1 = null;
File tmpDir2 = null;
try {
- tmpTar = File.createTempFile("test1", "tar");
+ tmpCpio = File.createTempFile("test1", "tar");
tmpDir1 = File.createTempFile("test1", "tmp");
tmpDir1.delete();
tmpDir1.mkdir();
@@ -109,23 +79,13 @@
_writeFile(new File(tmpDir1, "dir1/file3"), "file2", false);
_writeFile(new File(tmpDir1, "dir1/dir2/file4"), "file4", false);
- OutputStream os = null;
- try {
- os = new FileOutputStream(tmpTar);
- Tar.doTar(os, tmpDir1);
- }
- finally {
- if (os != null) {
- try {
- os.close();
- }
- catch(IOException e) {
- // ignore
- }
- }
+ try (OutputStream os = new FileOutputStream(tmpCpio)) {
+ Cpio.doArchive(os, tmpDir1);
}
- new ProcessBuilder("tar", "-C", tmpDir2.getPath(), "-xf",
tmpTar.getPath()).start().waitFor();
+ ProcessBuilder cpio = new ProcessBuilder("cpio", "--extract",
"--make-directories", "--file", tmpCpio.getPath());
+ cpio.directory(tmpDir2);
+ cpio.start().waitFor();
MessageDigest md1 = MessageDigest.getInstance("SHA");
MessageDigest md2 = MessageDigest.getInstance("SHA");
@@ -137,7 +97,7 @@
assertFalse(new File(tmpDir2, "file1").canExecute());
}
finally {
- for (File file : new File[] {tmpDir1, tmpDir2, tmpTar}) {
+ for (File file : new File[] {tmpDir1, tmpDir2, tmpCpio}) {
if (file != null) {
file.delete();
}
@@ -147,6 +107,6 @@
@Test(expected=FileNotFoundException.class)
public void testNoBase() throws IOException {
- Tar.doTar(new ByteArrayOutputStream(), new File("/asdasdsadasdasdsa"));
+ Cpio.doArchive(new ByteArrayOutputStream(), new
File("/asdasdsadasdasdsa"));
}
}
--
To view, visit http://gerrit.ovirt.org/17396
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1386e7d688a9ec7e28519fb407478fd17cbab4ca
Gerrit-PatchSet: 1
Gerrit-Project: ovirt-engine
Gerrit-Branch: master
Gerrit-Owner: Alon Bar-Lev <alo...@redhat.com>
_______________________________________________
Engine-patches mailing list
Engine-patches@ovirt.org
http://lists.ovirt.org/mailman/listinfo/engine-patches
