This change, and it's associated backports, seem to have broken TestConfigSetService (regardless of seed?) due to leaked files.
https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/8576/consoleText https://jenkins.thetaphi.de/view/Solr/job/Solr-main-Windows/3706/consoleText https://ci-builds.apache.org/job/Solr/job/Solr-Check-9.x/39/consoleText https://ci-builds.apache.org/job/Solr/job/Solr-check-9.4/1208/consoleText : Date: Wed, 13 Dec 2023 22:08:04 +0000 : From: jan...@apache.org : Reply-To: dev@solr.apache.org : To: "comm...@solr.apache.org" <comm...@solr.apache.org> : Subject: (solr) branch main updated: SOLR-16949: Restrict certain file types : from being uploaded to or downloaded from Config Sets : : This is an automated email from the ASF dual-hosted git repository. : : janhoy pushed a commit to branch main : in repository https://gitbox.apache.org/repos/asf/solr.git : : : The following commit(s) were added to refs/heads/main by this push: : new 15534754f49 SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets : 15534754f49 is described below : : commit 15534754f492079e52288dd11abaf1c4261b3ea4 : Author: Jan Høydahl <jan...@apache.org> : AuthorDate: Wed Dec 13 22:49:23 2023 +0100 : : SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets : --- : solr/CHANGES.txt | 2 + : solr/core/build.gradle | 2 + : .../org/apache/solr/cli/ConfigSetUploadTool.java | 2 + : .../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++- : .../solr/core/FileSystemConfigSetService.java | 28 +++- : .../org/apache/solr/core/backup/BackupManager.java | 23 ++- : .../handler/configsets/UploadConfigSetFileAPI.java | 8 +- : .../org/apache/solr/util/FileTypeMagicUtil.java | 166 +++++++++++++++++++++ : solr/core/src/resources/magic/executables | 74 +++++++++ : solr/core/src/test-files/magic/HelloWorld.java.txt | 5 + : .../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes : solr/core/src/test-files/magic/README.md | 29 ++++ : solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes : solr/core/src/test-files/magic/plain.txt | 1 + : solr/core/src/test-files/magic/shell.sh.txt | 2 + : .../org/apache/solr/cloud/TestConfigSetsAPI.java | 141 +++++++++++------ : .../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++ : solr/licenses/simplemagic-1.17.jar.sha1 | 1 + : solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++ : solr/licenses/simplemagic-NOTICE.txt | 0 : .../solr/common/cloud/ZkMaintenanceUtils.java | 2 + : versions.lock | 1 + : versions.props | 1 + : 23 files changed, 522 insertions(+), 56 deletions(-) : : diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt : index 40da8c435ac..2fd94304a24 100644 : --- a/solr/CHANGES.txt : +++ b/solr/CHANGES.txt : @@ -169,6 +169,8 @@ Other Changes : * SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting : cloud.sh has been updated to ignore the -slim.tgz version of the tarball. : : +* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman) : + : ================== 9.4.0 ================== : New Features : --------------------- : diff --git a/solr/core/build.gradle b/solr/core/build.gradle : index 61ecd1713af..ed2c8a370ae 100644 : --- a/solr/core/build.gradle : +++ b/solr/core/build.gradle : @@ -159,6 +159,8 @@ dependencies { : : compileOnly 'com.github.stephenc.jcip:jcip-annotations' : : + implementation 'com.j256.simplemagic:simplemagic' : + : // -- Test Dependencies : : testRuntimeOnly 'org.slf4j:jcl-over-slf4j' : diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java : index 5fd4a538bd7..6576742a195 100644 : --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java : +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java : @@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; : import org.apache.solr.common.cloud.SolrZkClient; : import org.apache.solr.common.cloud.ZkMaintenanceUtils; : import org.apache.solr.core.ConfigSetService; : +import org.apache.solr.util.FileTypeMagicUtil; : import org.slf4j.Logger; : import org.slf4j.LoggerFactory; : : @@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase { : + cli.getOptionValue("confname") : + " to ZooKeeper at " : + zkHost); : + FileTypeMagicUtil.assertConfigSetFolderLegal(confPath); : ZkMaintenanceUtils.uploadToZK( : zkClient, : confPath, : diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java : index f02404d636d..9abde098e1c 100644 : --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java : +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java : @@ -22,6 +22,7 @@ import java.nio.file.Path; : import java.util.ArrayList; : import java.util.Collections; : import java.util.List; : +import java.util.Locale; : import java.util.Map; : import java.util.Objects; : import org.apache.solr.client.solrj.cloud.SolrCloudManager; : @@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer; : import org.apache.solr.core.CoreDescriptor; : import org.apache.solr.core.SolrConfig; : import org.apache.solr.core.SolrResourceLoader; : +import org.apache.solr.util.FileTypeMagicUtil; : import org.apache.zookeeper.CreateMode; : import org.apache.zookeeper.KeeperException; : import org.apache.zookeeper.data.Stat; : @@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService { : try { : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) { : log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName); : + } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); : + throw new SolrException( : + SolrException.ErrorCode.BAD_REQUEST, : + String.format( : + Locale.ROOT, : + "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s", : + fileName, : + mimeType)); : } else { : // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false : zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true); : @@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService { : } else { : log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath); : byte[] data = zkClient.getData(fromZkFilePath, null, null, true); : - zkClient.makePath(toZkFilePath, data, true); : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { : + zkClient.makePath(toZkFilePath, data, true); : + } else { : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); : + log.warn( : + "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}", : + fromZkFilePath, : + mimeType); : + } : } : } : : diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java : index 1f1a42b15e1..4b041252211 100644 : --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java : +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java : @@ -37,6 +37,7 @@ import java.util.stream.Stream; : import org.apache.solr.common.SolrException; : import org.apache.solr.common.cloud.ZkMaintenanceUtils; : import org.apache.solr.common.util.Utils; : +import org.apache.solr.util.FileTypeMagicUtil; : import org.slf4j.Logger; : import org.slf4j.LoggerFactory; : : @@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends ConfigSetService { : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) { : log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName); : } else { : - Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName)); : - if (!Files.exists(filePath) || overwriteOnExists) { : - Files.write(filePath, data); : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { : + Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName)); : + if (!Files.exists(filePath) || overwriteOnExists) { : + Files.write(filePath, data); : + } : + } else { : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); : + log.warn( : + "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}", : + fileName, : + mimeType); : } : } : } : @@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends ConfigSetService { : "Not including uploading file to config, as it is a forbidden type: {}", : file.getFileName()); : } else { : - Files.copy( : - file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING); : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) { : + Files.copy( : + file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING); : + } else { : + String mimeType = : + FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file)); : + log.warn( : + "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}", : + file.getFileName(), : + mimeType); : + } : } : return FileVisitResult.CONTINUE; : } : diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java : index be6a1a83c2f..8ff78b27e08 100644 : --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java : +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java : @@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader; : import org.apache.solr.common.util.Utils; : import org.apache.solr.core.ConfigSetService; : import org.apache.solr.core.backup.repository.BackupRepository; : +import org.apache.solr.util.FileTypeMagicUtil; : import org.apache.zookeeper.CreateMode; : import org.apache.zookeeper.KeeperException; : import org.slf4j.Logger; : @@ -349,8 +350,16 @@ public class BackupManager { : if (data == null) { : data = new byte[0]; : } : - try (OutputStream os = repository.createOutput(uri)) { : - os.write(data); : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { : + try (OutputStream os = repository.createOutput(uri)) { : + os.write(data); : + } : + } else { : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); : + log.warn( : + "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}", : + filePath, : + mimeType); : } : } : } else { : @@ -379,7 +388,15 @@ public class BackupManager { : // probably ok since the config file should be small. : byte[] arr = new byte[(int) is.length()]; : is.readBytes(arr, 0, (int) is.length()); : - configSetService.uploadFileToConfig(configName, filePath, arr, false); : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) { : + configSetService.uploadFileToConfig(configName, filePath, arr, false); : + } else { : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr); : + log.warn( : + "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}", : + filePath, : + mimeType); : + } : } : } : break; : diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java : index 98889aff563..2380a79a92b 100644 : --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java : +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java : @@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams; : import org.apache.solr.core.CoreContainer; : import org.apache.solr.request.SolrQueryRequest; : import org.apache.solr.response.SolrQueryResponse; : +import org.apache.solr.util.FileTypeMagicUtil; : : /** : * V2 API for adding or updating a single file within a configset. : @@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase { : if (fixedSingleFilePath.charAt(0) == '/') { : fixedSingleFilePath = fixedSingleFilePath.substring(1); : } : + byte[] data = inputStream.readAllBytes(); : if (fixedSingleFilePath.isEmpty()) { : throw new SolrException( : SolrException.ErrorCode.BAD_REQUEST, : "The file path provided for upload, '" + singleFilePath + "', is not valid."); : - } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) { : + } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath) : + || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { : throw new SolrException( : SolrException.ErrorCode.BAD_REQUEST, : "The file type provided for upload, '" : @@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase { : // For creating the baseNode, the cleanup parameter is only allowed to be true when : // singleFilePath is not passed. : createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName); : - configSetService.uploadFileToConfig( : - configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite); : + configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite); : } : } : } : diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java : new file mode 100644 : index 00000000000..cfb6c9fa0af : --- /dev/null : +++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java : @@ -0,0 +1,166 @@ : +/* : + * Licensed to the Apache Software Foundation (ASF) under one or more : + * contributor license agreements. See the NOTICE file distributed with : + * this work for additional information regarding copyright ownership. : + * The ASF licenses this file to You under the Apache License, Version 2.0 : + * (the "License"); you may not use this file except in compliance with : + * the License. You may obtain a copy of the License at : + * : + * http://www.apache.org/licenses/LICENSE-2.0 : + * : + * Unless required by applicable law or agreed to in writing, software : + * distributed under the License is distributed on an "AS IS" BASIS, : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. : + * See the License for the specific language governing permissions and : + * limitations under the License. : + */ : + : +package org.apache.solr.util; : + : +import com.j256.simplemagic.ContentInfo; : +import com.j256.simplemagic.ContentInfoUtil; : +import com.j256.simplemagic.ContentType; : +import java.io.ByteArrayInputStream; : +import java.io.IOException; : +import java.io.InputStream; : +import java.nio.file.FileVisitResult; : +import java.nio.file.Files; : +import java.nio.file.Path; : +import java.nio.file.SimpleFileVisitor; : +import java.nio.file.attribute.BasicFileAttributes; : +import java.util.Arrays; : +import java.util.HashSet; : +import java.util.Locale; : +import java.util.Set; : +import org.apache.solr.common.SolrException; : + : +/** Utility class to guess the mime type of file based on its magic number. */ : +public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack { : + private final ContentInfoUtil util; : + private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", "..")); : + : + public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil(); : + : + FileTypeMagicUtil() { : + try { : + util = new ContentInfoUtil("/magic/executables", this); : + } catch (IOException e) { : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e); : + } : + } : + : + /** : + * Asserts that an entire configset folder is legal to upload. : + * : + * @param confPath the path to the folder : + * @throws SolrException if an illegal file is found in the folder structure : + */ : + public static void assertConfigSetFolderLegal(Path confPath) throws IOException { : + Files.walkFileTree( : + confPath, : + new SimpleFileVisitor<Path>() { : + @Override : + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) : + throws IOException { : + // Read first 100 bytes of the file to determine the mime type : + try (InputStream fileStream = Files.newInputStream(file)) { : + byte[] bytes = new byte[100]; : + fileStream.read(bytes); : + if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) { : + throw new SolrException( : + SolrException.ErrorCode.BAD_REQUEST, : + String.format( : + Locale.ROOT, : + "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s", : + file, : + FileTypeMagicUtil.INSTANCE.guessMimeType(bytes))); : + } : + return FileVisitResult.CONTINUE; : + } : + } : + : + @Override : + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) : + throws IOException { : + if (SKIP_FOLDERS.contains(dir.getFileName().toString())) : + return FileVisitResult.SKIP_SUBTREE; : + : + return FileVisitResult.CONTINUE; : + } : + }); : + } : + : + /** : + * Guess the mime type of file based on its magic number. : + * : + * @param stream input stream of the file : + * @return string with content-type or "application/octet-stream" if unknown : + */ : + public String guessMimeType(InputStream stream) { : + try { : + ContentInfo info = util.findMatch(stream); : + if (info == null) { : + return ContentType.OTHER.getMimeType(); : + } : + return info.getContentType().getMimeType(); : + } catch (IOException e) { : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); : + } : + } : + : + /** : + * Guess the mime type of file bytes based on its magic number. : + * : + * @param bytes the first bytes at start of the file : + * @return string with content-type or "application/octet-stream" if unknown : + */ : + public String guessMimeType(byte[] bytes) { : + return guessMimeType(new ByteArrayInputStream(bytes)); : + } : + : + @Override : + public void error(String line, String details, Exception e) { : + throw new SolrException( : + SolrException.ErrorCode.SERVER_ERROR, : + String.format(Locale.ROOT, "%s: %s", line, details), : + e); : + } : + : + /** : + * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types : + * are: : + * : + * <ul> : + * <li><code>application/x-java-applet</code>: java class file : + * <li><code>application/zip</code>: jar or zip archives : + * <li><code>application/x-tar</code>: tar archives : + * <li><code>text/x-shellscript</code>: shell or bash script : + * </ul> : + * : + * @param fileStream stream from the file content : + * @return true if file is among the forbidden mime-types : + */ : + public static boolean isFileForbiddenInConfigset(InputStream fileStream) { : + return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream)); : + } : + : + /** : + * Determine forbidden file type based on magic bytes matching of the first bytes of the file. : + * : + * @param bytes byte array of the file content : + * @return true if file is among the forbidden mime-types : + */ : + public static boolean isFileForbiddenInConfigset(byte[] bytes) { : + if (bytes == null || bytes.length == 0) : + return false; // A ZK znode may be a folder with no content : + return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes)); : + } : + : + private static final Set<String> forbiddenTypes = : + new HashSet<>( : + Arrays.asList( : + System.getProperty( : + "solr.configset.upload.mimetypes.forbidden", : + "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript") : + .split(","))); : +} : diff --git a/solr/core/src/resources/magic/executables b/solr/core/src/resources/magic/executables : new file mode 100644 : index 00000000000..04094eaf797 : --- /dev/null : +++ b/solr/core/src/resources/magic/executables : @@ -0,0 +1,74 @@ : +# Licensed to the Apache Software Foundation (ASF) under one or more : +# contributor license agreements. See the NOTICE file distributed with : +# this work for additional information regarding copyright ownership. : +# The ASF licenses this file to You under the Apache License, Version 2.0 : +# (the "License"); you may not use this file except in compliance with : +# the License. You may obtain a copy of the License at : +# : +# http://www.apache.org/licenses/LICENSE-2.0 : +# : +# Unless required by applicable law or agreed to in writing, software : +# distributed under the License is distributed on an "AS IS" BASIS, : +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. : +# See the License for the specific language governing permissions and : +# limitations under the License. : + : +# POSIX tar archives : +# URL: https://en.wikipedia.org/wiki/Tar_(computing) : +# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current : +# header mainly padded with nul bytes : +500 quad 0 : +!:strength /2 : +# filename or extended attribute printable strings in range space null til umlaut ue : +>0 ubeshort >0x1F00 : +>>0 ubeshort <0xFCFD : +# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad : +# at https://sourceforge.net/projects/s-tar/files/testscripts/ : +>>>508 ubelong&0x8B9E8DFF 0 : +# nul, space or ascii digit 0-7 at start of mode : +>>>>100 ubyte&0xC8 =0 : +>>>>>101 ubyte&0xC8 =0 : +# nul, space at end of check sum : +>>>>>>155 ubyte&0xDF =0 : +# space or ascii digit 0 at start of check sum : +>>>>>>>148 ubyte&0xEF =0x20 : +# check for specific 1st member name that indicates other mime type and file name suffix : +>>>>>>>>0 string TpmEmuTpms/permall : +!:mime application/x-tar : +!:ext tar : +# other stuff in padding : +# some implementations add new fields to the blank area at the end of the header record : +# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option : +>>257 ulong !0 tar archive (old) : +!:mime application/x-tar : +!:ext tar : +# magic in newer, GNU, posix variants : +>257 string =ustar : +# 2 last char of magic and UStar version because string expression does not work : +# 2 space characters followed by a null for GNU variant : +>>261 ubelong =0x72202000 POSIX tar archive (GNU) : +!:mime application/x-gtar : +!:ext tar/gtar : + : + : +# Zip archives (Greg Roelofs, c/o zip-b...@wkuvx1.wku.edu) : +0 string PK\005\006 Zip archive data (empty) : +0 string PK\003\004 Zip archive data : +!:strength +1 : +!:mime application/zip : +!:ext zip/cbz : + : + : +# JAVA : +0 belong 0xcafebabe : +>4 ubelong >30 compiled Java class data, : +!:mime application/x-java-applet : +#!:mime application/java-byte-code : +!:ext class : + : + : +# SHELL scripts : +#0 string/w : shell archive or script for antique kernel text : +0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable : +!:mime text/x-shellscript : +!:ext sh/bash : \ No newline at end of file : diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt b/solr/core/src/test-files/magic/HelloWorld.java.txt : new file mode 100644 : index 00000000000..ca9518d2afa : --- /dev/null : +++ b/solr/core/src/test-files/magic/HelloWorld.java.txt : @@ -0,0 +1,5 @@ : +class HelloWorld { : + public static void main(String[] args) { : + System.out.println("Hellow world"); : + } : +} : \ No newline at end of file : diff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin : new file mode 100644 : index 00000000000..e15d0a6c5b9 : Binary files /dev/null and b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ : diff --git a/solr/core/src/test-files/magic/README.md b/solr/core/src/test-files/magic/README.md : new file mode 100644 : index 00000000000..6e499a2f711 : --- /dev/null : +++ b/solr/core/src/test-files/magic/README.md : @@ -0,0 +1,29 @@ : +<!-- : + Licensed to the Apache Software Foundation (ASF) under one or more : + contributor license agreements. See the NOTICE file distributed with : + this work for additional information regarding copyright ownership. : + The ASF licenses this file to You under the Apache License, Version 2.0 : + the "License"); you may not use this file except in compliance with : + the License. You may obtain a copy of the License at : + : + http://www.apache.org/licenses/LICENSE-2.0 : + : + Unless required by applicable law or agreed to in writing, software : + distributed under the License is distributed on an "AS IS" BASIS, : + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. : + See the License for the specific language governing permissions and : + limitations under the License. : + --> : + : +The two binary files were created by the following commands: : + : +```bash : +echo "Hello" > hello.txt && \ : + tar -cvf hello.tar.bin hello.txt && \ : + rm hello.txt : + : +cp HelloWorld.java.txt HelloWorld.java && \ : + javac HelloWorld.java && \ : + mv HelloWorld.class HelloWorldJavaClass.class.bin && \ : + rm HelloWorld.java : +``` : \ No newline at end of file : diff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.bin : new file mode 100644 : index 00000000000..68ca23c362a : Binary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differ : diff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txt : new file mode 100644 : index 00000000000..70c379b63ff : --- /dev/null : +++ b/solr/core/src/test-files/magic/plain.txt : @@ -0,0 +1 @@ : +Hello world : \ No newline at end of file : diff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txt : new file mode 100644 : index 00000000000..9ea411e111d : --- /dev/null : +++ b/solr/core/src/test-files/magic/shell.sh.txt : @@ -0,0 +1,2 @@ : +#! /usr/bin/env bash : +echo Hello : \ No newline at end of file : diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java : index 1da60d85ff9..f7c9431c296 100644 : --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java : +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java : @@ -45,6 +45,7 @@ import java.util.LinkedHashMap; : import java.util.List; : import java.util.Locale; : import java.util.Map; : +import java.util.Objects; : import java.util.Properties; : import java.util.Set; : import java.util.concurrent.TimeUnit; : @@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : assertEquals( : "Can't overwrite an existing configset unless the overwrite parameter is set", : 400, : - uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false)); : + uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false)); : unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); : assertEquals( : "Expecting version to remain equal", : solrconfigZkVersion, : getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : zkClient.makePath(f, true); : } : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); : for (String f : extraFiles) { : assertTrue( : "Expecting file " + f + " to exist in ConfigSet but it's gone", : zkClient.exists(f, true)); : } : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false)); : + assertEquals( : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false)); : for (String f : extraFiles) { : assertFalse( : "Expecting file " + f + " to be deleted from ConfigSet but it wasn't", : @@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : .withConnTimeOut(45000, TimeUnit.MILLISECONDS) : .build()) { : String configPath = "/configs/" + configsetName + configsetSuffix; : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true)); : + assertEquals( : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false)); : for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { : String f = configPath + "/test." + fileEnding; : assertFalse( : @@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); : // Was untrusted, overwrite with untrusted : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : : // Was untrusted, overwrite with trusted but no cleanup : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false)); : + 0, : + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : : // Was untrusted, overwrite with trusted with cleanup : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false)); : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : assertEquals( : "Can't upload a trusted configset with an untrusted request", : 400, : - uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); : + uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); : assertEquals( : "Expecting version to remain equal", : solrconfigZkVersion, : @@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : assertEquals( : "Can't upload a trusted configset with an untrusted request", : 400, : - uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false)); : + uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false)); : assertEquals( : "Expecting version to remain equal", : solrconfigZkVersion, : @@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : : // Was trusted, overwrite with trusted no cleanup : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false)); : + 0, : + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : : // Was trusted, overwrite with trusted with cleanup : assertEquals( : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false)); : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false)); : assertTrue( : "Expecting version bump", : solrconfigZkVersion : @@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : .get("id")); : } : : + @Test : + public void testUploadWithForbiddenContent() throws Exception { : + // Uploads a config set containing a script, a class file and jar file, will return 400 error : + long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true); : + assertEquals(400, res); : + } : + : private static String getSecurityJson() { : return "{\n" : + " 'authentication':{\n" : @@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2) : throws IOException { : assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix)); : - return uploadConfigSet(configSetName, suffix, username, false, false, v2, false); : + return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false); : } : : private long uploadConfigSet( : @@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : boolean overwrite, : boolean cleanup, : boolean v2, : - boolean forbiddenTypes) : + boolean forbiddenTypes, : + boolean forbiddenContent) : throws IOException { : : + File zipFile; : + if (forbiddenTypes) { : + log.info("Uploading configset with forbidden file endings"); : + zipFile = : + createTempZipFileWithForbiddenTypes( : + "solr/configsets/upload/" + configSetName + "/solrconfig.xml"); : + } else if (forbiddenContent) { : + log.info("Uploading configset with forbidden file content"); : + zipFile = createTempZipFileWithForbiddenContent("magic"); : + } else { : + zipFile = createTempZipFile("solr/configsets/upload/" + configSetName); : + } : + : // Read zipped sample config : - return uploadGivenConfigSet( : - forbiddenTypes : - ? createTempZipFileWithForbiddenTypes( : - "solr/configsets/upload/" + configSetName + "/solrconfig.xml") : - : createTempZipFile("solr/configsets/upload/" + configSetName), : - configSetName, : - suffix, : - username, : - overwrite, : - cleanup, : - v2); : + return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2); : } : : private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2) : @@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { : } : } : : - private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException { : - OutputStream out = new FileOutputStream(zipfile); : - ZipOutputStream zout = new ZipOutputStream(out); : + /** Create a zip file (in the temp directory) containing files with forbidden content */ : + private File createTempZipFileWithForbiddenContent(String resourcePath) { : try { : - for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { : - zout.putNextEntry(new ZipEntry("test." + fileType)); : + final File zipFile = createTempFile("configset", "zip").toFile(); : + final File directory = SolrTestCaseJ4.getFile(resourcePath); : + if (log.isInfoEnabled()) { : + log.info("Directory: {}", directory.getAbsolutePath()); : + } : + zipWithForbiddenContent(directory, zipFile); : + if (log.isInfoEnabled()) { : + log.info("Zipfile: {}", zipFile.getAbsolutePath()); : + } : + return zipFile; : + } catch (IOException e) { : + throw new RuntimeException(e); : + } : + } : : - InputStream in = new FileInputStream(file); : - try { : - byte[] buffer = new byte[1024]; : - while (true) { : - int readCount = in.read(buffer); : - if (readCount < 0) { : - break; : + private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException { : + OutputStream out = Files.newOutputStream(zipfile.toPath()); : + assertTrue(directory.isDirectory()); : + try (ZipOutputStream zout = new ZipOutputStream(out)) { : + // Copy in all files from the directory : + for (File file : Objects.requireNonNull(directory.listFiles())) { : + zout.putNextEntry(new ZipEntry(file.getName())); : + zout.write(Files.readAllBytes(file.toPath())); : + zout.closeEntry(); : + } : + } : + } : + : + private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile) : + throws IOException { : + OutputStream out = new FileOutputStream(zipfile); : + try (ZipOutputStream zout = new ZipOutputStream(out)) { : + if (fileOrDirectory.isFile()) { : + // Create entries with given file, one for each forbidden endding : + for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { : + zout.putNextEntry(new ZipEntry("test." + fileType)); : + : + try (InputStream in = new FileInputStream(fileOrDirectory)) { : + byte[] buffer = new byte[1024]; : + while (true) { : + int readCount = in.read(buffer); : + if (readCount < 0) { : + break; : + } : + zout.write(buffer, 0, readCount); : } : - zout.write(buffer, 0, readCount); : } : - } finally { : - in.close(); : - } : : - zout.closeEntry(); : + zout.closeEntry(); : + } : + } : + if (fileOrDirectory.isDirectory()) { : + // Copy in all files from the directory : + for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) { : + zout.putNextEntry(new ZipEntry(file.getName())); : + zout.write(Files.readAllBytes(file.toPath())); : + zout.closeEntry(); : + } : } : - } finally { : - zout.close(); : } : } : : diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java : new file mode 100644 : index 00000000000..b8e9a35a3d9 : --- /dev/null : +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java : @@ -0,0 +1,54 @@ : +/* : + * Licensed to the Apache Software Foundation (ASF) under one or more : + * contributor license agreements. See the NOTICE file distributed with : + * this work for additional information regarding copyright ownership. : + * The ASF licenses this file to You under the Apache License, Version 2.0 : + * (the "License"); you may not use this file except in compliance with : + * the License. You may obtain a copy of the License at : + * : + * http://www.apache.org/licenses/LICENSE-2.0 : + * : + * Unless required by applicable law or agreed to in writing, software : + * distributed under the License is distributed on an "AS IS" BASIS, : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. : + * See the License for the specific language governing permissions and : + * limitations under the License. : + */ : + : +package org.apache.solr.util; : + : +import org.apache.solr.SolrTestCaseJ4; : + : +public class FileTypeMagicUtilTest extends SolrTestCaseJ4 { : + public void testGuessMimeType() { : + assertEquals( : + "application/x-java-applet", : + FileTypeMagicUtil.INSTANCE.guessMimeType( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); : + assertEquals( : + "application/zip", : + FileTypeMagicUtil.INSTANCE.guessMimeType( : + FileTypeMagicUtil.class.getResourceAsStream( : + "/runtimecode/containerplugin.v.1.jar.bin"))); : + assertEquals( : + "application/x-tar", : + FileTypeMagicUtil.INSTANCE.guessMimeType( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin"))); : + assertEquals( : + "text/x-shellscript", : + FileTypeMagicUtil.INSTANCE.guessMimeType( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); : + } : + : + public void testIsFileForbiddenInConfigset() { : + assertTrue( : + FileTypeMagicUtil.isFileForbiddenInConfigset( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); : + assertTrue( : + FileTypeMagicUtil.isFileForbiddenInConfigset( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); : + assertFalse( : + FileTypeMagicUtil.isFileForbiddenInConfigset( : + FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt"))); : + } : +} : diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1 : new file mode 100644 : index 00000000000..cf101094cc8 : --- /dev/null : +++ b/solr/licenses/simplemagic-1.17.jar.sha1 : @@ -0,0 +1 @@ : +b6e2d1e47d7172e57fa858a2e3940c09a590e61e : diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt : new file mode 100644 : index 00000000000..9228230f933 : --- /dev/null : +++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt : @@ -0,0 +1,15 @@ : +ISC License (https://opensource.org/licenses/ISC) : + : +Copyright 2021, Gray Watson : + : +Permission to use, copy, modify, and/or distribute this software for any : +purpose with or without fee is hereby granted, provided that the above : +copyright notice and this permission notice appear in all copies. : + : +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES : +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF : +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR : +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES : +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN : +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF : +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. : \ No newline at end of file : diff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txt : new file mode 100644 : index 00000000000..e69de29bb2d : diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java : index d571339880c..e40294a6683 100644 : --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java : +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java : @@ -348,6 +348,7 @@ public class ZkMaintenanceUtils { : USE_FORBIDDEN_FILE_TYPES); : return FileVisitResult.CONTINUE; : } : + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core : String zkNode = createZkNodeName(zkPath, rootPath, file); : try { : // if the path exists (and presumably we're uploading data to it) just set its data : @@ -437,6 +438,7 @@ public class ZkMaintenanceUtils { : if (isFileForbiddenInConfigSets(zkPath)) { : log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath); : } else { : + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core : if (copyDataDown(zkClient, zkPath, file) == 0) { : Files.createFile(file); : } : diff --git a/versions.lock b/versions.lock : index 9c0f9ef53d4..6b6ea467e0d 100644 : --- a/versions.lock : +++ b/versions.lock : @@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5) : com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201) : com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101) : com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784) : +com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830) : com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c) : com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36) : com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b) : diff --git a/versions.props b/versions.props : index 10b583c43e2..cfa127e1094 100644 : --- a/versions.props : +++ b/versions.props : @@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0 : com.google.errorprone:*=2.23.0 : com.google.guava:guava=32.1.3-jre : com.google.re2j:re2j=1.7 : +com.j256.simplemagic:simplemagic=1.17 : com.jayway.jsonpath:json-path=2.8.0 : com.lmax:disruptor=3.4.4 : com.tdunning:t-digest=3.1 : : -Hoss
--------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@solr.apache.org For additional commands, e-mail: dev-h...@solr.apache.org
