This is an automated email from the ASF dual-hosted git repository.
smiklosovic pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new b2037e473f Allow overriding arbitrary settings via environment
variables
b2037e473f is described below
commit b2037e473fb6438947d6ed9c58fbea5955cb72c4
Author: Paulo Motta <[email protected]>
AuthorDate: Mon Jul 7 15:23:05 2025 -0400
Allow overriding arbitrary settings via environment variables
This also allows overriding complex settings as a JSON value and adds
documentation about these overrides to conf/jvm-server.options
patch by Paulo Motta; reviewed by Stefan Miklosovic, David Capwell for
CASSANDRA-20749
---
CHANGES.txt | 1 +
conf/jvm-server.options | 16 ++
.../cassandra/config/CassandraRelevantEnv.java | 10 ++
.../config/CassandraRelevantProperties.java | 1 +
.../cassandra/config/DatabaseDescriptor.java | 5 +
.../org/apache/cassandra/config/Properties.java | 18 +++
.../cassandra/config/YamlConfigurationLoader.java | 131 +++++++++++++--
.../distributed/shared/WithEnvironment.java | 97 +++++++++++
.../config/YamlConfigurationLoaderTest.java | 180 ++++++++++++++++++++-
9 files changed, 440 insertions(+), 19 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index ba6b0ddc65..bc3fa7b424 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.1
+ * Allow overriding arbitrary settings via environment variables
(CASSANDRA-20749)
* Optimize MessagingService.getVersionOrdinal (CASSANDRA-20816)
* Optimize TrieMemtable#getFlushSet (CASSANDRA-20760)
* Support manual secondary index selection at the CQL level (CASSANDRA-18112)
diff --git a/conf/jvm-server.options b/conf/jvm-server.options
index c63863aa3b..d850592db0 100644
--- a/conf/jvm-server.options
+++ b/conf/jvm-server.options
@@ -43,6 +43,22 @@
# The directory location of the cassandra.yaml file.
#-Dcassandra.config=directory
+# Allow cassandra.yaml settings to be overriden via JVM properties
+# When this setting is enabled, cassandra.yaml settings can be overriden via
JVM properties in the format -Dcassandra.settings.<cassandra_yaml_property_name>
+# For example, override cassandra.yaml property 'cdc_enabled' via JVM property
-Dcassandra.settings.cdc_enabled
+# Nested properties can be specified using '.' as separator, for example:
-Dcassandra.settings.replica_filtering_protection.cached_rows_warn_threshold
+# Complex property values should be specified as a JSON string, for example:
-Dcassandra.settings.table_properties_warned="[\"sstable_preemptive_open_interval_in_mb\",
\"index_summary_resize_interval_in_minutes\"]"
+#-Dcassandra.config.allow_system_properties=true
+
+# Allow cassandra.yaml settings to be overriden via environment variables
+# When this setting is enabled, cassandra.yaml settings can be overriden via
environment variables in the format
'CASSANDRA_SETTINGS_<CASSANDRA_YAML_PROPERTY_NAME>'
+# For example, override cassandra.yaml property 'cdc_enabled' via env var:
export CASSANDRA_SETTINGS_CDC_ENABLED=true
+# Nested properties can be specified using '__' as separator, for example:
export
CASSANDRA_SETTINGS_REPLICA_FILTERING_PROTECTION__CACHED_ROWS_WARN_THRESHOLD=1000
+# Complex property values should be specified as a JSON string, for example:
export CASSANDRA_SETTINGS_TABLE_PROPERTIES_WARNED='["bloom_filter_fp_chance",
"default_time_to_live"]'
+# This feature can also be enabled via the
CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES environment variable via export
CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES=true'
+# The system property has precedence over the environment variable when both
are specified.
+#-Dcassandra.config.allow_environment_variables=true
+
# Sets the initial partitioner token for a node the first time the node is
started.
#-Dcassandra.initial_token=token
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
b/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
index d970cf7fde..e563c59176 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantEnv.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.config;
// checkstyle: suppress below 'blockSystemPropertyUsage'
import java.util.Arrays;
+import java.util.Optional;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -38,6 +39,10 @@ public enum CassandraRelevantEnv
/** By default, the standard Cassandra CLI layout is used for backward
compatibility, however,
* the new Picocli layout can be enabled by setting this property to the
{@code "picocli"}. */
CASSANDRA_CLI_LAYOUT("CASSANDRA_CLI_LAYOUT"),
+ /**
+ * Allow overriding
+ */
+
CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES("CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES")
;
CassandraRelevantEnv(String key)
@@ -61,6 +66,11 @@ public enum CassandraRelevantEnv
return Boolean.parseBoolean(System.getenv(key));
}
+ public boolean getBooleanOrDefault(boolean defaultValue)
+ {
+ return
Optional.ofNullable(System.getenv(key)).map(Boolean::parseBoolean).orElse(defaultValue);
+ }
+
public String getKey() {
return key;
}
diff --git
a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index a518497e8f..9492d97401 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -184,6 +184,7 @@ public enum CassandraRelevantProperties
* Default is set to false.
*/
COM_SUN_MANAGEMENT_JMXREMOTE_SSL_NEED_CLIENT_AUTH("com.sun.management.jmxremote.ssl.need.client.auth"),
+
CONFIG_ALLOW_ENVIRONMENT_VARIABLES("cassandra.config.allow_environment_variables"),
/** Defaults to false for 4.1 but plan to switch to true in a later
release the thinking is that environments
* may not work right off the bat so safer to add this feature disabled by
default */
CONFIG_ALLOW_SYSTEM_PROPERTIES("cassandra.config.allow_system_properties"),
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index f38dc0e9d5..d605d0cd98 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -476,6 +476,11 @@ public class DatabaseDescriptor
return conf;
}
+ public static boolean hasLoggedConfig()
+ {
+ return hasLoggedConfig;
+ }
+
@VisibleForTesting
public static Config loadConfig() throws ConfigurationException
{
diff --git a/src/java/org/apache/cassandra/config/Properties.java
b/src/java/org/apache/cassandra/config/Properties.java
index 79852d4af6..2e9296f257 100644
--- a/src/java/org/apache/cassandra/config/Properties.java
+++ b/src/java/org/apache/cassandra/config/Properties.java
@@ -91,6 +91,21 @@ public final class Properties
* @return map of all flattened properties
*/
public static Map<String, Property> flatten(Loader loader, Map<String,
Property> input, String delimiter)
+ {
+ return flatten(loader, input, delimiter, false);
+ }
+
+ /**
+ * Given a map of Properties, takes any "nested" property (non primitive,
value-type, or collection), and
+ * expands them, producing 1 or more Properties.
+ *
+ * @param loader for mapping type to map of properties
+ * @param input map to flatten
+ * @param delimiter for joining names
+ * @param withInnerProperties also adds intermediate properties among
flattened ones.
+ * @return map of all flattened properties
+ */
+ public static Map<String, Property> flatten(Loader loader, Map<String,
Property> input, String delimiter, boolean withInnerProperties)
{
Queue<Property> queue = new ArrayDeque<>(input.values());
@@ -106,6 +121,9 @@ public final class Properties
}
else
{
+ if (withInnerProperties)
+ output.put(prop.getName(), prop);
+
children.values().stream().map(p -> andThen(prop, p,
delimiter)).forEach(queue::add);
}
}
diff --git a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
index f37a42e8fa..37e45b5089 100644
--- a/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
+++ b/src/java/org/apache/cassandra/config/YamlConfigurationLoader.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeMap;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
@@ -41,6 +42,8 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.utils.JsonUtils;
+import org.apache.cassandra.utils.LocalizeString;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.TypeDescription;
@@ -58,13 +61,29 @@ import org.yaml.snakeyaml.parser.ParserImpl;
import org.yaml.snakeyaml.representer.Representer;
import org.yaml.snakeyaml.resolver.Resolver;
+import static
org.apache.cassandra.config.CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES;
import static
org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_DUPLICATE_CONFIG_KEYS;
import static
org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_NEW_OLD_CONFIG_KEYS;
import static
org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
+import static
org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_ENVIRONMENT_VARIABLES;
+import static
org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES;
import static org.apache.cassandra.config.Replacements.getNameReplacements;
public class YamlConfigurationLoader implements ConfigurationLoader
{
+ final static Set<String> OVERRIDABLE_CONFIG_NAMES;
+
+ static
+ {
+ // Configs can be overriden via system properties and environment
variables entirely or partially
+ // For example: sai_options.prioritize_over_legacy_index=true or
+ // sai_options: {prioritize_over_legacy_index=true,
segment_write_buffer_size=100MiB}
+ Loader loader = Properties.defaultLoader();
+ Map<String, Property> topLevelConfigs =
loader.getProperties(Config.class);
+ Map<String, Property> flattenedConfigs = Properties.flatten(loader,
loader.getProperties(Config.class), Properties.DELIMITER, true);
+ OVERRIDABLE_CONFIG_NAMES =
Collections.unmodifiableSet(Sets.union(topLevelConfigs.keySet(),
flattenedConfigs.keySet()));
+ }
+
private static final Logger logger =
LoggerFactory.getLogger(YamlConfigurationLoader.class);
/**
@@ -72,6 +91,9 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
* system properties do not conflict with other system properties; the
name "settings" matches system_views.settings.
*/
static final String SYSTEM_PROPERTY_PREFIX = "cassandra.settings.";
+ static final String ENVIRONMENT_VARIABLE_PREFIX = "CASSANDRA_SETTINGS_";
+ public static final String NESTED_CONFIG_SEPARATOR = ".";
+ public static final String NESTED_CONFIG_SEPARATOR_ENVIRONMENT = "__";
/**
* Inspect the classpath to find storage configuration file
@@ -154,27 +176,112 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
Yaml yaml = new Yaml(constructor);
Config result = loadConfig(yaml, configBytes);
propertiesChecker.check();
+ maybeAddEnvironmentVariables(result);
maybeAddSystemProperties(result);
return result;
}
private static void maybeAddSystemProperties(Object obj)
{
- if
(CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES.getBoolean())
+ if (CONFIG_ALLOW_SYSTEM_PROPERTIES.getBoolean())
{
+ Map<String, String> orderedPropertiesMap = new TreeMap<>();
java.util.Properties props = System.getProperties();
- Map<String, String> map = new HashMap<>();
- for (String name : props.stringPropertyNames())
+ props.stringPropertyNames().forEach(key ->
orderedPropertiesMap.put(key, props.getProperty(key)));
+
+ Map<String, Object> overridingProperties = new HashMap<>();
+ for (String originalKey : orderedPropertiesMap.keySet())
+ {
+ if (originalKey.startsWith(SYSTEM_PROPERTY_PREFIX))
+ {
+ String value = props.getProperty(originalKey);
+ String configKey =
originalKey.replace(SYSTEM_PROPERTY_PREFIX, "");
+ if (OVERRIDABLE_CONFIG_NAMES.contains(configKey))
+ {
+ if (value != null &&
!overridingProperties.containsKey(configKey))
+ {
+ if (!DatabaseDescriptor.hasLoggedConfig()) //
CASSANDRA-9909: Avoid flooding config during initialization
+ logger.warn("Detected JVM property {}={}
override for Cassandra configuration '{}'.", originalKey, value, configKey);
+ overridingProperties.put(configKey,
getScalarOrJsonTree(value));
+ }
+ }
+ else
+ {
+ logger.warn("Used sytem property variable {} to
override Cassandra configuration but there is no such system property
counter-part to override.", originalKey);
+ }
+ }
+ }
+ if (!overridingProperties.isEmpty())
+
updateFromMap(maybeFlattenNestedProperties(overridingProperties), false, obj);
+ }
+ }
+
+ private static void maybeAddEnvironmentVariables(Object obj)
+ {
+ if
(CONFIG_ALLOW_ENVIRONMENT_VARIABLES.getBoolean(CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getBooleanOrDefault(false)))
+ {
+ Map<String, String> orderedEnvironmentMap = new
TreeMap<>(System.getenv()); // checkstyle: suppress nearby
'blockSystemPropertyUsage'
+ Map<String, Object> overridingProperties = new HashMap<>();
+ for (Map.Entry<String, String> env :
orderedEnvironmentMap.entrySet())
+ {
+ String originalKey = env.getKey();
+ if (env.getKey().startsWith(ENVIRONMENT_VARIABLE_PREFIX))
+ {
+ String configKey =
LocalizeString.toLowerCaseLocalized(originalKey.replace(ENVIRONMENT_VARIABLE_PREFIX,
"")
+
.replace(NESTED_CONFIG_SEPARATOR_ENVIRONMENT, NESTED_CONFIG_SEPARATOR));
+ String configValue = env.getValue();
+ if (OVERRIDABLE_CONFIG_NAMES.contains(configKey))
+ {
+ if (configValue != null &&
!overridingProperties.containsKey(configKey))
+ {
+ if (!DatabaseDescriptor.hasLoggedConfig()) //
CASSANDRA-9909: Avoid flooding config during initialization
+ logger.warn("Detected environment variable
{}={} override for Cassandra configuration '{}'.", originalKey, configValue,
configKey);
+ overridingProperties.put(configKey,
getScalarOrJsonTree(configValue));
+ }
+ }
+ else
+ {
+ logger.warn("Used environment property variable {} to
override Cassandra configuration but there is no such environment property
counter-part to override.", originalKey);
+ }
+ }
+ }
+ if (!overridingProperties.isEmpty())
+
updateFromMap(maybeFlattenNestedProperties(overridingProperties), false, obj);
+ }
+ }
+
+ private static Map<String, Object>
maybeFlattenNestedProperties(Map<String, Object> overridingProperties)
+ {
+ Map<String, Object> copyOfProperties = new
HashMap<>(overridingProperties);
+ for (Map.Entry<String, Object> entry : overridingProperties.entrySet())
+ {
+ String[] parts = entry.getKey().split("\\.");
+ if (parts.length > 1 && !parts[parts.length -
1].equals("parameters") && !parts[parts.length - 1].equals("configurations"))
{
- if (name.startsWith(SYSTEM_PROPERTY_PREFIX))
+ if (entry.getValue() instanceof Map)
{
- String value = props.getProperty(name);
- if (value != null)
- map.put(name.replace(SYSTEM_PROPERTY_PREFIX, ""),
value);
+ copyOfProperties.remove(entry.getKey());
+ for (Map.Entry<String, Object> mapEntry : ((Map<String,
Object>) entry.getValue()).entrySet())
+ {
+ String newKey = entry.getKey() + '.' +
mapEntry.getKey();
+ Object newValue = mapEntry.getValue();
+ copyOfProperties.put(newKey, newValue);
+ }
}
}
- if (!map.isEmpty())
- updateFromMap(map, false, obj);
+ }
+ return copyOfProperties;
+ }
+
+ private static Object getScalarOrJsonTree(String value)
+ {
+ try
+ {
+ return JsonUtils.decodeJson(value);
+ }
+ catch (Exception e)
+ {
+ return value;
}
}
@@ -237,6 +344,7 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
T value = (T) constructor.getSingleData(klass);
if (shouldCheck)
propertiesChecker.check();
+ maybeAddEnvironmentVariables(value);
maybeAddSystemProperties(value);
return value;
}
@@ -371,7 +479,7 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
{
Replacement replacement = typeReplacements.get(name);
result = replacement.toProperty(getProperty0(type,
replacement.newName));
-
+
if (replacement.deprecated)
deprecationWarnings.add(replacement.oldName);
}
@@ -413,7 +521,7 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
private Property getProperty0(Class<? extends Object> type, String
name)
{
- if (name.contains("."))
+ if (name.contains(NESTED_CONFIG_SEPARATOR))
return getNestedProperty(type, name);
return getFlatProperty(type, name);
}
@@ -461,4 +569,3 @@ public class YamlConfigurationLoader implements
ConfigurationLoader
return loaderOptions;
}
}
-
diff --git
a/test/distributed/org/apache/cassandra/distributed/shared/WithEnvironment.java
b/test/distributed/org/apache/cassandra/distributed/shared/WithEnvironment.java
new file mode 100644
index 0000000000..03d09e3c71
--- /dev/null
+++
b/test/distributed/org/apache/cassandra/distributed/shared/WithEnvironment.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.shared;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public final class WithEnvironment implements AutoCloseable
+{
+ private final List<Environment> properties = new ArrayList<>();
+
+ public WithEnvironment(String... kvs)
+ {
+ with(kvs);
+ }
+
+ public void with(String... kvs)
+ {
+ assert kvs.length % 2 == 0 : "Input must have an even amount of inputs
but given " + kvs.length;
+ for (int i = 0; i <= kvs.length - 2; i = i + 2)
+ {
+ with(kvs[i], kvs[i + 1]);
+ }
+ }
+
+ public void with(String key, String value)
+ {
+ try
+ {
+ Map<String, String> writableEnv = getWritableEnv();
+ String previous = writableEnv.put(key, value);
+ properties.add(new Environment(key, previous));
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to set environment
variable", e);
+ }
+ }
+
+ private static Map<String, String> getWritableEnv() throws
NoSuchFieldException, IllegalAccessException
+ {
+ Map<String, String> env = System.getenv(); // checkstyle: suppress
nearby 'blockSystemPropertyUsage'
+ Class<?> cl = env.getClass();
+ Field field = cl.getDeclaredField("m");
+ field.setAccessible(true);
+ return (Map<String, String>) field.get(env);
+ }
+
+
+ @Override
+ public void close()
+ {
+ Collections.reverse(properties);
+ properties.forEach(s -> {
+ try
+ {
+ Map<String, String> writableEnv = getWritableEnv();
+ if (s.value == null)
+ writableEnv.remove(s.key);
+ else
+ writableEnv.put(s.key, s.value);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to set environment
variable", e);
+ }
+ });
+ properties.clear();
+ }
+
+ private static final class Environment
+ {
+ private final String key;
+ private final String value;
+
+ private Environment(String key, String value)
+ {
+ this.key = key;
+ this.value = value;
+ }
+ }
+}
diff --git
a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
index 68d5302047..afa820d60e 100644
--- a/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
+++ b/test/unit/org/apache/cassandra/config/YamlConfigurationLoaderTest.java
@@ -27,23 +27,30 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Predicate;
import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
import org.junit.Test;
+import org.apache.cassandra.distributed.shared.WithEnvironment;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.apache.cassandra.distributed.shared.WithProperties;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StartupChecks;
import org.apache.cassandra.repair.autorepair.AutoRepairConfig;
+import org.assertj.core.api.Assertions;
import org.yaml.snakeyaml.error.YAMLException;
+import static
org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_ENVIRONMENT_VARIABLES;
import static
org.apache.cassandra.config.CassandraRelevantProperties.CONFIG_ALLOW_SYSTEM_PROPERTIES;
import static
org.apache.cassandra.config.DataStorageSpec.DataStorageUnit.KIBIBYTES;
+import static
org.apache.cassandra.config.YamlConfigurationLoader.ENVIRONMENT_VARIABLE_PREFIX;
import static
org.apache.cassandra.config.YamlConfigurationLoader.SYSTEM_PROPERTY_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -138,28 +145,187 @@ public class YamlConfigurationLoaderTest
@Test
public void withSystemProperties()
{
- // for primitive types or data-types which use a String constructor,
we can support these as nested
- // if the type is a collection, then the string format doesn't make
sense and will fail with an error such as
- // Cannot create property=client_encryption_options.cipher_suites
for JavaBean=org.apache.cassandra.config.Config@1f59a598
- // No single argument constructor found for interface java.util.List
: null
- // the reason is that its not a scalar but a complex type (collection
type), so the map we use needs to have a collection to match.
- // It is possible that we define a common string representation for
these types so they can be written to; this
- // is an issue that SettingsTable may need to worry about.
try (WithProperties ignore = new WithProperties()
.set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
.with(SYSTEM_PROPERTY_PREFIX +
"storage_port", "123",
SYSTEM_PROPERTY_PREFIX +
"commitlog_sync", "batch",
SYSTEM_PROPERTY_PREFIX +
"seed_provider.class_name", "org.apache.cassandra.locator.SimpleSeedProvider",
+ SYSTEM_PROPERTY_PREFIX +
"seed_provider.parameters", "{\"seeds\": \"127.0.0.1:7000,127.0.0.1:7001\"}",
+ SYSTEM_PROPERTY_PREFIX +
"client_encryption_options.cipher_suites", "[\"FakeCipher\"]",
SYSTEM_PROPERTY_PREFIX +
"client_encryption_options.optional", Boolean.FALSE.toString(),
SYSTEM_PROPERTY_PREFIX +
"client_encryption_options.enabled", Boolean.TRUE.toString(),
+ SYSTEM_PROPERTY_PREFIX +
"sai_options", "{\"prioritize_over_legacy_index\": \"true\",
\"segment_write_buffer_size\": \"100MiB\"}",
+ SYSTEM_PROPERTY_PREFIX +
"crypto_provider", "{\"class_name\": \"MyClass\", \"parameters\":
{\"fail_on_missing_provider\": \"false\"}}",
+ SYSTEM_PROPERTY_PREFIX +
"table_properties_warned", "[\"bloom_filter_fp_chance\",
\"default_time_to_live\"]",
+ SYSTEM_PROPERTY_PREFIX +
"paxos_variant", "v2",
+ SYSTEM_PROPERTY_PREFIX +
"memtable.configurations", "{\"skiplist\": {\"class_name\":
\"SkipListMemtable\", \"parameters\": {\"skip_param1\":
\"skip_param1_value\"}}, \"trie\": {\"class_name\": \"TrieMemtable\",
\"parameters\": {\"trie_param1\": \"trie_param1_value\"}}, \"default\":
{\"inherits\": \"trie\"}}",
+ SYSTEM_PROPERTY_PREFIX +
"client_error_reporting_exclusions.subnets", "[\"127.0.0.1\",\"127.0.0.2\"]",
+ SYSTEM_PROPERTY_PREFIX +
"startup_checks", "{\"check_data_resurrection\": {\"enabled\": \"true\",
\"heartbeat_file\": \"/var/lib/cassandra/data/cassandra-heartbeat\"}}",
SYSTEM_PROPERTY_PREFIX +
"doesnotexist", Boolean.TRUE.toString()))
{
Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
assertThat(config.storage_port).isEqualTo(123);
assertThat(config.commitlog_sync).isEqualTo(Config.CommitLogSync.batch);
assertThat(config.seed_provider.class_name).isEqualTo("org.apache.cassandra.locator.SimpleSeedProvider");
+
assertThat(config.seed_provider.parameters.get("seeds")).isEqualTo("127.0.0.1:7000,127.0.0.1:7001");
+
assertThat(config.client_encryption_options.cipher_suites).isEqualTo(Collections.singletonList("FakeCipher"));
assertThat(config.client_encryption_options.optional).isFalse();
assertThat(config.client_encryption_options.enabled).isTrue();
+
assertThat(config.sai_options.prioritize_over_legacy_index).isTrue();
+
assertThat(config.sai_options.segment_write_buffer_size).isEqualTo(new
DataStorageSpec.IntMebibytesBound("100MiB"));
+ assertThat(config.crypto_provider.class_name).isEqualTo("MyClass");
+
assertThat(config.crypto_provider.parameters.get("fail_on_missing_provider")).isEqualTo(Boolean.FALSE.toString());
+
assertThat(config.table_properties_warned).isEqualTo(Set.of("bloom_filter_fp_chance",
"default_time_to_live"));
+ assertThat(config.paxos_variant).isEqualTo(Config.PaxosVariant.v2);
+ assertThat(config.client_error_reporting_exclusions).isEqualTo(new
SubnetGroups(Arrays.asList("127.0.0.2", "127.0.0.1")));
+ assertThat(config.startup_checks).hasSize(1);
+
assertThat(config.startup_checks.get(StartupChecks.StartupCheckType.check_data_resurrection).get("enabled")).isEqualTo(Boolean.TRUE.toString());
+
assertThat(config.startup_checks.get(StartupChecks.StartupCheckType.check_data_resurrection).get("heartbeat_file")).isEqualTo("/var/lib/cassandra/data/cassandra-heartbeat");
+ }
+
+ try (WithProperties ignore = new WithProperties()
+ .set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
+ .with(SYSTEM_PROPERTY_PREFIX +
"memtable.configurations", "{\"skiplist\": {\"class_name\":
\"SkipListMemtable\", \"parameters\": {\"skip_param1\":
\"skip_param1_value\"}}, \"trie\": {\"class_name\": \"TrieMemtable\",
\"parameters\": {\"trie_param1\": \"trie_param1_value\"}}, \"default\":
{\"inherits\": \"trie\"}}"))
+ {
+ Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
+ assertThat(config.memtable.configurations).hasSize(3);
+
assertThat(config.memtable.configurations.get("skiplist").class_name).isEqualTo("SkipListMemtable");
+
assertThat(config.memtable.configurations.get("skiplist").parameters.get("skip_param1")).isEqualTo("skip_param1_value");
+
assertThat(config.memtable.configurations.get("trie").class_name).isEqualTo("TrieMemtable");
+
assertThat(config.memtable.configurations.get("trie").parameters.get("trie_param1")).isEqualTo("trie_param1_value");
+
assertThat(config.memtable.configurations.get("default").inherits).isEqualTo("trie");
+ }
+
+ try (WithProperties ignore = new
WithProperties().set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
+ .with(SYSTEM_PROPERTY_PREFIX +
"crypto_provider.parameters", "{\"fail_on_missing_provider\": \"false\"}")
+ .with(SYSTEM_PROPERTY_PREFIX +
"crypto_provider.class_name", "MyClass"))
+ {
+ Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
+
+ ParameterizedClass cryptoProvider = config.crypto_provider;
+
+ Assert.assertEquals("MyClass", cryptoProvider.class_name);
+
+
Assert.assertTrue(cryptoProvider.parameters.containsKey("fail_on_missing_provider"));
+ String failOnMissingProviderValue =
cryptoProvider.parameters.get("fail_on_missing_provider");
+ Assert.assertNotNull(failOnMissingProviderValue);
+ Assert.assertEquals("false", failOnMissingProviderValue);
+ }
+
+ try (WithProperties ignore = new
WithProperties().set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
+
.with(SYSTEM_PROPERTY_PREFIX + "jmx_server_options.jmx_encryption_options",
+ '{' +
+ "\"enabled\":
true," +
+
"\"cipher_suites\": [\"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"]" +
+ '}'))
+ {
+ Config c = load("test/conf/cassandra-jmx-sslconfig.yaml");
+
+ Assert.assertTrue(c.jmx_server_options.enabled);
+
Assertions.assertThatCollection(c.jmx_server_options.jmx_encryption_options.cipher_suites).containsExactly("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384");
+ // preserved what was in yaml, overridden only what specified
+ Assert.assertEquals("test/conf/cassandra_ssl_test.truststore",
c.jmx_server_options.jmx_encryption_options.truststore);
+ }
+
+ try (WithProperties ignore = new
WithProperties().set(CONFIG_ALLOW_SYSTEM_PROPERTIES, true)
+
.with(SYSTEM_PROPERTY_PREFIX +
"jmx_server_options.jmx_encryption_options.enabled", "true"))
+ {
+ Config c = load("test/conf/cassandra-jmx-sslconfig.yaml");
+ Assert.assertTrue(c.jmx_server_options.enabled);
+ }
+ }
+
+ @Test
+ public void withEnvironmentVariables()
+ {
+ try (WithProperties ignore1 = new
WithProperties().set(CONFIG_ALLOW_ENVIRONMENT_VARIABLES, true);
+ WithEnvironment ignore2 = new
WithEnvironment(ENVIRONMENT_VARIABLE_PREFIX + "STORAGE_PORT", "123",
+
ENVIRONMENT_VARIABLE_PREFIX + "COMMITLOG_SYNC", "batch",
+
ENVIRONMENT_VARIABLE_PREFIX + "SEED_PROVIDER__class_name",
"org.apache.cassandra.locator.SimpleSeedProvider",
+
ENVIRONMENT_VARIABLE_PREFIX + "SEED_PROVIDER__parameters", "{\"seeds\":
\"127.0.0.1:7000,127.0.0.1:7001\"}",
+
ENVIRONMENT_VARIABLE_PREFIX + "CLIENT_ENCRYPTION_OPTIONS__cipher_suites",
"[\"FakeCipher\"]",
+
ENVIRONMENT_VARIABLE_PREFIX + "CLIENT_ENCRYPTION_OPTIONS__optional", "false",
+
ENVIRONMENT_VARIABLE_PREFIX + "CLIENT_ENCRYPTION_OPTIONS__enabled", "true",
+
ENVIRONMENT_VARIABLE_PREFIX + "SAI_OPTIONS",
"{\"prioritize_over_legacy_index\": \"true\", \"segment_write_buffer_size\":
\"100MiB\"}",
+
ENVIRONMENT_VARIABLE_PREFIX + "crypto_provider", "{\"class_name\": \"MyClass\",
\"parameters\": {\"fail_on_missing_provider\": \"false\"}}",
+
ENVIRONMENT_VARIABLE_PREFIX + "TABLE_PROPERTIES_WARNED",
"[\"bloom_filter_fp_chance\", \"default_time_to_live\"]",
+
ENVIRONMENT_VARIABLE_PREFIX + "PAXOS_VARIANT", "v2",
+
ENVIRONMENT_VARIABLE_PREFIX + "MEMTABLE__configurations", "{\"skiplist\":
{\"class_name\": \"SkipListMemtable\", \"parameters\": {\"skip_param1\":
\"skip_param1_value\"}}, \"trie\": {\"class_name\": \"TrieMemtable\",
\"parameters\": {\"trie_param1\": \"trie_param1_value\"}}, \"default\":
{\"inherits\": \"trie\"}}",
+
ENVIRONMENT_VARIABLE_PREFIX + "CLIENT_ERROR_REPORTING_EXCLUSIONS__subnets",
"[\"127.0.0.1\",\"127.0.0.2\"]",
+
ENVIRONMENT_VARIABLE_PREFIX + "STARTUP_CHECKS", "{\"check_data_resurrection\":
{\"enabled\": \"true\", \"heartbeat_file\":
\"/var/lib/cassandra/data/cassandra-heartbeat\"}}",
+
ENVIRONMENT_VARIABLE_PREFIX + "doesnotexist", "true"
+ ))
+ {
+ Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
+ assertThat(config.storage_port).isEqualTo(123);
+
assertThat(config.commitlog_sync).isEqualTo(Config.CommitLogSync.batch);
+
assertThat(config.seed_provider.class_name).isEqualTo("org.apache.cassandra.locator.SimpleSeedProvider");
+
assertThat(config.seed_provider.parameters.get("seeds")).isEqualTo("127.0.0.1:7000,127.0.0.1:7001");
+
assertThat(config.client_encryption_options.cipher_suites).isEqualTo(Collections.singletonList("FakeCipher"));
+ assertThat(config.client_encryption_options.optional).isFalse();
+ assertThat(config.client_encryption_options.enabled).isTrue();
+
assertThat(config.sai_options.prioritize_over_legacy_index).isTrue();
+
assertThat(config.sai_options.segment_write_buffer_size).isEqualTo(new
DataStorageSpec.IntMebibytesBound("100MiB"));
+ assertThat(config.crypto_provider.class_name).isEqualTo("MyClass");
+
assertThat(config.crypto_provider.parameters.get("fail_on_missing_provider")).isEqualTo(Boolean.FALSE.toString());
+
assertThat(config.table_properties_warned).isEqualTo(Set.of("bloom_filter_fp_chance",
"default_time_to_live"));
+ assertThat(config.paxos_variant).isEqualTo(Config.PaxosVariant.v2);
+ assertThat(config.memtable.configurations).hasSize(3);
+
assertThat(config.memtable.configurations.get("skiplist").class_name).isEqualTo("SkipListMemtable");
+
assertThat(config.memtable.configurations.get("skiplist").parameters.get("skip_param1")).isEqualTo("skip_param1_value");
+
assertThat(config.memtable.configurations.get("trie").class_name).isEqualTo("TrieMemtable");
+
assertThat(config.memtable.configurations.get("trie").parameters.get("trie_param1")).isEqualTo("trie_param1_value");
+
assertThat(config.memtable.configurations.get("default").inherits).isEqualTo("trie");
+ assertThat(config.client_error_reporting_exclusions).isEqualTo(new
SubnetGroups(Arrays.asList("127.0.0.2", "127.0.0.1")));
+ assertThat(config.startup_checks).hasSize(1);
+
assertThat(config.startup_checks.get(StartupChecks.StartupCheckType.check_data_resurrection).get("enabled")).isEqualTo("true");
+
assertThat(config.startup_checks.get(StartupChecks.StartupCheckType.check_data_resurrection).get("heartbeat_file")).isEqualTo("/var/lib/cassandra/data/cassandra-heartbeat");
+ }
+
+ try (WithEnvironment ignore = new
WithEnvironment(CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getKey(),
Boolean.TRUE.toString(),
+
ENVIRONMENT_VARIABLE_PREFIX + "memtable.configurations", "{\"skiplist\":
{\"class_name\": \"SkipListMemtable\", \"parameters\": {\"skip_param1\":
\"skip_param1_value\"}}, \"trie\": {\"class_name\": \"TrieMemtable\",
\"parameters\": {\"trie_param1\": \"trie_param1_value\"}}, \"default\":
{\"inherits\": \"trie\"}}"))
+ {
+ Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
+ assertThat(config.memtable.configurations).hasSize(3);
+
assertThat(config.memtable.configurations.get("skiplist").class_name).isEqualTo("SkipListMemtable");
+
assertThat(config.memtable.configurations.get("skiplist").parameters.get("skip_param1")).isEqualTo("skip_param1_value");
+
assertThat(config.memtable.configurations.get("trie").class_name).isEqualTo("TrieMemtable");
+
assertThat(config.memtable.configurations.get("trie").parameters.get("trie_param1")).isEqualTo("trie_param1_value");
+
assertThat(config.memtable.configurations.get("default").inherits).isEqualTo("trie");
+ }
+
+ try (WithEnvironment ignore = new
WithEnvironment(CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getKey(),
Boolean.TRUE.toString(),
+
ENVIRONMENT_VARIABLE_PREFIX + "crypto_provider.parameters",
"{\"fail_on_missing_provider\": \"false\"}",
+
ENVIRONMENT_VARIABLE_PREFIX + "crypto_provider.class_name", "MyClass"))
+ {
+ Config config =
YamlConfigurationLoader.fromMap(Collections.emptyMap(), true, Config.class);
+ ParameterizedClass cryptoProvider = config.crypto_provider;
+ Assert.assertEquals("MyClass", cryptoProvider.class_name);
+
Assert.assertTrue(cryptoProvider.parameters.containsKey("fail_on_missing_provider"));
+ String failOnMissingProviderValue =
cryptoProvider.parameters.get("fail_on_missing_provider");
+ Assert.assertNotNull(failOnMissingProviderValue);
+ Assert.assertEquals("false", failOnMissingProviderValue);
+ }
+
+ try (WithEnvironment ignore = new
WithEnvironment(CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getKey(),
Boolean.TRUE.toString(),
+
ENVIRONMENT_VARIABLE_PREFIX + "jmx_server_options.jmx_encryption_options",
+ '{' +
+ "\"enabled\":
true," +
+ "\"cipher_suites\":
[\"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\"]" +
+ '}'))
+ {
+ Config c = load("test/conf/cassandra-jmx-sslconfig.yaml");
+ Assert.assertTrue(c.jmx_server_options.enabled);
+
Assertions.assertThatCollection(c.jmx_server_options.jmx_encryption_options.cipher_suites).containsExactly("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384");
+ // preserved what was in yaml, overridden only what specified
+ Assert.assertEquals("test/conf/cassandra_ssl_test.truststore",
c.jmx_server_options.jmx_encryption_options.truststore);
+ }
+
+ try (WithEnvironment ignore = new
WithEnvironment(CassandraRelevantEnv.CASSANDRA_ALLOW_CONFIG_ENVIRONMENT_VARIABLES.getKey(),
Boolean.TRUE.toString(),
+
ENVIRONMENT_VARIABLE_PREFIX +
"jmx_server_options.jmx_encryption_options.enabled", "true"))
+ {
+ Config c = load("test/conf/cassandra-jmx-sslconfig.yaml");
+ Assert.assertTrue(c.jmx_server_options.enabled);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
