This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/3.1 by this push:
new e560e0e0c3 Add ZooKeeper Namespace Id-to-name mapping (#4996)
e560e0e0c3 is described below
commit e560e0e0c3d83bfea17e49fb7135ef9c7f99642c
Author: John K <69256191+meatballspaghe...@users.noreply.github.com>
AuthorDate: Fri Nov 15 01:01:58 2024 -0500
Add ZooKeeper Namespace Id-to-name mapping (#4996)
Add mapping for ZooKeeper Namespace Ids to Names in a single ZK node
- Implement Json-formatted mapping of namespace Ids to Names in
ZooKeeper. Stored on the "/namespaces" node. Initially created in
ZooKeeperInitializer.
- Create NamespaceMapping.java to hold methods initializing, adding,
retrieving, serializing, and deserializing namespace Id to Name map
in ZooKeeper.
- Build namespace id to name mapping of current ZooKeeper namespaces
in Upgrader11to12. Delete all current "/name" nodes.
- Add relevant test code to Upgrader11to12Test.
- Sort the mapping in serialize() for testing reliability and readability
- Remove now-unneeded Utils.checkNamespaceDoesNotExist()
This is part of #4698, but only for the namespace mapping. It does not yet
address table mappings.
---------
Co-authored-by: Christopher Tubbs <ctubb...@apache.org>
---
.../java/org/apache/accumulo/core/Constants.java | 1 -
.../accumulo/core/clientImpl/ClientContext.java | 6 +
.../accumulo/core/clientImpl/NamespaceMapping.java | 164 +++++++++++++++++++++
.../accumulo/core/clientImpl/Namespaces.java | 56 +------
.../core/clientImpl/NamespaceMappingTest.java | 72 +++++++++
.../accumulo/server/init/ZooKeeperInitializer.java | 7 +-
.../accumulo/server/tables/TableManager.java | 34 +++--
.../apache/accumulo/manager/tableOps/Utils.java | 12 --
.../create/PopulateZookeeperWithNamespace.java | 19 ++-
.../tableOps/namespace/rename/RenameNamespace.java | 23 +--
.../accumulo/manager/upgrade/Upgrader11to12.java | 29 +++-
.../manager/upgrade/Upgrader11to12Test.java | 43 +++++-
12 files changed, 353 insertions(+), 113 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java
b/core/src/main/java/org/apache/accumulo/core/Constants.java
index 411af753cf..ca9e0b62ff 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -48,7 +48,6 @@ public class Constants {
public static final String ZTABLE_NAMESPACE = "/namespace";
public static final String ZNAMESPACES = "/namespaces";
- public static final String ZNAMESPACE_NAME = "/name";
public static final String ZMANAGERS = "/managers";
public static final String ZMANAGER_LOCK = ZMANAGERS + "/lock";
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
index 899bad6c10..298a14329d 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
@@ -142,6 +142,7 @@ public class ClientContext implements AccumuloClient {
private final Supplier<SaslConnectionParams> saslSupplier;
private final Supplier<SslConnectionParams> sslSupplier;
private final Supplier<ScanServerSelector> scanServerSelectorSupplier;
+ private final NamespaceMapping namespaces;
private TCredentials rpcCreds;
private ThriftTransportPool thriftTransportPool;
private ZookeeperLockChecker zkLockChecker;
@@ -249,6 +250,7 @@ public class ClientContext implements AccumuloClient {
clientThreadPools = ThreadPools.getClientThreadPools(ueh);
}
}
+ this.namespaces = new NamespaceMapping(this);
}
public Ample getAmple() {
@@ -1114,4 +1116,8 @@ public class ClientContext implements AccumuloClient {
return this.zkLockChecker;
}
+ public NamespaceMapping getNamespaces() {
+ return namespaces;
+ }
+
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceMapping.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceMapping.java
new file mode 100644
index 0000000000..7a6ce7ab6b
--- /dev/null
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceMapping.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.clientImpl;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptySortedMap;
+import static java.util.Objects.requireNonNull;
+import static org.apache.accumulo.core.util.LazySingletons.GSON;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
+import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
+import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
+import org.apache.zookeeper.KeeperException;
+
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.gson.reflect.TypeToken;
+
+public class NamespaceMapping {
+ // type token must represent a mutable type, so it can be altered in the
mutateExisting methods
+ // without needing to make a copy
+ private static final Type MAP_TYPE = new TypeToken<TreeMap<String,String>>()
{}.getType();
+ private final ClientContext context;
+ private volatile SortedMap<NamespaceId,String> currentNamespaceMap =
emptySortedMap();
+ private volatile SortedMap<String,NamespaceId> currentNamespaceReverseMap =
emptySortedMap();
+ private volatile long lastMzxid;
+
+ public NamespaceMapping(ClientContext context) {
+ this.context = context;
+ }
+
+ public static void put(ZooReaderWriter zoo, String zPath, NamespaceId
namespaceId,
+ String namespaceName)
+ throws InterruptedException, KeeperException,
AcceptableThriftTableOperationException {
+ if (Namespace.DEFAULT.id().equals(namespaceId) ||
Namespace.ACCUMULO.id().equals(namespaceId)) {
+ throw new AssertionError(
+ "Putting built-in namespaces in map should not be possible after
init");
+ }
+ zoo.mutateExisting(zPath, data -> {
+ var namespaces = deserialize(data);
+ if (namespaces.containsKey(namespaceId.canonical())) {
+ throw new AcceptableThriftTableOperationException(null,
namespaceId.canonical(),
+ TableOperation.CREATE,
TableOperationExceptionType.NAMESPACE_EXISTS,
+ "Namespace Id already exists");
+ }
+ namespaces.put(namespaceId.canonical(), namespaceName);
+ return serialize(namespaces);
+ });
+ }
+
+ public static void remove(ZooReaderWriter zoo, String zPath, NamespaceId
namespaceId)
+ throws InterruptedException, KeeperException,
AcceptableThriftTableOperationException {
+ if (Namespace.DEFAULT.id().equals(namespaceId) ||
Namespace.ACCUMULO.id().equals(namespaceId)) {
+ throw new AssertionError("Removing built-in namespaces in map should not
be possible");
+ }
+ zoo.mutateExisting(zPath, data -> {
+ var namespaces = deserialize(data);
+ if (!namespaces.containsKey(namespaceId.canonical())) {
+ throw new AcceptableThriftTableOperationException(null,
namespaceId.canonical(),
+ TableOperation.DELETE,
TableOperationExceptionType.NAMESPACE_NOTFOUND,
+ "Namespace already removed while processing");
+ }
+ namespaces.remove(namespaceId.canonical());
+ return serialize(namespaces);
+ });
+ }
+
+ public static void rename(ZooReaderWriter zoo, String zPath, NamespaceId
namespaceId,
+ String oldName, String newName)
+ throws InterruptedException, KeeperException,
AcceptableThriftTableOperationException {
+ if (Namespace.DEFAULT.id().equals(namespaceId) ||
Namespace.ACCUMULO.id().equals(namespaceId)) {
+ throw new AssertionError("Renaming built-in namespaces in map should not
be possible");
+ }
+ zoo.mutateExisting(zPath, current -> {
+ var currentNamespaceMap = deserialize(current);
+ final String currentName =
currentNamespaceMap.get(namespaceId.canonical());
+ if (currentName.equals(newName)) {
+ return null; // assume in this case the operation is running again, so
we are done
+ }
+ if (!currentName.equals(oldName)) {
+ throw new AcceptableThriftTableOperationException(null, oldName,
TableOperation.RENAME,
+ TableOperationExceptionType.NAMESPACE_NOTFOUND, "Name changed
while processing");
+ }
+ if (currentNamespaceMap.containsValue(newName)) {
+ throw new AcceptableThriftTableOperationException(null, newName,
TableOperation.RENAME,
+ TableOperationExceptionType.NAMESPACE_EXISTS, "Namespace name
already exists");
+ }
+ currentNamespaceMap.put(namespaceId.canonical(), newName);
+ return serialize(currentNamespaceMap);
+ });
+ }
+
+ public static byte[] serialize(Map<String,String> map) {
+ return GSON.get().toJson(new TreeMap<>(map), MAP_TYPE).getBytes(UTF_8);
+ }
+
+ public static Map<String,String> deserialize(byte[] data) {
+ requireNonNull(data, "/namespaces node should not be null");
+ return GSON.get().fromJson(new String(data, UTF_8), MAP_TYPE);
+ }
+
+ private synchronized void update() {
+ final ZooCache zc = context.getZooCache();
+ final String zPath = context.getZooKeeperRoot() + Constants.ZNAMESPACES;
+ final ZooCache.ZcStat stat = new ZooCache.ZcStat();
+
+ byte[] data = zc.get(zPath, stat);
+ if (stat.getMzxid() > lastMzxid) {
+ if (data == null) {
+ throw new IllegalStateException("namespaces node should not be null");
+ } else {
+ Map<String,String> idToName = deserialize(data);
+ if (!idToName.containsKey(Namespace.DEFAULT.id().canonical())
+ || !idToName.containsKey(Namespace.ACCUMULO.id().canonical())) {
+ throw new IllegalStateException("Built-in namespace is not present
in map");
+ }
+ var converted = ImmutableSortedMap.<NamespaceId,String>naturalOrder();
+ var convertedReverse =
ImmutableSortedMap.<String,NamespaceId>naturalOrder();
+ idToName.forEach((idString, name) -> {
+ var id = NamespaceId.of(idString);
+ converted.put(id, name);
+ convertedReverse.put(name, id);
+ });
+ currentNamespaceMap = converted.build();
+ currentNamespaceReverseMap = convertedReverse.build();
+ }
+ lastMzxid = stat.getMzxid();
+ }
+ }
+
+ public SortedMap<NamespaceId,String> getIdToNameMap() {
+ update();
+ return currentNamespaceMap;
+ }
+
+ public SortedMap<String,NamespaceId> getNameToIdMap() {
+ update();
+ return currentNamespaceReverseMap;
+ }
+
+}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/Namespaces.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/Namespaces.java
index 4368b31896..366375f82f 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/Namespaces.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/Namespaces.java
@@ -18,21 +18,14 @@
*/
package org.apache.accumulo.core.clientImpl;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.function.BiConsumer;
-import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.NamespaceNotFoundException;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.util.tables.TableNameUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,9 +34,7 @@ public class Namespaces {
private static final Logger log = LoggerFactory.getLogger(Namespaces.class);
public static boolean exists(ClientContext context, NamespaceId namespaceId)
{
- ZooCache zc = context.getZooCache();
- List<String> namespaceIds = zc.getChildren(context.getZooKeeperRoot() +
Constants.ZNAMESPACES);
- return namespaceIds.contains(namespaceId.canonical());
+ return context.getNamespaces().getIdToNameMap().containsKey(namespaceId);
}
public static List<TableId> getTableIds(ClientContext context, NamespaceId
namespaceId)
@@ -70,39 +61,18 @@ public class Namespaces {
return names;
}
- /**
- * Gets all the namespaces from ZK. The first arg (t) the BiConsumer accepts
is the ID and the
- * second (u) is the namespaceName.
- */
- private static void getAllNamespaces(ClientContext context,
- BiConsumer<String,String> biConsumer) {
- final ZooCache zc = context.getZooCache();
- List<String> namespaceIds = zc.getChildren(context.getZooKeeperRoot() +
Constants.ZNAMESPACES);
- for (String id : namespaceIds) {
- byte[] path = zc.get(context.getZooKeeperRoot() + Constants.ZNAMESPACES
+ "/" + id
- + Constants.ZNAMESPACE_NAME);
- if (path != null) {
- biConsumer.accept(id, new String(path, UTF_8));
- }
- }
- }
-
/**
* Return sorted map with key = ID, value = namespaceName
*/
public static SortedMap<NamespaceId,String> getIdToNameMap(ClientContext
context) {
- SortedMap<NamespaceId,String> idMap = new TreeMap<>();
- getAllNamespaces(context, (id, name) -> idMap.put(NamespaceId.of(id),
name));
- return idMap;
+ return context.getNamespaces().getIdToNameMap();
}
/**
* Return sorted map with key = namespaceName, value = ID
*/
public static SortedMap<String,NamespaceId> getNameToIdMap(ClientContext
context) {
- SortedMap<String,NamespaceId> nameMap = new TreeMap<>();
- getAllNamespaces(context, (id, name) -> nameMap.put(name,
NamespaceId.of(id)));
- return nameMap;
+ return context.getNamespaces().getNameToIdMap();
}
/**
@@ -110,17 +80,12 @@ public class Namespaces {
*/
public static NamespaceId getNamespaceId(ClientContext context, String
namespaceName)
throws NamespaceNotFoundException {
- final ArrayList<NamespaceId> singleId = new ArrayList<>(1);
- getAllNamespaces(context, (id, name) -> {
- if (name.equals(namespaceName)) {
- singleId.add(NamespaceId.of(id));
- }
- });
- if (singleId.isEmpty()) {
+ var id = context.getNamespaces().getNameToIdMap().get(namespaceName);
+ if (id == null) {
throw new NamespaceNotFoundException(null, namespaceName,
"getNamespaceId() failed to find namespace");
}
- return singleId.get(0);
+ return id;
}
/**
@@ -150,13 +115,8 @@ public class Namespaces {
*/
public static String getNamespaceName(ClientContext context, NamespaceId
namespaceId)
throws NamespaceNotFoundException {
- String name;
- ZooCache zc = context.getZooCache();
- byte[] path = zc.get(context.getZooKeeperRoot() + Constants.ZNAMESPACES +
"/"
- + namespaceId.canonical() + Constants.ZNAMESPACE_NAME);
- if (path != null) {
- name = new String(path, UTF_8);
- } else {
+ String name = getIdToNameMap(context).get(namespaceId);
+ if (name == null) {
throw new NamespaceNotFoundException(namespaceId.canonical(), null,
"getNamespaceName() failed to find namespace");
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/clientImpl/NamespaceMappingTest.java
b/core/src/test/java/org/apache/accumulo/core/clientImpl/NamespaceMappingTest.java
new file mode 100644
index 0000000000..833f17ca4a
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/clientImpl/NamespaceMappingTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.clientImpl;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.junit.jupiter.api.Test;
+
+import com.google.gson.JsonSyntaxException;
+
+class NamespaceMappingTest {
+
+ @Test
+ void testSerialize() {
+ assertThrows(NullPointerException.class, () ->
NamespaceMapping.serialize(null));
+ assertEquals("{}", new String(NamespaceMapping.serialize(Map.of()),
UTF_8));
+ assertEquals("{\"a\":\"b\"}", new
String(NamespaceMapping.serialize(Map.of("a", "b")), UTF_8));
+ assertEquals("{\"a\":\"b\",\"c\":\"d\"}",
+ new String(NamespaceMapping.serialize(Map.of("a", "b", "c", "d")),
UTF_8));
+ }
+
+ @Test
+ void testDeserialize() {
+ assertThrows(NullPointerException.class, () ->
NamespaceMapping.deserialize(null));
+ assertEquals(null, NamespaceMapping.deserialize(new byte[0]));
+ assertTrue(NamespaceMapping.deserialize("{}".getBytes(UTF_8)) instanceof
TreeMap);
+ assertEquals(Map.of(), NamespaceMapping.deserialize("{}".getBytes(UTF_8)));
+ assertEquals(Map.of("a", "b"),
NamespaceMapping.deserialize("{\"a\":\"b\"}".getBytes(UTF_8)));
+ assertEquals(Map.of("a", "b", "c", "d"),
+
NamespaceMapping.deserialize("{\"a\":\"b\",\"c\":\"d\"}".getBytes(UTF_8)));
+
+ // check malformed json
+ assertThrows(JsonSyntaxException.class,
+ () -> NamespaceMapping.deserialize("-".getBytes(UTF_8)));
+ // check incorrect json type for string value
+ assertThrows(JsonSyntaxException.class,
+ () -> NamespaceMapping.deserialize("{\"a\":{}}".getBytes(UTF_8)));
+ // check valid json, but not a map
+ assertThrows(JsonSyntaxException.class,
+ () -> NamespaceMapping.deserialize("\"[\"a\"]\"".getBytes(UTF_8)));
+
+ // strange edge case because empty json array can be converted into an
empty map; we don't ever
+ // expect this to be found, but there's no easy way to check for this edge
case that is worth
+ // the effort
+ assertTrue(NamespaceMapping.deserialize("[]".getBytes(UTF_8)) instanceof
SortedMap);
+ assertEquals(Map.of(), NamespaceMapping.deserialize("[]".getBytes(UTF_8)));
+ }
+
+}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
index 50dfc76b23..1fc771dc09 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
@@ -21,10 +21,12 @@ package org.apache.accumulo.server.init;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
+import java.util.Map;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.clientImpl.Namespace;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -110,7 +112,10 @@ public class ZooKeeperInitializer {
String zkInstanceRoot = Constants.ZROOT + "/" + instanceId;
zoo.putPersistentData(zkInstanceRoot + Constants.ZTABLES,
Constants.ZTABLES_INITIAL_ID,
ZooUtil.NodeExistsPolicy.FAIL);
- zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES, new byte[0],
+ zoo.putPersistentData(zkInstanceRoot + Constants.ZNAMESPACES,
+ NamespaceMapping
+ .serialize(Map.of(Namespace.DEFAULT.id().canonical(),
Namespace.DEFAULT.name(),
+ Namespace.ACCUMULO.id().canonical(),
Namespace.ACCUMULO.name())),
ZooUtil.NodeExistsPolicy.FAIL);
TableManager.prepareNewNamespaceState(context, Namespace.DEFAULT.id(),
Namespace.DEFAULT.name(),
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
index 380942b8aa..15ce8d5941 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tables/TableManager.java
@@ -29,6 +29,9 @@ import java.util.Set;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.NamespaceNotFoundException;
+import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
+import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
@@ -67,14 +70,14 @@ public class TableManager {
private final ZooReaderWriter zoo;
private final ZooCache zooStateCache;
- public static void prepareNewNamespaceState(ZooReaderWriter zoo, final
PropStore propStore,
- InstanceId instanceId, NamespaceId namespaceId, String namespace,
- NodeExistsPolicy existsPolicy) throws KeeperException,
InterruptedException {
+ public static void prepareNewNamespaceState(final ServerContext context,
NamespaceId namespaceId,
+ String namespace, NodeExistsPolicy existsPolicy)
+ throws KeeperException, InterruptedException {
+ final PropStore propStore = context.getPropStore();
+ final InstanceId instanceId = context.getInstanceID();
log.debug("Creating ZooKeeper entries for new namespace {} (ID: {})",
namespace, namespaceId);
- String zPath = Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES
+ "/" + namespaceId;
-
- zoo.putPersistentData(zPath, new byte[0], existsPolicy);
- zoo.putPersistentData(zPath + Constants.ZNAMESPACE_NAME,
namespace.getBytes(UTF_8),
+ context.getZooReaderWriter().putPersistentData(
+ Constants.ZROOT + "/" + instanceId + Constants.ZNAMESPACES + "/" +
namespaceId, new byte[0],
existsPolicy);
var propKey = NamespacePropKey.of(instanceId, namespaceId);
if (!propStore.exists(propKey)) {
@@ -82,13 +85,6 @@ public class TableManager {
}
}
- public static void prepareNewNamespaceState(final ServerContext context,
NamespaceId namespaceId,
- String namespace, NodeExistsPolicy existsPolicy)
- throws KeeperException, InterruptedException {
- prepareNewNamespaceState(context.getZooReaderWriter(),
context.getPropStore(),
- context.getInstanceID(), namespaceId, namespace, existsPolicy);
- }
-
public static void prepareNewTableState(ZooReaderWriter zoo, PropStore
propStore,
InstanceId instanceId, TableId tableId, NamespaceId namespaceId, String
tableName,
TableState state, NodeExistsPolicy existsPolicy)
@@ -329,7 +325,15 @@ public class TableManager {
}
public void removeNamespace(NamespaceId namespaceId)
- throws KeeperException, InterruptedException {
+ throws KeeperException, InterruptedException,
AcceptableThriftTableOperationException {
+ try {
+ NamespaceMapping.remove(zoo, zkRoot + Constants.ZNAMESPACES,
namespaceId);
+ } catch (AcceptableThriftTableOperationException e) {
+ // ignore not found, because that's what we're trying to do anyway
+ if (e.getType() != TableOperationExceptionType.NAMESPACE_NOTFOUND) {
+ throw e;
+ }
+ }
zoo.recursiveDelete(zkRoot + Constants.ZNAMESPACES + "/" + namespaceId,
NodeMissingPolicy.SKIP);
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/Utils.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/Utils.java
index be051943a3..8fc09eac2c 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/Utils.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/Utils.java
@@ -258,18 +258,6 @@ public class Utils {
return Utils.getLock(env.getContext(), id, tid, LockType.READ);
}
- public static void checkNamespaceDoesNotExist(ServerContext context, String
namespace,
- NamespaceId namespaceId, TableOperation operation)
- throws AcceptableThriftTableOperationException {
-
- NamespaceId n = Namespaces.lookupNamespaceId(context, namespace);
-
- if (n != null && !n.equals(namespaceId)) {
- throw new AcceptableThriftTableOperationException(null, namespace,
operation,
- TableOperationExceptionType.NAMESPACE_EXISTS, null);
- }
- }
-
/**
* Given a fully-qualified Path and a flag indicating if the file info is
base64 encoded or not,
* retrieve the data from a file on the file system. It is assumed that the
file is textual and
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java
index 939b6f0916..b47223f151 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/create/PopulateZookeeperWithNamespace.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.manager.tableOps.namespace.create;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
import org.apache.accumulo.core.fate.Repo;
import
org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType;
@@ -50,17 +52,17 @@ class PopulateZookeeperWithNamespace extends ManagerRepo {
Utils.getTableNameLock().lock();
try {
- Utils.checkNamespaceDoesNotExist(manager.getContext(),
namespaceInfo.namespaceName,
- namespaceInfo.namespaceId, TableOperation.CREATE);
-
- TableManager.prepareNewNamespaceState(manager.getContext(),
namespaceInfo.namespaceId,
+ var context = manager.getContext();
+ NamespaceMapping.put(context.getZooReaderWriter(),
+ Constants.ZROOT + "/" + context.getInstanceID() +
Constants.ZNAMESPACES,
+ namespaceInfo.namespaceId, namespaceInfo.namespaceName);
+ TableManager.prepareNewNamespaceState(context, namespaceInfo.namespaceId,
namespaceInfo.namespaceName, NodeExistsPolicy.OVERWRITE);
- PropUtil.setProperties(manager.getContext(),
- NamespacePropKey.of(manager.getContext(), namespaceInfo.namespaceId),
+ PropUtil.setProperties(context, NamespacePropKey.of(context,
namespaceInfo.namespaceId),
namespaceInfo.props);
- manager.getContext().clearTableListCache();
+ context.clearTableListCache();
return new FinishCreateNamespace(namespaceInfo);
} finally {
@@ -70,8 +72,9 @@ class PopulateZookeeperWithNamespace extends ManagerRepo {
@Override
public void undo(long tid, Manager manager) throws Exception {
+ var context = manager.getContext();
manager.getTableManager().removeNamespace(namespaceInfo.namespaceId);
- manager.getContext().clearTableListCache();
+ context.clearTableListCache();
Utils.unreserveNamespace(manager, namespaceInfo.namespaceId, tid,
LockType.WRITE);
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java
index 3fdddf0758..b1fcb9f30b 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/namespace/rename/RenameNamespace.java
@@ -18,12 +18,9 @@
*/
package org.apache.accumulo.manager.tableOps.namespace.rename;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
import org.apache.accumulo.core.Constants;
-import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
-import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.fate.Repo;
import
org.apache.accumulo.core.fate.zookeeper.DistributedReadWriteLock.LockType;
@@ -59,23 +56,9 @@ public class RenameNamespace extends ManagerRepo {
Utils.getTableNameLock().lock();
try {
- Utils.checkNamespaceDoesNotExist(manager.getContext(), newName,
namespaceId,
- TableOperation.RENAME);
-
- final String tap = manager.getZooKeeperRoot() + Constants.ZNAMESPACES +
"/" + namespaceId
- + Constants.ZNAMESPACE_NAME;
+ NamespaceMapping.rename(zoo, manager.getZooKeeperRoot() +
Constants.ZNAMESPACES, namespaceId,
+ oldName, newName);
- zoo.mutateExisting(tap, current -> {
- final String currentName = new String(current, UTF_8);
- if (currentName.equals(newName)) {
- return null; // assume in this case the operation is running again,
so we are done
- }
- if (!currentName.equals(oldName)) {
- throw new AcceptableThriftTableOperationException(null, oldName,
TableOperation.RENAME,
- TableOperationExceptionType.NAMESPACE_NOTFOUND, "Name changed
while processing");
- }
- return newName.getBytes(UTF_8);
- });
manager.getContext().clearTableListCache();
} finally {
Utils.getTableNameLock().unlock();
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
index a130d11445..60d80aaa44 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java
@@ -26,17 +26,20 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IsolatedScanner;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -81,7 +84,10 @@ public class Upgrader11to12 implements Upgrader {
static final Set<Text> UPGRADE_FAMILIES =
Set.of(DataFileColumnFamily.NAME, CHOPPED,
ExternalCompactionColumnFamily.NAME);
- public static final String ZTRACERS = "/tracers";
+ private static final String ZTRACERS = "/tracers";
+
+ @VisibleForTesting
+ static final String ZNAMESPACE_NAME = "/name";
@Override
public void upgradeZookeeper(@NonNull ServerContext context) {
@@ -117,6 +123,27 @@ public class Upgrader11to12 implements Upgrader {
zrw.overwritePersistentData(rootBase, rtm.toJson().getBytes(UTF_8),
stat.getVersion());
log.info("Root metadata in ZooKeeper after upgrade: {}", rtm.toJson());
}
+
+ String zPath = Constants.ZROOT + "/" + context.getInstanceID() +
Constants.ZNAMESPACES;
+ byte[] namespacesData = zrw.getData(zPath);
+ if (namespacesData.length != 0) {
+ throw new IllegalStateException(
+ "Unexpected data found under namespaces node: " + new
String(namespacesData, UTF_8));
+ }
+ List<String> namespaceIdList = zrw.getChildren(zPath);
+ Map<String,String> namespaceMap = new HashMap<>();
+ for (String namespaceId : namespaceIdList) {
+ String namespaceNamePath = zPath + "/" + namespaceId + ZNAMESPACE_NAME;
+ namespaceMap.put(namespaceId, new
String(zrw.getData(namespaceNamePath), UTF_8));
+ }
+ byte[] mapping = NamespaceMapping.serialize(namespaceMap);
+ zrw.putPersistentData(zPath, mapping,
ZooUtil.NodeExistsPolicy.OVERWRITE);
+
+ for (String namespaceId : namespaceIdList) {
+ String namespaceNamePath = zPath + "/" + namespaceId + ZNAMESPACE_NAME;
+ zrw.delete(namespaceNamePath);
+ }
+
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException(
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
index d176a4c56c..bb21b4efec 100644
---
a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
@@ -20,8 +20,11 @@ package org.apache.accumulo.manager.upgrade;
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.accumulo.manager.upgrade.Upgrader11to12.UPGRADE_FAMILIES;
+import static
org.apache.accumulo.manager.upgrade.Upgrader11to12.ZNAMESPACE_NAME;
+import static org.easymock.EasyMock.aryEq;
import static org.easymock.EasyMock.capture;
import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createStrictMock;
import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
@@ -40,10 +43,13 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.clientImpl.NamespaceMapping;
import org.apache.accumulo.core.data.ColumnUpdate;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.Key;
@@ -346,14 +352,18 @@ public class Upgrader11to12Test {
Upgrader11to12 upgrader = new Upgrader11to12();
ServerContext context = createMock(ServerContext.class);
- ZooReaderWriter zrw = createMock(ZooReaderWriter.class);
+ ZooReaderWriter zrw = createStrictMock(ZooReaderWriter.class);
expect(context.getInstanceID()).andReturn(iid).anyTimes();
expect(context.getZooReaderWriter()).andReturn(zrw).anyTimes();
+ zrw.recursiveDelete(Constants.ZROOT + "/" + iid.canonical() + "/tracers",
+ ZooUtil.NodeMissingPolicy.SKIP);
+ expectLastCall().once();
+
Capture<Stat> statCapture = newCapture();
- expect(zrw.getData(eq("/accumulo/" + iid.canonical() + "/root_tablet"),
capture(statCapture)))
- .andAnswer(() -> {
+ expect(zrw.getData(eq(Constants.ZROOT + "/" + iid.canonical() +
"/root_tablet"),
+ capture(statCapture))).andAnswer(() -> {
Stat stat = statCapture.getValue();
stat.setCtime(System.currentTimeMillis());
stat.setMtime(System.currentTimeMillis());
@@ -364,12 +374,31 @@ public class Upgrader11to12Test {
}).once();
Capture<byte[]> byteCapture = newCapture();
- expect(zrw.overwritePersistentData(eq("/accumulo/" + iid.canonical() +
"/root_tablet"),
+ expect(zrw.overwritePersistentData(eq(Constants.ZROOT + "/" +
iid.canonical() + "/root_tablet"),
capture(byteCapture), eq(123))).andReturn(true).once();
- zrw.recursiveDelete("/accumulo/" + iid.canonical() + "/tracers",
- ZooUtil.NodeMissingPolicy.SKIP);
- expectLastCall().once();
+ expect(zrw.getData(eq(Constants.ZROOT + "/" + iid.canonical() +
Constants.ZNAMESPACES)))
+ .andReturn(new byte[0]).once();
+ Map<String,String> mockNamespaces = Map.of("ns1", "ns1name", "ns2",
"ns2name");
+ expect(zrw.getChildren(eq(Constants.ZROOT + "/" + iid.canonical() +
Constants.ZNAMESPACES)))
+ .andReturn(List.copyOf(mockNamespaces.keySet())).once();
+ for (String ns : mockNamespaces.keySet()) {
+ Supplier<String> pathMatcher = () -> eq(Constants.ZROOT + "/" +
iid.canonical()
+ + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME);
+
expect(zrw.getData(pathMatcher.get())).andReturn(mockNamespaces.get(ns).getBytes(UTF_8))
+ .once();
+ }
+ byte[] mapping = NamespaceMapping.serialize(mockNamespaces);
+ expect(
+ zrw.putPersistentData(eq(Constants.ZROOT + "/" + iid.canonical() +
Constants.ZNAMESPACES),
+ aryEq(mapping), eq(ZooUtil.NodeExistsPolicy.OVERWRITE)))
+ .andReturn(true).once();
+ for (String ns : mockNamespaces.keySet()) {
+ Supplier<String> pathMatcher = () -> eq(Constants.ZROOT + "/" +
iid.canonical()
+ + Constants.ZNAMESPACES + "/" + ns + ZNAMESPACE_NAME);
+ zrw.delete(pathMatcher.get());
+ expectLastCall().once();
+ }
replay(context, zrw);
