[
https://issues.apache.org/jira/browse/HADOOP-19472?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18033232#comment-18033232
]
ASF GitHub Bot commented on HADOOP-19472:
-----------------------------------------
anujmodi2021 commented on code in PR #7669:
URL: https://github.com/apache/hadoop/pull/7669#discussion_r2465584624
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java:
##########
@@ -262,6 +281,48 @@ public final class FileSystemConfigurations {
public static final int DEFAULT_FS_AZURE_BLOB_DELETE_THREAD =
DEFAULT_FS_AZURE_LISTING_ACTION_THREADS;
+ public static final boolean DEFAULT_WRITE_DYNAMIC_THREADPOOL_ENABLEMENT =
true;
Review Comment:
By default should be disabled
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestWriteThreadPoolSizeManager.java:
##########
@@ -0,0 +1,770 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerArray;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static
org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.ZERO;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class TestWriteThreadPoolSizeManager extends AbstractAbfsIntegrationTest {
+
+ private AbfsConfiguration mockConfig;
+ private static final double HIGH_CPU_UTILIZATION_THRESHOLD = 0.95;
+ private static final double LOW_CPU_UTILIZATION_THRESHOLD = 0.05;
+ private static final int THREAD_SLEEP_DURATION_MS = 200;
+ private static final String TEST_FILE_PATH = "testFilePath";
+ private static final String TEST_DIR_PATH = "testDirPath";
+ private static final int TEST_FILE_LENGTH = 1024 * 1024 * 8;
+ private static final int CONCURRENT_REQUEST_COUNT = 15;
+ private static final int THREAD_POOL_KEEP_ALIVE_TIME = 10;
+ private static final int LOW_TIER_MEMORY_MULTIPLIER = 4;
+ private static final int MEDIUM_TIER_MEMORY_MULTIPLIER = 6;
+ private static final int HIGH_TIER_MEMORY_MULTIPLIER = 8;
+ private static final int HIGH_CPU_THRESHOLD = 15;
+ private static final int MEDIUM_CPU_THRESHOLD = 10;
+ private static final int LOW_CPU_THRESHOLD = 5;
+ private static final int CPU_MONITORING_INTERVAL = 15;
+ private static final int WAIT_DURATION_MS = 3000;
+ private static final int LATCH_TIMEOUT_SECONDS = 60;
+ private static final int RESIZE_WAIT_TIME_MS = 6_000;
+ private static final double HIGH_CPU_USAGE_RATIO = 0.95;
+ private static final double LOW_CPU_USAGE_RATIO = 0.05;
+ private static final int SLEEP_DURATION_MS = 150;
+ private static final int AWAIT_TIMEOUT_SECONDS = 45;
+ private static final int RESIZER_JOIN_TIMEOUT_MS = 2_000;
+ private static final int WAIT_TIMEOUT_MS = 5000;
+ private static final int SLEEP_DURATION_30S_MS = 30000;
+ private static final int SMALL_PAUSE_MS = 50;
+ private static final int BURST_LOAD = 50;
+ private static final long LOAD_SLEEP_DURATION_MS = 2000;
+
+ TestWriteThreadPoolSizeManager() throws Exception {
+ super.setup();
+ }
+
+ /**
+ * Common setup to prepare a mock configuration for each test.
+ */
+ @BeforeEach
+ public void setUp() {
+ mockConfig = mock(AbfsConfiguration.class);
+
when(mockConfig.getWriteConcurrentRequestCount()).thenReturn(CONCURRENT_REQUEST_COUNT);
+
when(mockConfig.getWriteThreadPoolKeepAliveTime()).thenReturn(THREAD_POOL_KEEP_ALIVE_TIME);
+
when(mockConfig.getLowTierMemoryMultiplier()).thenReturn(LOW_TIER_MEMORY_MULTIPLIER);
+
when(mockConfig.getMediumTierMemoryMultiplier()).thenReturn(MEDIUM_TIER_MEMORY_MULTIPLIER);
+
when(mockConfig.getHighTierMemoryMultiplier()).thenReturn(HIGH_TIER_MEMORY_MULTIPLIER);
+ when(mockConfig.getWriteHighCpuThreshold()).thenReturn(HIGH_CPU_THRESHOLD);
+
when(mockConfig.getWriteMediumCpuThreshold()).thenReturn(MEDIUM_CPU_THRESHOLD);
+ when(mockConfig.getWriteLowCpuThreshold()).thenReturn(LOW_CPU_THRESHOLD);
+
when(mockConfig.getWriteCpuMonitoringInterval()).thenReturn(CPU_MONITORING_INTERVAL);
+ }
+
+ /**
+ * Ensures that {@link WriteThreadPoolSizeManager#getInstance(String,
AbfsConfiguration)} returns a singleton per key.
+ */
+ @Test
+ void testGetInstanceReturnsSingleton() {
Review Comment:
Its no more a singleton right?
May be we don't need this test anymore.
> ABFS: Enhance performance of ABFS driver for write-heavy workloads
> ------------------------------------------------------------------
>
> Key: HADOOP-19472
> URL: https://issues.apache.org/jira/browse/HADOOP-19472
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/azure
> Affects Versions: 3.4.1
> Reporter: Anmol Asrani
> Assignee: Anmol Asrani
> Priority: Minor
> Labels: pull-request-available
> Fix For: 3.4.1
>
>
> The goal of this work item is to enhance the performance of ABFS Driver for
> write-heavy workloads by improving concurrency within writes.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
