morningman commented on code in PR #52636:
URL: https://github.com/apache/doris/pull/52636#discussion_r2188899434
##########
fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java:
##########
@@ -30,18 +31,90 @@
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
/**
* This class is used to convert sql with different dialects using sql
convertor service.
* The sql convertor service is a http service which is used to convert sql.
+ * <p>
+ * Features:
+ * - Support multiple URLs (comma separated)
+ * - Blacklist mechanism for failed URLs
+ * - Automatic failover and retry
+ * - URL caching and smart selection
*/
public class HttpDialectUtils {
private static final Logger LOG =
LogManager.getLogger(HttpDialectUtils.class);
- public static String convertSql(String targetURL, String originStmt,
String dialect,
+ // Cache URL manager instances to avoid duplicate parsing
+ private static final ConcurrentHashMap<String, UrlManager> urlManagerCache
= new ConcurrentHashMap<>();
+
+ // Blacklist recovery time (ms): 1 minute
+ private static final long BLACKLIST_RECOVERY_TIME_MS = 60 * 1000;
+ // Connection timeout period (ms): 3 seconds
+ private static final int CONNECTION_TIMEOUT_MS = 3000;
+ // Read timeout period (ms): 10 seconds
+ private static final int READ_TIMEOUT_MS = 10000;
+
+ public static String convertSql(String targetURLs, String originStmt,
String dialect,
String[] features, String config) {
+ UrlManager urlManager = getOrCreateUrlManager(targetURLs);
ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect, features, config);
+ String requestStr = convertRequest.toJson();
+
+ // Try to convert SQL using intelligent URL selection strategy
+ return tryConvertWithIntelligentSelection(urlManager, requestStr,
originStmt);
+ }
+
+ /**
+ * Try to convert SQL using intelligent URL selection strategy
+ * CRITICAL: This method ensures 100% success rate when ANY service is
available
+ */
+ private static String tryConvertWithIntelligentSelection(
+ UrlManager urlManager, String requestStr, String originStmt) {
+ // Strategy: Try ALL URLs in intelligent order, regardless of
blacklist status
+ // This ensures 100% success rate when any service is actually
available
+ List<String> allUrls = urlManager.getAllUrlsInPriorityOrder();
+
+ for (String url : allUrls) {
+ try {
+ String result = doConvertSql(url, requestStr);
+ if (result != null && !result.equals(originStmt)) {
Review Comment:
Why need this condition? `!result.equals(originStmt)`
##########
fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java:
##########
@@ -30,18 +31,90 @@
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
/**
* This class is used to convert sql with different dialects using sql
convertor service.
* The sql convertor service is a http service which is used to convert sql.
+ * <p>
+ * Features:
+ * - Support multiple URLs (comma separated)
+ * - Blacklist mechanism for failed URLs
+ * - Automatic failover and retry
+ * - URL caching and smart selection
*/
public class HttpDialectUtils {
private static final Logger LOG =
LogManager.getLogger(HttpDialectUtils.class);
- public static String convertSql(String targetURL, String originStmt,
String dialect,
+ // Cache URL manager instances to avoid duplicate parsing
+ private static final ConcurrentHashMap<String, UrlManager> urlManagerCache
= new ConcurrentHashMap<>();
Review Comment:
We need to consider how to remove the unused entries.
So I suggest to use a Caffein Cache, and set a expire time so that it can
remote unused entries automatically.
##########
fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java:
##########
@@ -30,18 +31,90 @@
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
/**
* This class is used to convert sql with different dialects using sql
convertor service.
* The sql convertor service is a http service which is used to convert sql.
+ * <p>
+ * Features:
+ * - Support multiple URLs (comma separated)
+ * - Blacklist mechanism for failed URLs
+ * - Automatic failover and retry
+ * - URL caching and smart selection
*/
public class HttpDialectUtils {
private static final Logger LOG =
LogManager.getLogger(HttpDialectUtils.class);
- public static String convertSql(String targetURL, String originStmt,
String dialect,
+ // Cache URL manager instances to avoid duplicate parsing
+ private static final ConcurrentHashMap<String, UrlManager> urlManagerCache
= new ConcurrentHashMap<>();
+
+ // Blacklist recovery time (ms): 1 minute
+ private static final long BLACKLIST_RECOVERY_TIME_MS = 60 * 1000;
+ // Connection timeout period (ms): 3 seconds
+ private static final int CONNECTION_TIMEOUT_MS = 3000;
+ // Read timeout period (ms): 10 seconds
+ private static final int READ_TIMEOUT_MS = 10000;
+
+ public static String convertSql(String targetURLs, String originStmt,
String dialect,
String[] features, String config) {
+ UrlManager urlManager = getOrCreateUrlManager(targetURLs);
ConvertRequest convertRequest = new ConvertRequest(originStmt,
dialect, features, config);
+ String requestStr = convertRequest.toJson();
+
+ // Try to convert SQL using intelligent URL selection strategy
+ return tryConvertWithIntelligentSelection(urlManager, requestStr,
originStmt);
+ }
+
+ /**
+ * Try to convert SQL using intelligent URL selection strategy
+ * CRITICAL: This method ensures 100% success rate when ANY service is
available
+ */
+ private static String tryConvertWithIntelligentSelection(
+ UrlManager urlManager, String requestStr, String originStmt) {
+ // Strategy: Try ALL URLs in intelligent order, regardless of
blacklist status
+ // This ensures 100% success rate when any service is actually
available
+ List<String> allUrls = urlManager.getAllUrlsInPriorityOrder();
+
+ for (String url : allUrls) {
+ try {
+ String result = doConvertSql(url, requestStr);
+ if (result != null && !result.equals(originStmt)) {
+ // Conversion succeeded, mark URL as healthy (remove from
blacklist)
+ urlManager.markUrlAsHealthy(url);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Successfully converted SQL using URL: {}",
url);
+ }
+ return result;
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to convert SQL using URL: {}, error: {}",
url, e.getMessage());
+ // Add failed URL to blacklist for future optimization
+ urlManager.markUrlAsBlacklisted(url);
+ // Continue trying next URL - this is CRITICAL for 100%
success rate
+ }
+ }
+
+ LOG.warn("All URLs failed to convert SQL, return original SQL");
Review Comment:
remove this log, no useful info
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
