xy720 commented on a change in pull request #4163:
URL: https://github.com/apache/incubator-doris/pull/4163#discussion_r460531801
##########
File path:
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
##########
@@ -92,12 +90,41 @@ public void submitEtlJob(long loadJobId, String loadLabel,
EtlJobConfig etlJobCo
// delete outputPath
deleteEtlOutputPath(etlJobConfig.outputPath, brokerDesc);
- // upload app resource and jobconfig to hdfs
+ // prepare dpp archive
+ SparkRepository.SparkArchive archive = resource.prepareArchive();
+ Preconditions.checkNotNull(archive);
+ List<SparkRepository.SparkLibrary> libraries = archive.libraries;
+ Optional<SparkRepository.SparkLibrary> dppLibrary = libraries.stream().
+ filter(library -> library.libType ==
SparkRepository.SparkLibrary.LibType.DPP).findFirst();
+ Optional<SparkRepository.SparkLibrary> spark2xLibrary =
libraries.stream().
+ filter(library -> library.libType ==
SparkRepository.SparkLibrary.LibType.SPARK2X).findFirst();
+ if (!dppLibrary.isPresent() || !spark2xLibrary.isPresent()) {
+ throw new LoadException("failed to get library from remote
archive");
+ }
+
+ // spark home
+ String sparkHome = Config.spark_home_default_dir;
+ // etl config path
String configsHdfsDir = etlJobConfig.outputPath + "/" + JOB_CONFIG_DIR
+ "/";
- String appResourceHdfsPath = configsHdfsDir + APP_RESOURCE_NAME;
+ // etl config json path
String jobConfigHdfsPath = configsHdfsDir + CONFIG_FILE_NAME;
+ // spark submit app resource path
+ String appResourceHdfsPath = dppLibrary.get().remotePath;
+ // spark yarn archive path
+ String jobArchiveHdfsPath = spark2xLibrary.get().remotePath;
+ // spark yarn stage dir
+ String jobStageHdfsPath = resource.getWorkingDir();
+
+ // update archive and stage configs here
+ Map<String, String> sparkConfigs = resource.getSparkConfigs();
+ if (Strings.isNullOrEmpty(sparkConfigs.get("spark.yarn.archive"))) {
Review comment:
If user has set `spark.yarn.archive`in resource, we prefer to use the
archive set by user, otherwise we use archive generated by SparkRepository.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
