Nataneljpwd commented on code in PR #64391:
URL: https://github.com/apache/airflow/pull/64391#discussion_r3006537806
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -545,9 +551,41 @@ def _resolve_kerberos_principal(self, principal: str |
None) -> str:
func = kerberos.get_kerberos_principal
except AttributeError:
# Fallback for older versions of Airflow
- func = kerberos.get_kerberos_principle # type:
ignore[attr-defined]
+ func = getattr(kerberos, "get_kerberos_principle")
Review Comment:
Why was this changed? To remove the type ignore?
##########
providers/apache/spark/src/airflow/providers/apache/spark/hooks/spark_submit.py:
##########
@@ -545,9 +551,41 @@ def _resolve_kerberos_principal(self, principal: str |
None) -> str:
func = kerberos.get_kerberos_principal
except AttributeError:
# Fallback for older versions of Airflow
- func = kerberos.get_kerberos_principle # type:
ignore[attr-defined]
+ func = getattr(kerberos, "get_kerberos_principle")
return func(principal)
+ def _run_post_submit_commands(self) -> None:
+ """
+ Run any post-submit shell commands configured on this hook.
+
+ Called after the Spark job finishes (success or on_kill). Typical use
case
+ is killing sidecars like Istio that don't shut down automatically.
+ Failures are logged as warnings and never raise.
+ """
+ for cmd in self._post_submit_commands:
+ self.log.info("Running post-submit command: %s", cmd)
+ try:
+ result = subprocess.run(
+ cmd,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ check=False,
+ timeout=30,
+ )
Review Comment:
This executes on the airflow worker, is this what we want?
As the example references localhost, and it will not run on the spark pod
Or is the problem with the spark itself?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
