dheerajturaga commented on code in PR #64326:
URL: https://github.com/apache/airflow/pull/64326#discussion_r3061012692
##########
airflow-core/src/airflow/models/dagbag.py:
##########
@@ -44,14 +46,24 @@ class DBDagBag:
:meta private:
"""
- def __init__(self, load_op_links: bool = True) -> None:
- self._dags: dict[UUID, SerializedDagModel] = {} # dag_version_id to
dag
+ def __init__(self, load_op_links: bool = True, max_cache_size: int | None
= None) -> None:
+ self._max_dag_version_cache_size = max_cache_size # None = unbounded
+ self._dags: OrderedDict[UUID, SerializedDagModel] = OrderedDict()
self.load_op_links = load_op_links
def _read_dag(self, serialized_dag_model: SerializedDagModel) ->
SerializedDAG | None:
serialized_dag_model.load_op_links = self.load_op_links
if dag := serialized_dag_model.dag:
- self._dags[serialized_dag_model.dag_version_id] =
serialized_dag_model
+ version_id = serialized_dag_model.dag_version_id
+ self._dags[version_id] = serialized_dag_model
+ self._dags.move_to_end(version_id)
+ if (
+ self._max_dag_version_cache_size is not None
+ and len(self._dags) > self._max_dag_version_cache_size
+ ):
+ self._dags.popitem(last=False)
+ Stats.incr("dag_bag.cache.evictions")
+ Stats.gauge("dag_bag.cache.size", len(self._dags), rate=0.1)
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
