update nodes and scalesets in a consistent order
@@ -33,22 +33,30 @@ def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa:
# NOTE: Update pools first, such that scalesets impacted by pool updates
# (such as shutdown or resize) happen during this iteration `timer_worker`
# rather than the following iteration.
+
pools = Pool.search()
for pool in pools:
if pool.state in PoolState.needs_work():
logging.info("update pool: %s (%s)", pool.pool_id, pool.name)
process_state_updates(pool)
- elif pool.state in PoolState.available() and pool.autoscale:
+
+ if pool.state in PoolState.available() and pool.autoscale:
autoscale_pool(pool)
+ # NOTE: Nodes, and Scalesets should be processed in a consistent order such
+ # during 'pool scale down' operations. This means that pools that are
+ # scaling down will more likely remove from the same scalesets over time.
+ # By more likely removing from the same scalesets, we are more likely to
+ # get to empty scalesets, which can safely be deleted.
+
Node.mark_outdated_nodes()
nodes = Node.search_states(states=NodeState.needs_work())
- for node in nodes:
+ for node in sorted(nodes, key=lambda x: x.machine_id):
logging.info("update node: %s", node.machine_id)
process_state_updates(node)
scalesets = Scaleset.search()
- for scaleset in scalesets:
+ for scaleset in sorted(scalesets, key=lambda x: x.scaleset_id):
process_scaleset(scaleset)
events = get_events()
