service_capacity_modeling/models/org/netflix/elasticsearch.py
def _estimate_elasticsearch_requirement(
    node_type: str,
    instance: Instance,
    desires: CapacityDesires,
    working_set: float,
    reads_per_second: float,
    max_rps_to_disk: int,
    zones_in_region: int = 3,
    copies_per_region: int = 3,
    jvm_memory_overhead=1.2,
) -> CapacityRequirement:
    """Estimate the capacity required for one zone given a regional desire

    The input desires should be the **regional** desire, and this function will
    return the zonal capacity requirement
    """
    # Keep half of the cores free for background work (merging mostly)
    needed_cores = math.ceil(sqrt_staffed_cores(desires) * 1.5)
    # Keep half of the bandwidth available for backup
    needed_network_mbps = simple_network_mbps(desires) * 2

    needed_disk = math.ceil(
        (1.0 / desires.data_shape.estimated_compression_ratio.mid)
        * desires.data_shape.estimated_state_size_gib.mid
        * copies_per_region,
    )

    # Rough estimate of how many instances we would need just for the the CPU
    # Note that this is a lower bound, we might end up with more.
    needed_cores = math.ceil(
        max(1, needed_cores // (instance.cpu_ghz / desires.core_reference_ghz))
    )
    rough_count = math.ceil(needed_cores / instance.cpu)

    # Generally speaking we want fewer than some number of reads per second
    # hitting disk per instance. If we don't have many reads we don't need to
    # hold much data in memory.
    instance_rps = max(1, reads_per_second // rough_count)
    disk_rps = instance_rps * _es_io_per_read(max(1, needed_disk // rough_count))
    rps_working_set = min(1.0, disk_rps / max_rps_to_disk)

    mem_allocation_mbps = needed_network_mbps * jvm_memory_overhead
    heap_allocation_gibps = (mem_allocation_mbps / 8) / 1024
    network_heap = heap_allocation_gibps * 2

    # If disk RPS will be smaller than our target because there are no
    # reads, we don't need to hold as much data in memory
    needed_memory = min(working_set, rps_working_set) * needed_disk + network_heap

    # Now convert to per zone
    needed_cores = needed_cores // zones_in_region
    needed_disk = needed_disk // zones_in_region
    needed_memory = int(needed_memory // zones_in_region)
    logger.debug(
        "Need (cpu, mem, disk, working) = (%s, %s, %s, %f)",
        needed_cores,
        needed_memory,
        needed_disk,
        working_set,
    )

    return CapacityRequirement(
        requirement_type=f"elasticsearch-{node_type}-zonal",
        core_reference_ghz=desires.core_reference_ghz,
        cpu_cores=certain_int(needed_cores),
        mem_gib=certain_float(needed_memory),
        disk_gib=certain_float(needed_disk),
        network_mbps=certain_float(needed_network_mbps),
        context={
            "working_set": min(working_set, rps_working_set),
            "rps_working_set": rps_working_set,
            "disk_slo_working_set": working_set,
            "replication_factor": copies_per_region,
            "compression_ratio": round(
                1.0 / desires.data_shape.estimated_compression_ratio.mid, 2
            ),
            "read_per_second": reads_per_second,
        },
    )
Estimate the capacity required for one zone given a regional desire

    The input desires should be the **regional** desire, and this function will
    return the zonal capacity requirement
    
Estimate the capacity required for one zone given a regional desire
The input desires should be the **regional** desire, and this function will
return the zonal capacity requirement
Estimate the capacity required for one zone given a regional desire The input desires should be the ** regional ** desire and this function will return the zonal capacity requirement
def _estimate_elasticsearch_requirement(
    node_type: str,
    instance: Instance,
    desires: CapacityDesires,
    working_set: float,
    reads_per_second: float,
    max_rps_to_disk: int,
    zones_in_region: int = 3,
    copies_per_region: int = 3,
    jvm_memory_overhead=1.2,
) -> CapacityRequirement:
    needed_cores = math.ceil(sqrt_staffed_cores(desires) * 1.5)
    needed_network_mbps = simple_network_mbps(desires) * 2
    needed_disk = math.ceil(
        (1.0 / desires.data_shape.estimated_compression_ratio.mid)
        * desires.data_shape.estimated_state_size_gib.mid
        * copies_per_region,
    )
    needed_cores = math.ceil(
        max(1, needed_cores // (instance.cpu_ghz / desires.core_reference_ghz))
    )
    rough_count = math.ceil(needed_cores / instance.cpu)
    instance_rps = max(1, reads_per_second // rough_count)
    disk_rps = instance_rps * _es_io_per_read(max(1, needed_disk // rough_count))
    rps_working_set = min(1.0, disk_rps / max_rps_to_disk)
    mem_allocation_mbps = needed_network_mbps * jvm_memory_overhead
    heap_allocation_gibps = (mem_allocation_mbps / 8) / 1024
    network_heap = heap_allocation_gibps * 2
    needed_memory = min(working_set, rps_working_set) * needed_disk + network_heap
    needed_cores = needed_cores // zones_in_region
    needed_disk = needed_disk // zones_in_region
    needed_memory = int(needed_memory // zones_in_region)
    logger.debug(
        "Need (cpu, mem, disk, working) = (%s, %s, %s, %f)",
        needed_cores,
        needed_memory,
        needed_disk,
        working_set,
    )
    return CapacityRequirement(
        requirement_type=f"elasticsearch-{node_type}-zonal",
        core_reference_ghz=desires.core_reference_ghz,
        cpu_cores=certain_int(needed_cores),
        mem_gib=certain_float(needed_memory),
        disk_gib=certain_float(needed_disk),
        network_mbps=certain_float(needed_network_mbps),
        context={
            "working_set": min(working_set, rps_working_set),
            "rps_working_set": rps_working_set,
            "disk_slo_working_set": working_set,
            "replication_factor": copies_per_region,
            "compression_ratio": round(
                1.0 / desires.data_shape.estimated_compression_ratio.mid, 2
            ),
            "read_per_second": reads_per_second,
        },
    )
def _estimate_elasticsearch_requirement ( node_type : str , instance : Instance , desires : CapacityDesires , working_set : float , reads_per_second : float , max_rps_to_disk : int , zones_in_region : int = 3 , copies_per_region : int = 3 , jvm_memory_overhead = 1.2 , ) -> CapacityRequirement : needed_cores = math . ceil ( sqrt_staffed_cores ( desires ) * 1.5 ) needed_network_mbps = simple_network_mbps ( desires ) * 2 needed_disk = math . ceil ( ( 1.0 / desires . data_shape . estimated_compression_ratio . mid ) * desires . data_shape . estimated_state_size_gib . mid * copies_per_region , ) needed_cores = math . ceil ( max ( 1 , needed_cores // ( instance . cpu_ghz / desires . core_reference_ghz ) ) ) rough_count = math . ceil ( needed_cores / instance . cpu ) instance_rps = max ( 1 , reads_per_second // rough_count ) disk_rps = instance_rps * _es_io_per_read ( max ( 1 , needed_disk // rough_count ) ) rps_working_set = min ( 1.0 , disk_rps / max_rps_to_disk ) mem_allocation_mbps = needed_network_mbps * jvm_memory_overhead heap_allocation_gibps = ( mem_allocation_mbps / 8 ) / 1024 network_heap = heap_allocation_gibps * 2 needed_memory = min ( working_set , rps_working_set ) * needed_disk + network_heap needed_cores = needed_cores // zones_in_region needed_disk = needed_disk // zones_in_region needed_memory = int ( needed_memory // zones_in_region ) logger . debug ( "Need (cpu, mem, disk, working) = (%s, %s, %s, %f)" , needed_cores , needed_memory , needed_disk , working_set , ) return CapacityRequirement ( requirement_type = f"elasticsearch-{node_type}-zonal" , core_reference_ghz = desires . core_reference_ghz , cpu_cores = certain_int ( needed_cores ) , mem_gib = certain_float ( needed_memory ) , disk_gib = certain_float ( needed_disk ) , network_mbps = certain_float ( needed_network_mbps ) , context = { "working_set" : min ( working_set , rps_working_set ) , "rps_working_set" : rps_working_set , "disk_slo_working_set" : working_set , "replication_factor" : copies_per_region , "compression_ratio" : round ( 1.0 / desires . data_shape . estimated_compression_ratio . mid , 2 ) , "read_per_second" : reads_per_second , } , )
Estimate the capacity required for one zone given a regional desire
The input desires should be the **regional** desire, and this function will
return the zonal capacity requirement
Estimate the capacity required for one zone given a regional desire The input desires should be the ** regional ** desire and this function will return the zonal capacity requirement
"""Estimate the capacity required for one zone given a regional desire

    The input desires should be the **regional** desire, and this function will
    return the zonal capacity requirement
    """ # Keep half of the cores free for background work (merging mostly) # Keep half of the bandwidth available for backup # Rough estimate of how many instances we would need just for the the CPU # Note that this is a lower bound, we might end up with more. # Generally speaking we want fewer than some number of reads per second # hitting disk per instance. If we don't have many reads we don't need to # hold much data in memory. # If disk RPS will be smaller than our target because there are no # reads, we don't need to hold as much data in memory # Now convert to per zone
node_type
str
instance
Instance
desires
CapacityDesires
working_set
float
reads_per_second
float
max_rps_to_disk
int
zones_in_region
int
copies_per_region
int
jvm_memory_overhead