<NME> jit.rs
<BEF> use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
use std::iter::FromIterator;
use std::mem;
use std::ptr::NonNull;

use analysis::AnalysisType;
use codegen;
use control_flow;
use control_flow::WasmStructure;
use cpu::cpu;
use cpu::global_pointers;
use cpu::memory;
use cpu_context::CpuContext;
use jit_instructions;
use opstats;
use page::Page;
use profiler;
use profiler::stat;
use state_flags::CachedStateFlags;
use util::SafeToU16;
use wasmgen::wasm_builder::{Label, WasmBuilder, WasmLocal};

#[derive(Copy, Clone, Eq, Hash, PartialEq)]
#[repr(transparent)]
pub struct WasmTableIndex(u16);
impl WasmTableIndex {
    pub fn to_u16(self) -> u16 { self.0 }
}

mod unsafe_jit {
    use jit::{CachedStateFlags, WasmTableIndex};

    extern "C" {
        pub fn codegen_finalize(
            wasm_table_index: WasmTableIndex,
            phys_addr: u32,
            state_flags: CachedStateFlags,
            ptr: u32,
            len: u32,
        );
        pub fn jit_clear_func(wasm_table_index: WasmTableIndex);
    }
}

fn codegen_finalize(
    wasm_table_index: WasmTableIndex,
    phys_addr: u32,
    state_flags: CachedStateFlags,
    ptr: u32,
    len: u32,
) {
    unsafe { unsafe_jit::codegen_finalize(wasm_table_index, phys_addr, state_flags, ptr, len) }
}

pub fn jit_clear_func(wasm_table_index: WasmTableIndex) {
    unsafe { unsafe_jit::jit_clear_func(wasm_table_index) }
}

// less branches will generate if-else, more will generate brtable
pub const BRTABLE_CUTOFF: usize = 10;

pub const WASM_TABLE_SIZE: u32 = 900;

pub const HASH_PRIME: u32 = 6151;

pub const CHECK_JIT_STATE_INVARIANTS: bool = false;

pub const JIT_USE_LOOP_SAFETY: bool = true;

pub const JIT_THRESHOLD: u32 = 200 * 1000;

pub const MAX_EXTRA_BASIC_BLOCKS: usize = 250;

const MAX_INSTRUCTION_LENGTH: u32 = 16;

#[allow(non_upper_case_globals)]
static mut jit_state: NonNull<JitState> =
    unsafe { NonNull::new_unchecked(mem::align_of::<JitState>() as *mut _) };

pub fn get_jit_state() -> &'static mut JitState { unsafe { jit_state.as_mut() } }

#[no_mangle]
pub fn rust_init() {
    let x = Box::new(JitState::create_and_initialise());
    unsafe {
        jit_state = NonNull::new(Box::into_raw(x)).unwrap()
    }

    use std::panic;

    panic::set_hook(Box::new(|panic_info| {
        console_log!("{}", panic_info.to_string());
    }));
}

pub struct Entry {
    #[cfg(any(debug_assertions, feature = "profiler"))]
    pub len: u32,

    #[cfg(debug_assertions)]
    pub opcode: u32,

    pub initial_state: u16,
    pub wasm_table_index: WasmTableIndex,
    pub state_flags: CachedStateFlags,
}

enum PageState {
    Compiling { entries: Vec<(u32, Entry)> },
    CompilingWritten,
}

pub struct JitState {
    wasm_builder: WasmBuilder,

    // as an alternative to HashSet, we could use a bitmap of 4096 bits here
    // (faster, but uses much more memory)
    // or a compressed bitmap (likely faster)
    // or HashSet<u32> rather than nested
    entry_points: HashMap<Page, HashSet<u16>>,
    hot_pages: [u32; HASH_PRIME as usize],

    wasm_table_index_free_list: Vec<WasmTableIndex>,
    used_wasm_table_indices: HashMap<WasmTableIndex, HashSet<Page>>,
    // All pages from used_wasm_table_indices
    // Used to improve the performance of jit_dirty_page and jit_page_has_code
    all_pages: HashSet<Page>,
    cache: HashMap<u32, Entry>,
    compiling: Option<(WasmTableIndex, PageState)>,
}

pub fn check_jit_state_invariants(ctx: &mut JitState) {
    if !CHECK_JIT_STATE_INVARIANTS {
        return;
    }
    let mut all_pages = HashSet::new();
    for pages in ctx.used_wasm_table_indices.values() {
        all_pages.extend(pages);
    }
    dbg_assert!(ctx.all_pages == all_pages);
}

impl JitState {
    pub fn create_and_initialise() -> JitState {
        // don't assign 0 (XXX: Check)
        let wasm_table_indices = (1..=(WASM_TABLE_SIZE - 1) as u16).map(|x| WasmTableIndex(x));

        JitState {
            wasm_builder: WasmBuilder::new(),

            entry_points: HashMap::new(),
            hot_pages: [0; HASH_PRIME as usize],

            wasm_table_index_free_list: Vec::from_iter(wasm_table_indices),
            used_wasm_table_indices: HashMap::new(),
            all_pages: HashSet::new(),
            cache: HashMap::new(),
            compiling: None,
        }
    }
}

#[derive(PartialEq, Eq)]
pub enum BasicBlockType {
    Normal {
        next_block_addr: Option<u32>,
        jump_offset: i32,
        jump_offset_is_32: bool,
    },
    ConditionalJump {
        next_block_addr: Option<u32>,
        next_block_branch_taken_addr: Option<u32>,
        condition: u8,
        jump_offset: i32,
        jump_offset_is_32: bool,
    },
    // Set eip to an absolute value (ret, jmp r/m, call r/m)
    AbsoluteEip,
    Exit,
}

pub struct BasicBlock {
    pub addr: u32,
    pub virt_addr: i32,
    pub last_instruction_addr: u32,
    pub end_addr: u32,
    pub is_entry_block: bool,
    pub ty: BasicBlockType,
    pub has_sti: bool,
    pub number_of_instructions: u32,
}

#[derive(Copy, Clone, PartialEq)]
pub struct CachedCode {
    pub wasm_table_index: WasmTableIndex,
    pub initial_state: u16,
}

impl CachedCode {
    pub const NONE: CachedCode = CachedCode {
        wasm_table_index: WasmTableIndex(0),
        initial_state: 0,
    };
}

#[derive(PartialEq)]
pub enum InstructionOperand {
    WasmLocal(WasmLocal),
    Immediate(i32),
    Other,
}
pub enum Instruction {
    Cmp {
        dest: InstructionOperand,
        source: InstructionOperand,
        opsize: i32,
    },
    Sub {
        opsize: i32,
    },
    // Any instruction that sets last_result
    Arithmetic {
        opsize: i32,
    },
    Other,
}

pub struct JitContext<'a> {
    pub cpu: &'a mut CpuContext,
    pub builder: &'a mut WasmBuilder,
    pub register_locals: &'a mut Vec<WasmLocal>,
    pub start_of_current_instruction: u32,
    pub exit_with_fault_label: Label,
    pub exit_label: Label,
    pub current_instruction: Instruction,
    pub previous_instruction: Instruction,
    pub instruction_counter: WasmLocal,
}
impl<'a> JitContext<'a> {
    pub fn reg(&self, i: u32) -> WasmLocal { self.register_locals[i as usize].unsafe_clone() }
}

pub const JIT_INSTR_BLOCK_BOUNDARY_FLAG: u32 = 1 << 0;

fn jit_hot_hash_page(page: Page) -> u32 { page.to_u32() % HASH_PRIME }

pub fn is_near_end_of_page(address: u32) -> bool {
    address & 0xFFF >= 0x1000 - MAX_INSTRUCTION_LENGTH
}

pub fn jit_find_cache_entry(phys_address: u32, state_flags: CachedStateFlags) -> CachedCode {
    if is_near_end_of_page(phys_address) {
        profiler::stat_increment(stat::RUN_INTERPRETED_NEAR_END_OF_PAGE);
    }

    let ctx = get_jit_state();

    match ctx.cache.get(&phys_address) {
        Some(entry) => {
            if entry.state_flags == state_flags {
                return CachedCode {
                    wasm_table_index: entry.wasm_table_index,
                    initial_state: entry.initial_state,
                };
            }
            else {
                profiler::stat_increment(stat::RUN_INTERPRETED_DIFFERENT_STATE);
            }
        },
        None => {},
    }

    return CachedCode::NONE;
}

#[no_mangle]
pub fn jit_find_cache_entry_in_page(
    phys_address: u32,
    wasm_table_index: WasmTableIndex,
    state_flags: u32,
) -> i32 {
    profiler::stat_increment(stat::INDIRECT_JUMP);

    let state_flags = CachedStateFlags::of_u32(state_flags);
    let ctx = get_jit_state();

    match ctx.cache.get(&phys_address) {
        Some(entry) => {
            if entry.state_flags == state_flags && entry.wasm_table_index == wasm_table_index {
                return entry.initial_state as i32;
            }
        },
        None => {},
    }

    profiler::stat_increment(stat::INDIRECT_JUMP_NO_ENTRY);

    return -1;
}

pub fn record_entry_point(phys_address: u32) {
    let ctx = get_jit_state();
    if is_near_end_of_page(phys_address) {
        return;
    }
    let page = Page::page_of(phys_address);
    let offset_in_page = phys_address as u16 & 0xFFF;
    let mut is_new = false;
    ctx.entry_points
        .entry(page)
        .or_insert_with(|| {
            is_new = true;
            HashSet::new()
        })
        .insert(offset_in_page);

    if is_new {
        cpu::tlb_set_has_code(page, true);
    }
}

// Maximum number of pages per wasm module. Necessary for the following reasons:
// - There is an upper limit on the size of a single function in wasm (currently ~7MB in all browsers)
//   See https://github.com/WebAssembly/design/issues/1138
// - v8 poorly handles large br_table elements and OOMs on modules much smaller than the above limit
//   See https://bugs.chromium.org/p/v8/issues/detail?id=9697 and https://bugs.chromium.org/p/v8/issues/detail?id=9141
//   Will hopefully be fixed in the near future by generating direct control flow
const MAX_PAGES: usize = 3;

fn jit_find_basic_blocks(
    ctx: &mut JitState,
    entry_points: HashSet<i32>,
    cpu: CpuContext,
) -> Vec<BasicBlock> {
    fn follow_jump(
        virt_target: i32,
        ctx: &mut JitState,
        pages: &mut HashSet<Page>,
        page_blacklist: &mut HashSet<Page>,
        max_pages: usize,
        marked_as_entry: &mut HashSet<i32>,
        to_visit_stack: &mut Vec<i32>,
    ) -> Option<u32> {
        if is_near_end_of_page(virt_target as u32) {
            return None;
        }
        let phys_target = match cpu::translate_address_read_no_side_effects(virt_target) {
            Err(()) => {
                dbg_log!("Not analysing {:x} (page not mapped)", virt_target);
                return None;
            },
            Ok(t) => t,
        };

        let phys_page = Page::page_of(phys_target);

        if !pages.contains(&phys_page) && pages.len() == max_pages
            || page_blacklist.contains(&phys_page)
        {
            return None;
        }

        if !pages.contains(&phys_page) {
            // page seen for the first time, handle entry points
            if let Some(entry_points) = ctx.entry_points.get(&phys_page) {
                if entry_points.iter().all(|&entry_point| {
                    ctx.cache
                        .contains_key(&(phys_page.to_address() | u32::from(entry_point)))
                }) {
                    profiler::stat_increment(stat::COMPILE_PAGE_SKIPPED_NO_NEW_ENTRY_POINTS);
                    page_blacklist.insert(phys_page);
                    return None;
                }

                let address_hash = jit_hot_hash_page(phys_page) as usize;
                ctx.hot_pages[address_hash] = 0;

                for &addr_low in entry_points {
                    let addr = virt_target & !0xFFF | addr_low as i32;
                    to_visit_stack.push(addr);
                    marked_as_entry.insert(addr);
                }
            }
            else {
                // no entry points: ignore this page?
            }

            pages.insert(phys_page);
            dbg_assert!(pages.len() <= max_pages);
        }

        to_visit_stack.push(virt_target);
        Some(phys_target)
    }

    let mut to_visit_stack: Vec<i32> = Vec::new();
    let mut marked_as_entry: HashSet<i32> = HashSet::new();
    let mut basic_blocks: BTreeMap<u32, BasicBlock> = BTreeMap::new();
    let mut pages: HashSet<Page> = HashSet::new();
    let mut page_blacklist = HashSet::new();

    // 16-bit doesn't not work correctly, most likely due to instruction pointer wrap-around
    let max_pages = if cpu.state_flags.is_32() { MAX_PAGES } else { 1 };

    for virt_addr in entry_points {
        let ok = follow_jump(
            virt_addr,
            ctx,
            &mut pages,
            &mut page_blacklist,
            max_pages,
            &mut marked_as_entry,
            &mut to_visit_stack,
        );
        dbg_assert!(ok.is_some());
        dbg_assert!(marked_as_entry.contains(&virt_addr));
    }

    while let Some(to_visit) = to_visit_stack.pop() {
        let phys_addr = match cpu::translate_address_read_no_side_effects(to_visit) {
            Err(()) => {
                dbg_log!("Not analysing {:x} (page not mapped)", to_visit);
                continue;
            },
            Ok(phys_addr) => phys_addr,
        };

        if basic_blocks.contains_key(&phys_addr) {
            continue;
        }

        if is_near_end_of_page(phys_addr) {
            // Empty basic block, don't insert
            profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);
            continue;
        }

        let mut current_address = phys_addr;
        let mut current_block = BasicBlock {
            addr: current_address,
            virt_addr: to_visit,
            last_instruction_addr: 0,
            end_addr: 0,
            ty: BasicBlockType::Exit,
            is_entry_block: false,
            has_sti: false,
            number_of_instructions: 0,
        };
        loop {
            let addr_before_instruction = current_address;
            let mut cpu = &mut CpuContext {
                eip: current_address,
                ..cpu
            };
            let analysis = ::analysis::analyze_step(&mut cpu);
            current_block.number_of_instructions += 1;
            let has_next_instruction = !analysis.no_next_instruction;
            current_address = cpu.eip;

            dbg_assert!(Page::page_of(current_address) == Page::page_of(addr_before_instruction));
            let current_virt_addr = to_visit & !0xFFF | current_address as i32 & 0xFFF;

            match analysis.ty {
                AnalysisType::Normal | AnalysisType::STI => {
                    dbg_assert!(has_next_instruction);
                    dbg_assert!(!analysis.absolute_jump);

                    if current_block.has_sti {
                        // Convert next instruction after STI (i.e., the current instruction) into block boundary

                        marked_as_entry.insert(current_virt_addr);
                        to_visit_stack.push(current_virt_addr);

                        current_block.last_instruction_addr = addr_before_instruction;
                        current_block.end_addr = current_address;
                        break;
                    }

                    if analysis.ty == AnalysisType::STI {
                        current_block.has_sti = true;

                        dbg_assert!(
                            !is_near_end_of_page(current_address),
                            "TODO: Handle STI instruction near end of page"
                        );
                    }
                    else {
                        // Only split non-STI blocks (one instruction needs to run after STI before
                        // handle_irqs may be called)

                        if basic_blocks.contains_key(&current_address) {
                            current_block.last_instruction_addr = addr_before_instruction;
                            current_block.end_addr = current_address;
                            dbg_assert!(!is_near_end_of_page(current_address));
                            current_block.ty = BasicBlockType::Normal {
                                next_block_addr: Some(current_address),
                                jump_offset: 0,
                                jump_offset_is_32: true,
                            };
                            break;
                        }
                    }
                },
                AnalysisType::Jump {
                    offset,
                    is_32,
                    condition: Some(condition),
                } => {
                    dbg_assert!(!analysis.absolute_jump);
                    // conditional jump: continue at next and continue at jump target

                    let jump_target = if is_32 {
                        current_virt_addr + offset
                    }
                    else {
                        cpu.cs_offset as i32
                            + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)
                    };

                    dbg_assert!(has_next_instruction);
                    to_visit_stack.push(current_virt_addr);

                    let next_block_addr = if is_near_end_of_page(current_address) {
                        None
                    }
                    else {
                        Some(current_address)
                    };

                    current_block.ty = BasicBlockType::ConditionalJump {
                        next_block_addr,
                        next_block_branch_taken_addr: follow_jump(
                            jump_target,
                            ctx,
                            &mut pages,
                            &mut page_blacklist,
                            max_pages,
                            &mut marked_as_entry,
                            &mut to_visit_stack,
                        ),
                        condition,
                        jump_offset: offset,
                        jump_offset_is_32: is_32,
                    };

                    current_block.last_instruction_addr = addr_before_instruction;
                    current_block.end_addr = current_address;

                    break;
                },
                AnalysisType::Jump {
                    offset,
                    is_32,
                    condition: None,
                } => {
                    dbg_assert!(!analysis.absolute_jump);
                    // non-conditional jump: continue at jump target

                    let jump_target = if is_32 {
                        current_virt_addr + offset
                    }
                    else {
                        cpu.cs_offset as i32
                            + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)
                    };

                    if has_next_instruction {
                        // Execution will eventually come back to the next instruction (CALL)
                        marked_as_entry.insert(current_virt_addr);
                        to_visit_stack.push(current_virt_addr);
                    }

                    current_block.ty = BasicBlockType::Normal {
                        next_block_addr: follow_jump(
                            jump_target,
                            ctx,
                            &mut pages,
                            &mut page_blacklist,
                            max_pages,
                            &mut marked_as_entry,
                            &mut to_visit_stack,
                        ),
                        jump_offset: offset,
                        jump_offset_is_32: is_32,
                    };
                    current_block.last_instruction_addr = addr_before_instruction;
                    current_block.end_addr = current_address;

                    break;
                },
                AnalysisType::BlockBoundary => {
                    // a block boundary but not a jump, get out

                    if has_next_instruction {
                        // block boundary, but execution will eventually come back
                        // to the next instruction. Create a new basic block
                        // starting at the next instruction and register it as an
                        // entry point
                        marked_as_entry.insert(current_virt_addr);
                        to_visit_stack.push(current_virt_addr);
                    }

                    if analysis.absolute_jump {
                        current_block.ty = BasicBlockType::AbsoluteEip;
                    }

                    current_block.last_instruction_addr = addr_before_instruction;
                    current_block.end_addr = current_address;
                    break;
                },
            }

            if is_near_end_of_page(current_address) {
                current_block.last_instruction_addr = addr_before_instruction;
                current_block.end_addr = current_address;
                profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);
                break;
            }
        }

        let previous_block = basic_blocks
            .range(..current_block.addr)
            .next_back()
            .filter(|(_, previous_block)| (!previous_block.has_sti))
            .map(|(_, previous_block)| previous_block.clone());

        if let Some(previous_block) = previous_block {
            if current_block.addr < previous_block.end_addr {
                // If this block overlaps with the previous block, re-analyze the previous block
                to_visit_stack.push(previous_block.virt_addr);

                let addr = previous_block.addr;
                let old_block = basic_blocks.remove(&addr);
                dbg_assert!(old_block.is_some());

                // Note that this does not ensure the invariant that two consecutive blocks don't
                // overlay. For that, we also need to check the following block.
            }
        }

        dbg_assert!(current_block.addr < current_block.end_addr);
        dbg_assert!(current_block.addr <= current_block.last_instruction_addr);
        dbg_assert!(current_block.last_instruction_addr < current_block.end_addr);

        basic_blocks.insert(current_block.addr, current_block);
    }

    dbg_assert!(pages.len() <= max_pages);

    for block in basic_blocks.values_mut() {
        if marked_as_entry.contains(&block.virt_addr) {
            block.is_entry_block = true;
        }
    }

    let basic_blocks: Vec<BasicBlock> = basic_blocks.into_iter().map(|(_, block)| block).collect();

    for i in 0..basic_blocks.len() - 1 {
        let next_block_addr = basic_blocks[i + 1].addr;
        let next_block_end_addr = basic_blocks[i + 1].end_addr;
        let next_block_is_entry = basic_blocks[i + 1].is_entry_block;
        let block = &basic_blocks[i];
        dbg_assert!(block.addr < next_block_addr);
        if next_block_addr < block.end_addr {
            dbg_log!(
                "Overlapping first=[from={:x} to={:x} is_entry={}] second=[from={:x} to={:x} is_entry={}]",
                block.addr,
                block.end_addr,
                block.is_entry_block as u8,
                next_block_addr,
                next_block_end_addr,
                next_block_is_entry as u8
            );
        }
    }

    basic_blocks
}

#[no_mangle]
#[cfg(debug_assertions)]
pub fn jit_force_generate_unsafe(virt_addr: i32) {
    let ctx = get_jit_state();
    let phys_addr = cpu::translate_address_read(virt_addr).unwrap();
    record_entry_point(phys_addr);
    let cs_offset = cpu::get_seg_cs() as u32;
    let state_flags = cpu::pack_current_state_flags();
    jit_analyze_and_generate(ctx, virt_addr, phys_addr, cs_offset, state_flags);
}

#[inline(never)]
fn jit_analyze_and_generate(
    ctx: &mut JitState,
    virt_entry_point: i32,
    phys_entry_point: u32,
    cs_offset: u32,
    state_flags: CachedStateFlags,
) {
    let page = Page::page_of(phys_entry_point);

    if ctx.compiling.is_some() {
        return;
    }

    let entry_points = ctx.entry_points.get(&page);

    let entry_points = match entry_points {
        None => return,
        Some(entry_points) => entry_points,
    };

    if entry_points.iter().all(|&entry_point| {
        ctx.cache
            .contains_key(&(page.to_address() | u32::from(entry_point)))
    }) {
        profiler::stat_increment(stat::COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS);
        return;
    }

    profiler::stat_increment(stat::COMPILE);

    let cpu = CpuContext {
        eip: 0,
        prefixes: 0,
        cs_offset,
        state_flags,
    };

    dbg_assert!(
        cpu::translate_address_read_no_side_effects(virt_entry_point).unwrap() == phys_entry_point
    );
    let virt_page = Page::page_of(virt_entry_point as u32);
    let entry_points: HashSet<i32> = entry_points
        .iter()
        .map(|e| virt_page.to_address() as i32 | *e as i32)
        .collect();
    let basic_blocks = jit_find_basic_blocks(ctx, entry_points, cpu.clone());

    let mut pages = HashSet::new();

    for b in basic_blocks.iter() {
        // Remove this assertion once page-crossing jit is enabled
        dbg_assert!(Page::page_of(b.addr) == Page::page_of(b.end_addr));
        pages.insert(Page::page_of(b.addr));
    }

    let print = false;

    for b in basic_blocks.iter() {
        if !print {
            break;
        }
        let last_instruction_opcode = memory::read32s(b.last_instruction_addr);
        let op = opstats::decode(last_instruction_opcode as u32);
        dbg_log!(
            "BB: 0x{:x} {}{:02x} {} {}",
            b.addr,
            if op.is_0f { "0f" } else { "" },
            op.opcode,
            if b.is_entry_block { "entry" } else { "noentry" },
            match &b.ty {
                BasicBlockType::ConditionalJump {
                    next_block_addr: Some(next_block_addr),
                    next_block_branch_taken_addr: Some(next_block_branch_taken_addr),
                    ..
                } => format!(
                    "0x{:x} 0x{:x}",
                    next_block_addr, next_block_branch_taken_addr
                ),
                BasicBlockType::ConditionalJump {
                    next_block_addr: None,
                    next_block_branch_taken_addr: Some(next_block_branch_taken_addr),
                    ..
                } => format!("0x{:x}", next_block_branch_taken_addr),
                BasicBlockType::ConditionalJump {
                    next_block_addr: Some(next_block_addr),
                    next_block_branch_taken_addr: None,
                    ..
                } => format!("0x{:x}", next_block_addr),
                BasicBlockType::ConditionalJump {
                    next_block_addr: None,
                    next_block_branch_taken_addr: None,
                    ..
                } => format!(""),
                BasicBlockType::Normal {
                    next_block_addr: Some(next_block_addr),
                    ..
                } => format!("0x{:x}", next_block_addr),
                BasicBlockType::Normal {
                    next_block_addr: None,
                    ..
                } => format!(""),
                BasicBlockType::Exit => format!(""),
                BasicBlockType::AbsoluteEip => format!(""),
            }
        );
    }

    let graph = control_flow::make_graph(&basic_blocks);
    let mut structure = control_flow::loopify(&graph);

    if print {
        dbg_log!("before blockify:");
        for group in &structure {
            dbg_log!("=> Group");
            group.print(0);
        }
    }

    control_flow::blockify(&mut structure, &graph);

    if cfg!(debug_assertions) {
        control_flow::assert_invariants(&structure);
    }

    if print {
        dbg_log!("after blockify:");
        for group in &structure {
            dbg_log!("=> Group");
            group.print(0);
        }
    }

    if ctx.wasm_table_index_free_list.is_empty() {
        dbg_log!("wasm_table_index_free_list empty, clearing cache");

        // When no free slots are available, delete all cached modules. We could increase the
        // size of the table, but this way the initial size acts as an upper bound for the
        // number of wasm modules that we generate, which we want anyway to avoid getting our
        // tab killed by browsers due to memory constraints.
        jit_clear_cache(ctx);

        profiler::stat_increment(stat::INVALIDATE_ALL_MODULES_NO_FREE_WASM_INDICES);

        dbg_log!(
            "after jit_clear_cache: {} free",
            ctx.wasm_table_index_free_list.len(),
        );

        // This assertion can fail if all entries are pending (not possible unless
        // WASM_TABLE_SIZE is set very low)
        dbg_assert!(!ctx.wasm_table_index_free_list.is_empty());
    }

    // allocate an index in the wasm table
    let wasm_table_index = ctx
        .wasm_table_index_free_list
        .pop()
        .expect("allocate wasm table index");
    dbg_assert!(wasm_table_index != WasmTableIndex(0));

    dbg_assert!(!pages.is_empty());
    dbg_assert!(pages.len() <= MAX_PAGES);
    ctx.used_wasm_table_indices
        .insert(wasm_table_index, pages.clone());
    ctx.all_pages.extend(pages.clone());

    let basic_block_by_addr: HashMap<u32, BasicBlock> =
        basic_blocks.into_iter().map(|b| (b.addr, b)).collect();

    let entries = jit_generate_module(
        structure,
        &basic_block_by_addr,
        cpu.clone(),
        &mut ctx.wasm_builder,
        wasm_table_index,
        state_flags,
    );
    dbg_assert!(!entries.is_empty());

    profiler::stat_increment_by(
        stat::COMPILE_WASM_TOTAL_BYTES,
        ctx.wasm_builder.get_output_len() as u64,
    );
    profiler::stat_increment_by(stat::COMPILE_PAGE, pages.len() as u64);

    cpu::tlb_set_has_code_multiple(&pages, true);

    dbg_assert!(ctx.compiling.is_none());
    ctx.compiling = Some((wasm_table_index, PageState::Compiling { entries }));

    let phys_addr = page.to_address();

    // will call codegen_finalize_finished asynchronously when finished
    codegen_finalize(
        wasm_table_index,
        phys_addr,
        state_flags,
        ctx.wasm_builder.get_output_ptr() as u32,
        ctx.wasm_builder.get_output_len(),
    );

    profiler::stat_increment(stat::COMPILE_SUCCESS);

    check_jit_state_invariants(ctx);
}

#[no_mangle]
pub fn codegen_finalize_finished(
    wasm_table_index: WasmTableIndex,
    phys_addr: u32,
    state_flags: CachedStateFlags,
) {
    let ctx = get_jit_state();

    dbg_assert!(wasm_table_index != WasmTableIndex(0));

    dbg_log!(
        "Finished compiling for page at {:x}",
        Page::page_of(phys_addr).to_address()
    );

    let entries = match mem::replace(&mut ctx.compiling, None) {
        None => {
            dbg_assert!(false);
            return;
        },
        Some((in_progress_wasm_table_index, PageState::CompilingWritten)) => {
            dbg_assert!(wasm_table_index == in_progress_wasm_table_index);

            profiler::stat_increment(stat::INVALIDATE_MODULE_WRITTEN_WHILE_COMPILED);
            free_wasm_table_index(ctx, wasm_table_index);
            return;
        },
        Some((in_progress_wasm_table_index, PageState::Compiling { entries })) => {
            dbg_assert!(wasm_table_index == in_progress_wasm_table_index);
            entries
        },
    };

    let mut check_for_unused_wasm_table_index = HashSet::new();

    dbg_assert!(!entries.is_empty());
    for (addr, entry) in entries {
        let maybe_old_entry = ctx.cache.insert(addr, entry);

        if let Some(old_entry) = maybe_old_entry {
            check_for_unused_wasm_table_index.insert(old_entry.wasm_table_index);

            profiler::stat_increment(stat::JIT_CACHE_OVERRIDE);
            if old_entry.state_flags != state_flags {
                profiler::stat_increment(stat::JIT_CACHE_OVERRIDE_DIFFERENT_STATE_FLAGS)
            }
        }
    }

    for index in check_for_unused_wasm_table_index {
        let pages = ctx.used_wasm_table_indices.get(&index).unwrap();

        let mut is_used = false;
        'outer: for p in pages {
            for addr in p.address_range() {
                if let Some(entry) = ctx.cache.get(&addr) {
                    if entry.wasm_table_index == index {
                        is_used = true;
                        break 'outer;
                    }
                }
            }
        }

        if !is_used {
            profiler::stat_increment(stat::INVALIDATE_MODULE_UNUSED_AFTER_OVERWRITE);
            free_wasm_table_index(ctx, index);
        }

        if !is_used {
            for (_, entry) in &ctx.cache {
                dbg_assert!(entry.wasm_table_index != index);
            }
        }
        else {
            let mut ok = false;
            for (_, entry) in &ctx.cache {
                if entry.wasm_table_index == index {
                    ok = true;
                    break;
                }
            }
            dbg_assert!(ok);
        }
    }

    check_jit_state_invariants(ctx);
}

fn jit_generate_module(
    structure: Vec<WasmStructure>,
    basic_blocks: &HashMap<u32, BasicBlock>,
    mut cpu: CpuContext,
    builder: &mut WasmBuilder,
    wasm_table_index: WasmTableIndex,
    state_flags: CachedStateFlags,
) -> Vec<(u32, Entry)> {
    builder.reset();

    let mut register_locals = (0..8)
        .map(|i| {
            builder.load_fixed_i32(global_pointers::get_reg32_offset(i));
            builder.set_new_local()
        })
        .collect();

    builder.const_i32(0);
    let instruction_counter = builder.set_new_local();

    let exit_label = builder.block_void();
    let exit_with_fault_label = builder.block_void();
    let main_loop_label = builder.loop_void();
    if JIT_USE_LOOP_SAFETY {
        builder.get_local(&instruction_counter);
        builder.const_i32(cpu::LOOP_COUNTER);
        builder.geu_i32();
        if cfg!(feature = "profiler") {
            builder.if_void();
            codegen::gen_debug_track_jit_exit(builder, 0);
            builder.br(exit_label);
            builder.block_end();
        }
        else {
            builder.br_if(exit_label);
        }
    }
    let brtable_default = builder.block_void();

    let ctx = &mut JitContext {
        cpu: &mut cpu,
        builder,
        register_locals: &mut register_locals,
        start_of_current_instruction: 0,
        exit_with_fault_label,
        exit_label,
        current_instruction: Instruction::Other,
        previous_instruction: Instruction::Other,
        instruction_counter,
    };

    let entry_blocks = {
        let mut nodes = &structure;
        let result;
        loop {
            match &nodes[0] {
                WasmStructure::Dispatcher(e) => {
                    result = e.clone();
                    break;
                },
                WasmStructure::Loop { .. } => {
                    dbg_assert!(false);
                },
                WasmStructure::BasicBlock(_) => {
                    dbg_assert!(false);
                },
                // Note: We could use these blocks as entry points, which will yield
                // more entries for free, but it requires adding those to the dispatcher
                // It's to be investigated if this yields a performance improvement
                // See also the comment at the bottom of this function when creating entry
                // points
                WasmStructure::Block(children) => {
                    nodes = children;
                },
            }
        }
        result
    };

    let mut index_for_addr = HashMap::new();
    for (i, &addr) in entry_blocks.iter().enumerate() {
        index_for_addr.insert(addr, i as i32);
    }
    for b in basic_blocks.values() {
        if !index_for_addr.contains_key(&b.addr) {
            let i = index_for_addr.len();
            index_for_addr.insert(b.addr, i as i32);
        }
    }

    let mut label_for_addr: HashMap<u32, (Label, Option<i32>)> = HashMap::new();

    enum Work {
        WasmStructure(WasmStructure),
        BlockEnd {
            label: Label,
            targets: Vec<u32>,
            olds: HashMap<u32, (Label, Option<i32>)>,
        },
        LoopEnd {
            label: Label,
            entries: Vec<u32>,
            olds: HashMap<u32, (Label, Option<i32>)>,
        },
    }
    let mut work: VecDeque<Work> = structure
        .into_iter()
        .map(|x| Work::WasmStructure(x))
        .collect();

    while let Some(block) = work.pop_front() {
        let next_addr: Option<Vec<u32>> = work.iter().find_map(|x| match x {
            Work::WasmStructure(l) => Some(l.head().collect()),
            _ => None,
        });
        let target_block = &ctx.builder.arg_local_initial_state.unsafe_clone();

        match block {
            Work::WasmStructure(WasmStructure::BasicBlock(addr)) => {
                let block = basic_blocks.get(&addr).unwrap();
                jit_generate_basic_block(ctx, block);

                if block.has_sti {
                    match block.ty {
                        BasicBlockType::ConditionalJump {
                            condition,
                            jump_offset,
                            jump_offset_is_32,
                            ..
                        } => {
                            codegen::gen_set_eip_low_bits(
                                ctx.builder,
                                block.end_addr as i32 & 0xFFF,
                            );
                            codegen::gen_condition_fn(ctx, condition);
                            ctx.builder.if_void();
                            if jump_offset_is_32 {
                                codegen::gen_relative_jump(ctx.builder, jump_offset);
                            }
                            else {
                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                            }
                            ctx.builder.block_end();
                        },
                        BasicBlockType::Normal {
                            jump_offset,
                            jump_offset_is_32,
                            ..
                        } => {
        jit_instructions::jit_instruction(&mut cpu, builder, &mut instruction_flags);
        let end_eip = cpu.eip;

        #[cfg(debug_assertions)]
        {
            if ::config::CHECK_CPU_EXCEPTIONS {
                // only the last instruction in each basic block is allowed to raise
                if end_eip < stop_addr {
                    let fn_idx =
                        builder.get_fn_idx("assert_no_cpu_exception", module_init::FN0_TYPE_INDEX);
                    builder.instruction_body.call_fn(fn_idx);
                }
            }
        }

        let instruction_length = end_eip - start_eip;
        let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;

                            }
                            else {
                                codegen::gen_set_eip_low_bits(
                                    ctx.builder,
                                    block.end_addr as i32 & 0xFFF,
                                );
                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                            }
                        },
                        BasicBlockType::Exit => {},
                        BasicBlockType::AbsoluteEip => {},
                    };
                    codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
                    codegen::gen_move_registers_from_locals_to_memory(ctx);
                    codegen::gen_fn0_const(ctx.builder, "handle_irqs");
                    codegen::gen_update_instruction_counter(ctx);
                    ctx.builder.return_();
                    continue;
                }

                match &block.ty {
                    BasicBlockType::Exit => {
                        // Exit this function
                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
                        codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);
                        ctx.builder.br(ctx.exit_label);
                    },
                    BasicBlockType::AbsoluteEip => {
                        // Check if we can stay in this module, if not exit
                        codegen::gen_get_eip(ctx.builder);
                        let new_eip = ctx.builder.set_new_local();
                        codegen::gen_get_phys_eip_plus_mem(ctx, &new_eip);
                        ctx.builder.const_i32(unsafe { memory::mem8 } as i32);
                        ctx.builder.sub_i32();
                        ctx.builder.free_local(new_eip);

                        ctx.builder.const_i32(wasm_table_index.to_u16() as i32);
                        ctx.builder.const_i32(state_flags.to_u32() as i32);
                        ctx.builder.call_fn3_ret("jit_find_cache_entry_in_page");
                        ctx.builder.tee_local(target_block);
                        ctx.builder.const_i32(0);
                        ctx.builder.ge_i32();
                        // TODO: Could make this unconditional by including exit_label in the main br_table
                        ctx.builder.br_if(main_loop_label);

                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
                        ctx.builder.br(ctx.exit_label);
                    },
                    &BasicBlockType::Normal {
                        next_block_addr: None,
                        jump_offset,
                        jump_offset_is_32,
                    } => {
                        if jump_offset_is_32 {
                            codegen::gen_set_eip_low_bits_and_jump_rel32(
                                ctx.builder,
                                block.end_addr as i32 & 0xFFF,
                                jump_offset,
                            );
                        }
                        else {
                            codegen::gen_set_eip_low_bits(
                                ctx.builder,
                                block.end_addr as i32 & 0xFFF,
                            );
                            codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                        }

                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);
                        codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);
                        ctx.builder.br(ctx.exit_label);
                    },
                    &BasicBlockType::Normal {
                        next_block_addr: Some(next_block_addr),
                        jump_offset,
                        jump_offset_is_32,
                    } => {
                        // Unconditional jump to next basic block
                        // - All instructions that don't change eip
                        // - Unconditional jumps

                        if Page::page_of(next_block_addr) != Page::page_of(block.addr) {
                            if jump_offset_is_32 {
                                codegen::gen_set_eip_low_bits_and_jump_rel32(
                                    ctx.builder,
                                    block.end_addr as i32 & 0xFFF,
                                    jump_offset,
                                );
                            }
                            else {
                                codegen::gen_set_eip_low_bits(
                                    ctx.builder,
                                    block.end_addr as i32 & 0xFFF,
                                );
                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                            }

                            codegen::gen_profiler_stat_increment(
                                ctx.builder,
                                stat::NORMAL_PAGE_CHANGE,
                            );

                            codegen::gen_page_switch_check(
                                ctx,
                                next_block_addr,
                                block.last_instruction_addr,
                            );

                            #[cfg(debug_assertions)]
                            codegen::gen_fn2_const(
                                ctx.builder,
                                "check_page_switch",
                                block.addr,
                                next_block_addr,
                            );
                        }

                        if next_addr
                            .as_ref()
                            .map_or(false, |n| n.contains(&next_block_addr))
                        {
                            // Blocks are consecutive
                            if next_addr.unwrap().len() > 1 {
                                let target_index = *index_for_addr.get(&next_block_addr).unwrap();
                                if cfg!(feature = "profiler") {
                                    ctx.builder.const_i32(target_index);
                                    ctx.builder.call_fn1("debug_set_dispatcher_target");
                                }
                                ctx.builder.const_i32(target_index);
                                ctx.builder.set_local(target_block);
                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::NORMAL_FALLTHRU_WITH_TARGET_BLOCK,
                                );
                            }
                            else {
                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::NORMAL_FALLTHRU,
                                );
                            }
                        }
                        else {
                            let &(br, target_index) = label_for_addr.get(&next_block_addr).unwrap();
                            if let Some(target_index) = target_index {
                                if cfg!(feature = "profiler") {
                                    ctx.builder.const_i32(target_index);
                                    ctx.builder.call_fn1("debug_set_dispatcher_target");
                                }
                                ctx.builder.const_i32(target_index);
                                ctx.builder.set_local(target_block);
                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::NORMAL_BRANCH_WITH_TARGET_BLOCK,
                                );
                            }
                            else {
                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::NORMAL_BRANCH,
                                );
                            }
                            ctx.builder.br(br);
                        }
                    },
                    &BasicBlockType::ConditionalJump {
                        next_block_addr,
                        next_block_branch_taken_addr,
                        condition,
                        jump_offset,
                        jump_offset_is_32,
                    } => {
                        // Conditional jump to next basic block
                        // - jnz, jc, loop, jcxz, etc.

                        // Generate:
                        // (1) condition()
                        // (2) br_if()
                        // (3) br()
                        // Except:
                        // If we need to update eip in case (2), it's replaced by if { update_eip(); br() }
                        // If case (3) can fall through to the next basic block, the branch is eliminated
                        // Dispatcher target writes can be generated in either case
                        // Condition may be inverted if it helps generate a fallthrough instead of the second branch

                        codegen::gen_profiler_stat_increment(ctx.builder, stat::CONDITIONAL_JUMP);

                        #[derive(PartialEq)]
                        enum Case {
                            BranchTaken,
                            BranchNotTaken,
                        }

                        let mut handle_case = |case: Case, is_first| {
                            // first case generates condition and *has* to branch away,
                            // second case branches unconditionally or falls through

                            if is_first {
                                if case == Case::BranchNotTaken {
                                    codegen::gen_condition_fn_negated(ctx, condition);
                                }
                                else {
                                    codegen::gen_condition_fn(ctx, condition);
                                }
                            }

                            let next_block_addr = if case == Case::BranchTaken {
                                next_block_branch_taken_addr
                            }
                            else {
                                next_block_addr
                            };

                            if let Some(next_block_addr) = next_block_addr {
                                if Page::page_of(next_block_addr) != Page::page_of(block.addr) {
                                    dbg_assert!(case == Case::BranchTaken); // currently not possible in other case
                                    if is_first {
                                        ctx.builder.if_i32();
                                    }
                                    if jump_offset_is_32 {
                                        codegen::gen_set_eip_low_bits_and_jump_rel32(
                                            ctx.builder,
                                            block.end_addr as i32 & 0xFFF,
                                            jump_offset,
                                        );
                                    }
                                    else {
                                        codegen::gen_set_eip_low_bits(
                                            ctx.builder,
                                            block.end_addr as i32 & 0xFFF,
                                        );
                                        codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                                    }

                                    codegen::gen_profiler_stat_increment(
                                        ctx.builder,
                                        stat::CONDITIONAL_JUMP_PAGE_CHANGE,
                                    );
                                    codegen::gen_page_switch_check(
                                        ctx,
                                        next_block_addr,
                                        block.last_instruction_addr,
                                    );

                                    #[cfg(debug_assertions)]
                                    codegen::gen_fn2_const(
                                        ctx.builder,
                                        "check_page_switch",
                                        block.addr,
                                        next_block_addr,
                                    );

                                    if is_first {
                                        ctx.builder.const_i32(1);
                                        ctx.builder.else_();
                                        ctx.builder.const_i32(0);
                                        ctx.builder.block_end();
                                    }
                                }

                                if next_addr
                                    .as_ref()
                                    .map_or(false, |n| n.contains(&next_block_addr))
                                {
                                    // blocks are consecutive

                                    // fallthrough, has to be second
                                    dbg_assert!(!is_first);

                                    if next_addr.as_ref().unwrap().len() > 1 {
                                        let target_index =
                                            *index_for_addr.get(&next_block_addr).unwrap();
                                        if cfg!(feature = "profiler") {
                                            ctx.builder.const_i32(target_index);
                                            ctx.builder.call_fn1("debug_set_dispatcher_target");
                                        }
                                        ctx.builder.const_i32(target_index);
                                        ctx.builder.set_local(target_block);
                                        codegen::gen_profiler_stat_increment(
                                            ctx.builder,
                                            stat::CONDITIONAL_JUMP_FALLTHRU_WITH_TARGET_BLOCK,
                                        );
                                    }
                                    else {
                                        codegen::gen_profiler_stat_increment(
                                            ctx.builder,
                                            stat::CONDITIONAL_JUMP_FALLTHRU,
                                        );
                                    }
                                }
                                else {
                                    let &(br, target_index) =
                                        label_for_addr.get(&next_block_addr).unwrap();
                                    if let Some(target_index) = target_index {
                                        if cfg!(feature = "profiler") {
                                            // Note: Currently called unconditionally, even if the
                                            // br_if below doesn't branch
                                            ctx.builder.const_i32(target_index);
                                            ctx.builder.call_fn1("debug_set_dispatcher_target");
                                        }
                                        ctx.builder.const_i32(target_index);
                                        ctx.builder.set_local(target_block);
                                    }

                                    if is_first {
                                        if cfg!(feature = "profiler") {
                                            ctx.builder.if_void();
                                            codegen::gen_profiler_stat_increment(
                                                ctx.builder,
                                                if target_index.is_some() {
                                                    stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK
                                                }
                                                else {
                                                    stat::CONDITIONAL_JUMP_BRANCH
                                                },
                                            );
                                            ctx.builder.br(br);
                                            ctx.builder.block_end();
                                        }
                                        else {
                                            ctx.builder.br_if(br);
                                        }
                                    }
                                    else {
                                        codegen::gen_profiler_stat_increment(
                                            ctx.builder,
                                            if target_index.is_some() {
                                                stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK
                                            }
                                            else {
                                                stat::CONDITIONAL_JUMP_BRANCH
                                            },
                                        );
                                        ctx.builder.br(br);
                                    }
                                }
                            }
                            else {
                                // target is outside of this module, update eip and exit
                                if is_first {
                                    ctx.builder.if_void();
                                }

                                if case == Case::BranchTaken {
                                    if jump_offset_is_32 {
                                        codegen::gen_set_eip_low_bits_and_jump_rel32(
                                            ctx.builder,
                                            block.end_addr as i32 & 0xFFF,
                                            jump_offset,
                                        );
                                    }
                                    else {
                                        codegen::gen_set_eip_low_bits(
                                            ctx.builder,
                                            block.end_addr as i32 & 0xFFF,
                                        );
                                        codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                                    }
                                }
                                else {
                                    codegen::gen_set_eip_low_bits(
                                        ctx.builder,
                                        block.end_addr as i32 & 0xFFF,
                                    );
                                }

                                codegen::gen_debug_track_jit_exit(
                                    ctx.builder,
                                    block.last_instruction_addr,
                                );
                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::CONDITIONAL_JUMP_EXIT,
                                );
                                ctx.builder.br(ctx.exit_label);

                                if is_first {
                                    ctx.builder.block_end();
                                }
                            }
                        };

                        let branch_taken_is_fallthrough = next_block_branch_taken_addr
                            .map_or(false, |addr| {
                                next_addr.as_ref().map_or(false, |n| n.contains(&addr))
                            });
                        let branch_not_taken_is_fallthrough = next_block_addr
                            .map_or(false, |addr| {
                                next_addr.as_ref().map_or(false, |n| n.contains(&addr))
                            });

                        if branch_not_taken_is_fallthrough && branch_taken_is_fallthrough {
                            let next_block_addr = next_block_addr.unwrap();
                            let next_block_branch_taken_addr =
                                next_block_branch_taken_addr.unwrap();

                            dbg_assert!(
                                Page::page_of(next_block_addr) == Page::page_of(block.addr)
                            ); // currently not possible

                            if Page::page_of(next_block_branch_taken_addr)
                                != Page::page_of(block.addr)
                            {
                                if jump_offset_is_32 {
                                    codegen::gen_set_eip_low_bits_and_jump_rel32(
                                        ctx.builder,
                                        block.end_addr as i32 & 0xFFF,
                                        jump_offset,
                                    );
                                }
                                else {
                                    codegen::gen_set_eip_low_bits(
                                        ctx.builder,
                                        block.end_addr as i32 & 0xFFF,
                                    );
                                    codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);
                                }

                                codegen::gen_profiler_stat_increment(
                                    ctx.builder,
                                    stat::CONDITIONAL_JUMP_PAGE_CHANGE,
                                );
                                codegen::gen_page_switch_check(
                                    ctx,
                                    next_block_branch_taken_addr,
                                    block.last_instruction_addr,
                                );

                                #[cfg(debug_assertions)]
                                codegen::gen_fn2_const(
                                    ctx.builder,
                                    "check_page_switch",
                                    block.addr,
                                    next_block_branch_taken_addr,
                                );
                            }

                            if next_addr.unwrap().len() > 1 {
                                let target_index_taken =
                                    *index_for_addr.get(&next_block_branch_taken_addr).unwrap();
                                let target_index_not_taken =
                                    *index_for_addr.get(&next_block_addr).unwrap();

                                codegen::gen_condition_fn(ctx, condition);
                                ctx.builder.if_i32();
                                ctx.builder.const_i32(target_index_taken);
                                ctx.builder.else_();
                                ctx.builder.const_i32(target_index_not_taken);
                                ctx.builder.block_end();
                                ctx.builder.set_local(target_block);
                            }
                        }
                        else if branch_taken_is_fallthrough {
                            handle_case(Case::BranchNotTaken, true);
                            handle_case(Case::BranchTaken, false);
                        }
                        else {
                            handle_case(Case::BranchTaken, true);
                            handle_case(Case::BranchNotTaken, false);
                        }
                    },
                }
            },
            Work::WasmStructure(WasmStructure::Dispatcher(entries)) => {
                profiler::stat_increment(stat::COMPILE_DISPATCHER);

                if cfg!(feature = "profiler") {
                    ctx.builder.get_local(target_block);
                    ctx.builder.const_i32(index_for_addr.len() as i32);
                    ctx.builder.call_fn2("check_dispatcher_target");
                }

                if entries.len() > BRTABLE_CUTOFF {
                    // generate a brtable
                    codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_LARGE);
                    let mut cases = Vec::new();
                    for &addr in &entries {
                        let &(label, target_index) = label_for_addr.get(&addr).unwrap();
                        let &index = index_for_addr.get(&addr).unwrap();
                        dbg_assert!(target_index.is_none() || target_index == Some(index));
                        while index as usize >= cases.len() {
                            cases.push(brtable_default);
                        }
                        cases[index as usize] = label;
                    }
                    ctx.builder.get_local(target_block);
                    ctx.builder.brtable(brtable_default, &mut cases.iter());
                }
                else {
                    // generate a if target == block.addr then br block.label ...
                    codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_SMALL);
                    let nexts: HashSet<u32> = next_addr
                        .as_ref()
                        .map_or(HashSet::new(), |nexts| nexts.iter().copied().collect());
                    for &addr in &entries {
                        if nexts.contains(&addr) {
                            continue;
                        }
                        let index = *index_for_addr.get(&addr).unwrap();
                        let &(label, _) = label_for_addr.get(&addr).unwrap();
                        ctx.builder.get_local(target_block);
                        ctx.builder.const_i32(index);
                        ctx.builder.eq_i32();
                        ctx.builder.br_if(label);
                    }
                }
            },
            Work::WasmStructure(WasmStructure::Loop(children)) => {
                profiler::stat_increment(stat::COMPILE_WASM_LOOP);

                let entries: Vec<u32> = children[0].head().collect();
                let label = ctx.builder.loop_void();
                codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP);

                if entries.len() == 1 {
                    let addr = entries[0];
                    codegen::gen_set_eip_low_bits(ctx.builder, addr as i32 & 0xFFF);
                    profiler::stat_increment(stat::COMPILE_WITH_LOOP_SAFETY);
                    codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP_SAFETY);
                    if JIT_USE_LOOP_SAFETY {
                        ctx.builder.get_local(&ctx.instruction_counter);
                        ctx.builder.const_i32(cpu::LOOP_COUNTER);
                        ctx.builder.geu_i32();
                        if cfg!(feature = "profiler") {
                            ctx.builder.if_void();
                            codegen::gen_debug_track_jit_exit(ctx.builder, addr);
                            ctx.builder.br(exit_label);
                            ctx.builder.block_end();
                        }
                        else {
                            ctx.builder.br_if(exit_label);
                        }
                    }
                }

                let mut olds = HashMap::new();
                for &target in entries.iter() {
                    let index = if entries.len() == 1 {
                        None
                    }
                    else {
                        Some(*index_for_addr.get(&target).unwrap())
                    };
                    let old = label_for_addr.insert(target, (label, index));
                    if let Some(old) = old {
                        olds.insert(target, old);
                    }
                }

                work.push_front(Work::LoopEnd {
                    label,
                    entries,
                    olds,
                });
                for c in children.into_iter().rev() {
                    work.push_front(Work::WasmStructure(c));
                }
            },
            Work::LoopEnd {
                label,
                entries,
                olds,
            } => {
                for target in entries {
                    let old = label_for_addr.remove(&target);
                    dbg_assert!(old.map(|(l, _)| l) == Some(label));
                }
                for (target, old) in olds {
                    let old = label_for_addr.insert(target, old);
                    dbg_assert!(old.is_none());
                }

                ctx.builder.block_end();
            },
            Work::WasmStructure(WasmStructure::Block(children)) => {
                profiler::stat_increment(stat::COMPILE_WASM_BLOCK);

                let targets = next_addr.clone().unwrap();
                let label = ctx.builder.block_void();
                let mut olds = HashMap::new();
                for &target in targets.iter() {
                    let index = if targets.len() == 1 {
                        None
                    }
                    else {
                        Some(*index_for_addr.get(&target).unwrap())
                    };
                    let old = label_for_addr.insert(target, (label, index));
                    if let Some(old) = old {
                        olds.insert(target, old);
                    }
                }

                work.push_front(Work::BlockEnd {
                    label,
                    targets,
                    olds,
                });
                for c in children.into_iter().rev() {
                    work.push_front(Work::WasmStructure(c));
                }
            },
            Work::BlockEnd {
                label,
                targets,
                olds,
            } => {
                for target in targets {
                    let old = label_for_addr.remove(&target);
                    dbg_assert!(old.map(|(l, _)| l) == Some(label));
                }
                for (target, old) in olds {
                    let old = label_for_addr.insert(target, old);
                    dbg_assert!(old.is_none());
                }

                ctx.builder.block_end();
            },
        }
    }

    dbg_assert!(label_for_addr.is_empty());

    {
        ctx.builder.block_end(); // default case for the brtable
        ctx.builder.unreachable();
    }
    {
        ctx.builder.block_end(); // main loop
    }
    {
        // exit-with-fault case
        ctx.builder.block_end();
        codegen::gen_move_registers_from_locals_to_memory(ctx);
        codegen::gen_fn0_const(ctx.builder, "trigger_fault_end_jit");
        codegen::gen_update_instruction_counter(ctx);
        ctx.builder.return_();
    }
    {
        // exit
        ctx.builder.block_end();
        codegen::gen_move_registers_from_locals_to_memory(ctx);
        codegen::gen_update_instruction_counter(ctx);
    }

    for local in ctx.register_locals.drain(..) {
        ctx.builder.free_local(local);
    }
    ctx.builder
        .free_local(ctx.instruction_counter.unsafe_clone());

    ctx.builder.finish();

    let mut entries = Vec::new();

    for &addr in entry_blocks.iter() {
        let block = basic_blocks.get(&addr).unwrap();
        let index = *index_for_addr.get(&addr).unwrap();

        profiler::stat_increment(stat::COMPILE_ENTRY_POINT);

        dbg_assert!(block.addr < block.end_addr);
        // Note: We also insert blocks that weren't originally marked as entries here
        //       This doesn't have any downside, besides making the hash table slightly larger

        let initial_state = index.safe_to_u16();

        let entry = Entry {
            wasm_table_index,
            initial_state,
            state_flags,

            #[cfg(any(debug_assertions, feature = "profiler"))]
            len: block.end_addr - block.addr,

            #[cfg(debug_assertions)]
            opcode: memory::read32s(block.addr) as u32,
        };
        entries.push((block.addr, entry));
    }

    for b in basic_blocks.values() {
        if b.is_entry_block {
            dbg_assert!(entries.iter().find(|(addr, _)| *addr == b.addr).is_some());
        }
    }

    return entries;
}

fn jit_generate_basic_block(ctx: &mut JitContext, block: &BasicBlock) {
    let needs_eip_updated = match block.ty {
        BasicBlockType::Exit => true,
        _ => false,
    };

    profiler::stat_increment(stat::COMPILE_BASIC_BLOCK);

    let start_addr = block.addr;
    let last_instruction_addr = block.last_instruction_addr;
    let stop_addr = block.end_addr;

    // First iteration of do-while assumes the caller confirms this condition
    dbg_assert!(!is_near_end_of_page(start_addr));

    if cfg!(feature = "profiler") {
        ctx.builder.const_i32(start_addr as i32);
        ctx.builder.call_fn1("enter_basic_block");
    }

    ctx.builder.get_local(&ctx.instruction_counter);
    ctx.builder.const_i32(block.number_of_instructions as i32);
    ctx.builder.add_i32();
    ctx.builder.set_local(&ctx.instruction_counter);

    ctx.cpu.eip = start_addr;
    ctx.current_instruction = Instruction::Other;
    ctx.previous_instruction = Instruction::Other;

    loop {
        let mut instruction = 0;
        if cfg!(feature = "profiler") {
            instruction = memory::read32s(ctx.cpu.eip) as u32;
            opstats::gen_opstats(ctx.builder, instruction);
            opstats::record_opstat_compiled(instruction);
        }

        if ctx.cpu.eip == last_instruction_addr {
            // Before the last instruction:
            // - Set eip to *after* the instruction
            // - Set previous_eip to *before* the instruction
            if needs_eip_updated {
                codegen::gen_set_previous_eip_offset_from_eip_with_low_bits(
                    ctx.builder,
                    last_instruction_addr as i32 & 0xFFF,
                );
                codegen::gen_set_eip_low_bits(ctx.builder, stop_addr as i32 & 0xFFF);
            }
        }

        let wasm_length_before = ctx.builder.instruction_body_length();

        ctx.start_of_current_instruction = ctx.cpu.eip;
        let start_eip = ctx.cpu.eip;
        let mut instruction_flags = 0;
        jit_instructions::jit_instruction(ctx, &mut instruction_flags);
        let end_eip = ctx.cpu.eip;

        let instruction_length = end_eip - start_eip;
        let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;

        let wasm_length = ctx.builder.instruction_body_length() - wasm_length_before;
        opstats::record_opstat_size_wasm(instruction, wasm_length as u64);

        dbg_assert!((end_eip == stop_addr) == (start_eip == last_instruction_addr));
        dbg_assert!(instruction_length < MAX_INSTRUCTION_LENGTH);

        let end_addr = ctx.cpu.eip;

        if end_addr == stop_addr {
            // no page was crossed
            dbg_assert!(Page::page_of(end_addr) == Page::page_of(start_addr));
            break;
        }

        if was_block_boundary || is_near_end_of_page(end_addr) || end_addr > stop_addr {
            dbg_log!(
                "Overlapping basic blocks start={:x} expected_end={:x} end={:x} was_block_boundary={} near_end_of_page={}",
                start_addr,
                stop_addr,
                end_addr,
                was_block_boundary,
                is_near_end_of_page(end_addr)
            );
            dbg_assert!(false);
            break;
        }

        ctx.previous_instruction = mem::replace(&mut ctx.current_instruction, Instruction::Other);
    }
}

pub fn jit_increase_hotness_and_maybe_compile(
    virt_address: i32,
    phys_address: u32,
    cs_offset: u32,
    state_flags: CachedStateFlags,
    hotness: u32,
) {
    let ctx = get_jit_state();
    let page = Page::page_of(phys_address);
    let address_hash = jit_hot_hash_page(page) as usize;
    ctx.hot_pages[address_hash] += hotness;
    if ctx.hot_pages[address_hash] >= JIT_THRESHOLD {
        if ctx.compiling.is_some() {
            return;
        }
        // only try generating if we're in the correct address space
        if cpu::translate_address_read_no_side_effects(virt_address) == Ok(phys_address) {
            ctx.hot_pages[address_hash] = 0;
            jit_analyze_and_generate(ctx, virt_address, phys_address, cs_offset, state_flags)
        }
        else {
            profiler::stat_increment(stat::COMPILE_WRONG_ADDRESS_SPACE);
        }
    };
}

fn free_wasm_table_index(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {
    if CHECK_JIT_STATE_INVARIANTS {
        dbg_assert!(!ctx.wasm_table_index_free_list.contains(&wasm_table_index));

        match &ctx.compiling {
            Some((wasm_table_index_compiling, _)) => {
                dbg_assert!(
                    *wasm_table_index_compiling != wasm_table_index,
                    "Attempt to free wasm table index that is currently being compiled"
                );
            },
            _ => {},
        }
    }

    match ctx.used_wasm_table_indices.remove(&wasm_table_index) {
        None => {
            dbg_assert!(false);
        },
        Some(_pages) => {
            //dbg_assert!(!pages.is_empty()); // only if CompilingWritten
        },
    }
    ctx.wasm_table_index_free_list.push(wasm_table_index);

    dbg_assert!(
        ctx.wasm_table_index_free_list.len() + ctx.used_wasm_table_indices.len()
            == WASM_TABLE_SIZE as usize - 1
    );

    // It is not strictly necessary to clear the function, but it will fail more predictably if we
    // accidentally use the function and may garbage collect unused modules earlier
    jit_clear_func(wasm_table_index);

    rebuild_all_pages(ctx);

    check_jit_state_invariants(ctx);
}

pub fn rebuild_all_pages(ctx: &mut JitState) {
    // rebuild ctx.all_pages
    let mut all_pages = HashSet::new();
    for pages in ctx.used_wasm_table_indices.values() {
        all_pages.extend(pages);
    }
    ctx.all_pages = all_pages;
}

/// Register a write in this page: Delete all present code
pub fn jit_dirty_page(ctx: &mut JitState, page: Page) {
    let mut did_have_code = false;

    if ctx.all_pages.contains(&page) {
        profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_CODE);
        did_have_code = true;
        let mut index_to_free = HashSet::new();

        let compiling = match &ctx.compiling {
            Some((wasm_table_index, _)) => Some(*wasm_table_index),
            None => None,
        };

        for (&wasm_table_index, pages) in &ctx.used_wasm_table_indices {
            if Some(wasm_table_index) != compiling && pages.contains(&page) {
                index_to_free.insert(wasm_table_index);
            }
        }

        match &ctx.compiling {
            None => {},
            Some((_, PageState::CompilingWritten)) => {},
            Some((wasm_table_index, PageState::Compiling { .. })) => {
                let pages = ctx
                    .used_wasm_table_indices
                    .get_mut(wasm_table_index)
                    .unwrap();
                if pages.contains(&page) {
                    pages.clear();
                    ctx.compiling = Some((*wasm_table_index, PageState::CompilingWritten));
                    rebuild_all_pages(ctx);
                }
            },
        }

        for index in &index_to_free {
            match ctx.used_wasm_table_indices.get(&index) {
                None => {
                    dbg_assert!(false);
                },
                Some(pages) => {
                    for &p in pages {
                        for addr in p.address_range() {
                            if let Some(e) = ctx.cache.get(&addr) {
                                if index_to_free.contains(&e.wasm_table_index) {
                                    ctx.cache.remove(&addr);
                                }
                            }
                        }
                    }
                },
            }
        }

        for index in index_to_free {
            profiler::stat_increment(stat::INVALIDATE_MODULE_DIRTY_PAGE);
            free_wasm_table_index(ctx, index)
        }
    }

    match ctx.entry_points.remove(&page) {
        None => {},
        Some(_entry_points) => {
            profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_ENTRY_POINTS);
            did_have_code = true;

            // don't try to compile code in this page anymore until it's hot again
            ctx.hot_pages[jit_hot_hash_page(page) as usize] = 0;
        },
    }

    for pages in ctx.used_wasm_table_indices.values() {
        dbg_assert!(!pages.contains(&page));
    }

    check_jit_state_invariants(ctx);

    dbg_assert!(!ctx.all_pages.contains(&page));
    dbg_assert!(!jit_page_has_code_ctx(ctx, page));

    if did_have_code {
        cpu::tlb_set_has_code(page, false);
    }

    if !did_have_code {
        profiler::stat_increment(stat::DIRTY_PAGE_DID_NOT_HAVE_CODE);
    }
}

#[no_mangle]
pub fn jit_dirty_cache(start_addr: u32, end_addr: u32) {
    dbg_assert!(start_addr < end_addr);

    let start_page = Page::page_of(start_addr);
    let end_page = Page::page_of(end_addr - 1);

    for page in start_page.to_u32()..end_page.to_u32() + 1 {
        jit_dirty_page(get_jit_state(), Page::page_of(page << 12));
    }
}

/// dirty pages in the range of start_addr and end_addr, which must span at most two pages
pub fn jit_dirty_cache_small(start_addr: u32, end_addr: u32) {
    dbg_assert!(start_addr < end_addr);

    let start_page = Page::page_of(start_addr);
    let end_page = Page::page_of(end_addr - 1);

    let ctx = get_jit_state();
    jit_dirty_page(ctx, start_page);

    // Note: This can't happen when paging is enabled, as writes across
    //       boundaries are split up on two pages
    if start_page != end_page {
        dbg_assert!(start_page.to_u32() + 1 == end_page.to_u32());
        jit_dirty_page(ctx, end_page);
    }
}

#[no_mangle]
pub fn jit_clear_cache_js() { jit_clear_cache(get_jit_state()) }

pub fn jit_clear_cache(ctx: &mut JitState) {
    let mut pages_with_code = HashSet::new();

    for page in ctx.entry_points.keys() {
        pages_with_code.insert(*page);
    }
    for &p in &ctx.all_pages {
        pages_with_code.insert(p);
    }
    for addr in ctx.cache.keys() {
        dbg_assert!(pages_with_code.contains(&Page::page_of(*addr)));
    }
    for pages in ctx.used_wasm_table_indices.values() {
        dbg_assert!(pages_with_code.is_superset(pages));
    }

    for page in pages_with_code {
        jit_dirty_page(ctx, page);
    }
}

pub fn jit_page_has_code(page: Page) -> bool { jit_page_has_code_ctx(get_jit_state(), page) }

pub fn jit_page_has_code_ctx(ctx: &mut JitState, page: Page) -> bool {
    ctx.all_pages.contains(&page) || ctx.entry_points.contains_key(&page)
}

#[no_mangle]
pub fn jit_get_wasm_table_index_free_list_count() -> u32 {
    if cfg!(feature = "profiler") {
        get_jit_state().wasm_table_index_free_list.len() as u32
    }
    else {
        0
    }
}
#[no_mangle]
pub fn jit_get_cache_size() -> u32 {
    if cfg!(feature = "profiler") { get_jit_state().cache.len() as u32 } else { 0 }
}

#[cfg(feature = "profiler")]
pub fn check_missed_entry_points(phys_address: u32, state_flags: CachedStateFlags) {
    let ctx = get_jit_state();

    // backwards until beginning of page
    for offset in 0..=(phys_address & 0xFFF) {
        let addr = phys_address - offset;
        dbg_assert!(phys_address >= addr);

        if let Some(entry) = ctx.cache.get(&addr) {
            if entry.state_flags != state_flags || phys_address >= addr + entry.len {
                // give up search on first entry that is not a match
                break;
            }

            profiler::stat_increment(stat::RUN_INTERPRETED_MISSED_COMPILED_ENTRY_LOOKUP);

            let last_jump_type = unsafe { cpu::debug_last_jump.name() };
            let last_jump_addr = unsafe { cpu::debug_last_jump.phys_address() }.unwrap_or(0);
            let last_jump_opcode =
                if last_jump_addr != 0 { memory::read32s(last_jump_addr) } else { 0 };

            let opcode = memory::read32s(phys_address);
            dbg_log!(
                "Compiled exists, but no entry point, \
                 start={:x} end={:x} phys_addr={:x} opcode={:02x} {:02x} {:02x} {:02x}. \
                 Last jump at {:x} ({}) opcode={:02x} {:02x} {:02x} {:02x}",
                addr,
                addr + entry.len,
                phys_address,
                opcode & 0xFF,
                opcode >> 8 & 0xFF,
                opcode >> 16 & 0xFF,
                opcode >> 16 & 0xFF,
                last_jump_addr,
                last_jump_type,
                last_jump_opcode & 0xFF,
                last_jump_opcode >> 8 & 0xFF,
                last_jump_opcode >> 16 & 0xFF,
                last_jump_opcode >> 16 & 0xFF,
            );
        }
    }
}

#[no_mangle]
#[cfg(feature = "profiler")]
pub fn debug_set_dispatcher_target(_target_index: i32) {
    //dbg_log!("About to call dispatcher target_index={}", target_index);
}

#[no_mangle]
#[cfg(feature = "profiler")]
pub fn check_dispatcher_target(target_index: i32, max: i32) {
    //dbg_log!("Dispatcher called target={}", target_index);
    dbg_assert!(target_index >= 0);
    dbg_assert!(target_index < max);
}

#[no_mangle]
#[cfg(feature = "profiler")]
pub fn enter_basic_block(phys_eip: u32) {
    let eip =
        unsafe { cpu::translate_address_read(*global_pointers::instruction_pointer).unwrap() };
    if Page::page_of(eip) != Page::page_of(phys_eip) {
        dbg_log!(
            "enter basic block failed block=0x{:x} actual eip=0x{:x}",
            phys_eip,
            eip
        );
        panic!();
    }
}

#[no_mangle]
#[cfg(feature = "profiler")]
pub fn get_config(index: u32) -> u32 {
    match index {
        0 => MAX_PAGES as u32,
        1 => JIT_USE_LOOP_SAFETY as u32,
        2 => MAX_EXTRA_BASIC_BLOCKS as u32,
        _ => 0,
    }
}

<MSG> Clean up

<DFF> @@ -1136,18 +1136,6 @@ fn jit_generate_basic_block(
         jit_instructions::jit_instruction(&mut cpu, builder, &mut instruction_flags);
         let end_eip = cpu.eip;
 
-        #[cfg(debug_assertions)]
-        {
-            if ::config::CHECK_CPU_EXCEPTIONS {
-                // only the last instruction in each basic block is allowed to raise
-                if end_eip < stop_addr {
-                    let fn_idx =
-                        builder.get_fn_idx("assert_no_cpu_exception", module_init::FN0_TYPE_INDEX);
-                    builder.instruction_body.call_fn(fn_idx);
-                }
-            }
-        }
-
         let instruction_length = end_eip - start_eip;
         let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;
 

{'commit': 'diff --git a/src/rust/config.rs b/src/rust/config.rs\nindex 3c1b712c..8b137891 100644\n--- a/src/rust/config.rs\n+++ b/src/rust/config.rs\n@@ -1 +1 @@\n-pub const CHECK_CPU_EXCEPTIONS: bool = false;\n+\ndiff --git a/src/rust/cpu2/cpu.rs b/src/rust/cpu2/cpu.rs\nindex 8985ac44..238f79a0 100644\n--- a/src/rust/cpu2/cpu.rs\n+++ b/src/rust/cpu2/cpu.rs\n@@ -232,7 +232,6 @@ pub const TSC_RATE: f64 = (50i32 * 1000i32) as f64;\n pub static mut jit_block_boundary: bool = 0 != 0i32;\n \n pub static mut must_not_fault: bool = 0 != 0i32;\n-pub static mut current_cpu_exception: i32 = -1i32;\n pub static mut rdtsc_imprecision_offset: u64 = 0i32 as u64;\n pub static mut rdtsc_last_value: u64 = 0i32 as u64;\n pub static mut tsc_offset: u64 = 0i32 as u64;\n@@ -781,7 +780,6 @@ pub unsafe fn cycle_internal() -> () {\n                 (wasm_table_index as u32).wrapping_add(256i32 as u32) as i32,\n                 initial_state,\n             );\n-            clear_current_cpu_exception();\n             profiler_stat_increment_by(\n                 S_RUN_FROM_CACHE_STEPS,\n                 (*timestamp_counter).wrapping_sub(initial_tsc as u32) as i32,\n@@ -835,7 +833,6 @@ unsafe fn jit_run_interpreted(mut phys_addr: i32) -> () {\n     *instruction_pointer += 1;\n     *timestamp_counter = (*timestamp_counter).wrapping_add(1);\n     run_instruction(opcode | (*is_32 as i32) << 8i32);\n-    clear_current_cpu_exception();\n     while !jit_block_boundary && 0 != same_page(*previous_ip, *instruction_pointer) as i32 {\n         *previous_ip.offset(0isize) = *instruction_pointer.offset(0isize);\n         *timestamp_counter = (*timestamp_counter).wrapping_add(1);\n@@ -844,12 +841,9 @@ unsafe fn jit_run_interpreted(mut phys_addr: i32) -> () {\n             logop(*previous_ip.offset(0isize), opcode_0);\n         }\n         run_instruction(opcode_0 | (*is_32 as i32) << 8i32);\n-        clear_current_cpu_exception();\n     }\n }\n \n-pub unsafe fn clear_current_cpu_exception() -> () { current_cpu_exception = -1i32; }\n-\n #[no_mangle]\n pub unsafe fn pack_current_state_flags() -> cached_state_flags {\n     return ((*is_32 as i32) << 0i32\n@@ -914,11 +908,8 @@ pub unsafe fn trigger_de() -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_DE, 0 != 0i32, 0 != 0i32, 0i32);\n-    set_current_cpu_exception(CPU_EXCEPTION_DE);\n }\n \n-pub unsafe fn set_current_cpu_exception(mut n: i32) -> () { current_cpu_exception = n; }\n-\n #[no_mangle]\n pub unsafe fn trigger_ud() -> () {\n     dbg_log_c!("#ud");\n@@ -930,7 +921,6 @@ pub unsafe fn trigger_ud() -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_UD, 0 != 0i32, 0 != 0i32, 0i32);\n-    set_current_cpu_exception(CPU_EXCEPTION_UD);\n }\n \n pub unsafe fn trigger_nm() -> () {\n@@ -941,7 +931,6 @@ pub unsafe fn trigger_nm() -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_NM, 0 != 0i32, 0 != 0i32, 0i32);\n-    set_current_cpu_exception(CPU_EXCEPTION_NM);\n }\n \n #[no_mangle]\n@@ -953,7 +942,6 @@ pub unsafe fn trigger_gp_non_raising(mut code: i32) -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_GP, 0 != 0i32, 0 != 1i32, code);\n-    set_current_cpu_exception(CPU_EXCEPTION_GP);\n }\n \n pub unsafe fn virt_boundary_read16(mut low: i32, mut high: i32) -> i32 {\n@@ -1015,19 +1003,9 @@ pub unsafe fn virt_boundary_write32(mut low: i32, mut high: i32, mut value: i32)\n }\n \n pub unsafe fn safe_read8(mut addr: i32) -> Result<i32, ()> {\n-    assert_no_cpu_exception();\n     return Ok(read8(translate_address_read(addr)?));\n }\n \n-pub unsafe fn assert_no_cpu_exception() -> () {\n-    return;\n-    if current_cpu_exception != -1i32 {\n-        dbg_log_c!("Expected no cpu exception, got %d", current_cpu_exception);\n-        dbg_trace();\n-        dbg_assert!(0 != 0i32);\n-    };\n-}\n-\n pub unsafe fn safe_read16(mut address: i32) -> Result<i32, ()> {\n     let mut base: i32 = (address as u32 >> 12i32) as i32;\n     let mut entry: i32 = *tlb_data.offset(base as isize);\n@@ -1045,7 +1023,6 @@ pub unsafe fn safe_read16(mut address: i32) -> Result<i32, ()> {\n }\n \n pub unsafe fn safe_read16_slow(mut addr: i32) -> Result<i32, ()> {\n-    assert_no_cpu_exception();\n     if addr & 4095i32 == 4095i32 {\n         return Ok(safe_read8(addr)? | safe_read8(addr + 1i32)? << 8i32);\n     }\n@@ -1055,7 +1032,6 @@ pub unsafe fn safe_read16_slow(mut addr: i32) -> Result<i32, ()> {\n }\n \n pub unsafe fn safe_read32s(mut address: i32) -> Result<i32, ()> {\n-    assert_no_cpu_exception();\n     let mut base: i32 = (address as u32 >> 12i32) as i32;\n     let mut entry: i32 = *tlb_data.offset(base as isize);\n     let mut info_bits: i32 = entry & 4095i32 & !TLB_READONLY & !TLB_GLOBAL & !TLB_HAS_CODE;\n@@ -1122,7 +1098,6 @@ pub unsafe fn safe_read16_slow_jit(addr: i32) -> i32 {\n             v\n         },\n         Err(()) => {\n-            dbg_log!("page fault");\n             *page_fault = true;\n             -1\n         },\n@@ -1137,7 +1112,6 @@ pub unsafe fn safe_read32s_slow_jit(addr: i32) -> i32 {\n             v\n         },\n         Err(()) => {\n-            dbg_log!("page fault");\n             *page_fault = true;\n             -1\n         },\n@@ -1146,7 +1120,6 @@ pub unsafe fn safe_read32s_slow_jit(addr: i32) -> i32 {\n \n pub unsafe fn safe_read64s(mut addr: i32) -> Result<reg64, ()> {\n     let mut addr_phys: i32 = 0;\n-    assert_no_cpu_exception();\n     let mut x: reg64 = reg64 { i8_0: [0; 8] };\n     if addr & 4095i32 > 4096i32 - 8i32 {\n         x.u32_0[0usize] = safe_read32s(addr)? as u32;\n@@ -1161,7 +1134,6 @@ pub unsafe fn safe_read64s(mut addr: i32) -> Result<reg64, ()> {\n \n pub unsafe fn safe_read128s(mut addr: i32) -> Result<reg128, ()> {\n     let mut addr_phys: i32 = 0;\n-    assert_no_cpu_exception();\n     let mut x: reg128 = reg128 { i8_0: [0; 16] };\n     if addr & 4095i32 > 4096i32 - 16i32 {\n         x.u64_0[0usize] = safe_read64s(addr)?.u64_0[0usize];\n@@ -1175,7 +1147,6 @@ pub unsafe fn safe_read128s(mut addr: i32) -> Result<reg128, ()> {\n }\n \n pub unsafe fn safe_write8(mut addr: i32, mut value: i32) -> Result<(), ()> {\n-    assert_no_cpu_exception();\n     write8(translate_address_write(addr)?, value);\n     Ok(())\n }\n@@ -1201,7 +1172,6 @@ pub unsafe fn safe_write16(mut address: i32, mut value: i32) -> Result<(), ()> {\n }\n \n pub unsafe fn safe_write16_slow(mut addr: i32, mut value: i32) -> Result<(), ()> {\n-    assert_no_cpu_exception();\n     let mut phys_low: i32 = translate_address_write(addr)? as i32;\n     if addr & 4095i32 == 4095i32 {\n         virt_boundary_write16(\n@@ -1217,7 +1187,6 @@ pub unsafe fn safe_write16_slow(mut addr: i32, mut value: i32) -> Result<(), ()>\n }\n \n pub unsafe fn safe_write32(mut address: i32, mut value: i32) -> Result<(), ()> {\n-    assert_no_cpu_exception();\n     let mut base: i32 = (address as u32 >> 12i32) as i32;\n     let mut entry: i32 = *tlb_data.offset(base as isize);\n     let mut info_bits: i32 = entry & 4095i32 & !TLB_GLOBAL & !if *cpl as i32 == 3i32 {\n@@ -1307,7 +1276,6 @@ pub unsafe fn safe_write32_slow_jit(addr: i32, value: i32) {\n }\n \n pub unsafe fn safe_write64(mut addr: i32, mut value: i64) -> Result<(), ()> {\n-    assert_no_cpu_exception();\n     if addr & 4095i32 > 4096i32 - 8i32 {\n         writable_or_pagefault(addr, 8i32)?;\n         safe_write32(addr, value as i32).unwrap();\n@@ -1321,7 +1289,6 @@ pub unsafe fn safe_write64(mut addr: i32, mut value: i64) -> Result<(), ()> {\n }\n \n pub unsafe fn safe_write128(mut addr: i32, mut value: reg128) -> Result<(), ()> {\n-    assert_no_cpu_exception();\n     if addr & 4095i32 > 4096i32 - 16i32 {\n         writable_or_pagefault(addr, 16i32)?;\n         safe_write64(addr, value.u64_0[0usize] as i64).unwrap();\n@@ -1700,7 +1667,6 @@ pub unsafe fn trigger_np(mut code: i32) -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_NP, 0 != 0i32, 0 != 1i32, code);\n-    set_current_cpu_exception(CPU_EXCEPTION_NP);\n }\n \n #[no_mangle]\n@@ -1712,7 +1678,6 @@ pub unsafe fn trigger_ss(mut code: i32) -> () {\n     }\n     *instruction_pointer = *previous_ip;\n     call_interrupt_vector(CPU_EXCEPTION_SS, 0 != 0i32, 0 != 1i32, code);\n-    set_current_cpu_exception(CPU_EXCEPTION_SS);\n }\n \n #[no_mangle]\ndiff --git a/src/rust/jit.rs b/src/rust/jit.rs\nindex e5c8a4e5..78d30f62 100644\n--- a/src/rust/jit.rs\n+++ b/src/rust/jit.rs\n@@ -1136,18 +1136,6 @@ fn jit_generate_basic_block(\n         jit_instructions::jit_instruction(&mut cpu, builder, &mut instruction_flags);\n         let end_eip = cpu.eip;\n \n-        #[cfg(debug_assertions)]\n-        {\n-            if ::config::CHECK_CPU_EXCEPTIONS {\n-                // only the last instruction in each basic block is allowed to raise\n-                if end_eip < stop_addr {\n-                    let fn_idx =\n-                        builder.get_fn_idx("assert_no_cpu_exception", module_init::FN0_TYPE_INDEX);\n-                    builder.instruction_body.call_fn(fn_idx);\n-                }\n-            }\n-        }\n-\n         let instruction_length = end_eip - start_eip;\n         let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;\n \n', 'commit_subject': 'Clean up', 'repo_name': 'copy/v86', 'committer': {'name': 'Fabian', 'email': 'f84e2e2dadd87384fb55f25886926b777e8378f1@copy.sh', 'time_sec': '1598833794', 'tz_offset': '-300', 'date': {'seconds': '1598833794'}}, 'commit_tree': 'd418df6fad8a5fd23003434bed83d9f8c14c72de', 'commit_message': 'Clean up\n', 'language_name': 'JavaScript', 'license': 'bsd-2-clause', 'hunks': '@@ -1136,18 +1136,6 @@ fn jit_generate_basic_block(\n         jit_instructions::jit_instruction(&mut cpu, builder, &mut instruction_flags);\n         let end_eip = cpu.eip;\n \n-        #[cfg(debug_assertions)]\n-        {\n-            if ::config::CHECK_CPU_EXCEPTIONS {\n-                // only the last instruction in each basic block is allowed to raise\n-                if end_eip < stop_addr {\n-                    let fn_idx =\n-                        builder.get_fn_idx("assert_no_cpu_exception", module_init::FN0_TYPE_INDEX);\n-                    builder.instruction_body.call_fn(fn_idx);\n-                }\n-            }\n-        }\n-\n         let instruction_length = end_eip - start_eip;\n         let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;\n \n', 'addition_count': 0, 'deletion_count': 12, 'src_file': 'a/src/rust/jit.rs', 'tgt_file': 'b/src/rust/jit.rs', 'file_extension': '.rs', 'before_file': 'use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};\nuse std::iter::FromIterator;\nuse std::mem;\nuse std::ptr::NonNull;\n\nuse analysis::AnalysisType;\nuse codegen;\nuse control_flow;\nuse control_flow::WasmStructure;\nuse cpu::cpu;\nuse cpu::global_pointers;\nuse cpu::memory;\nuse cpu_context::CpuContext;\nuse jit_instructions;\nuse opstats;\nuse page::Page;\nuse profiler;\nuse profiler::stat;\nuse state_flags::CachedStateFlags;\nuse util::SafeToU16;\nuse wasmgen::wasm_builder::{Label, WasmBuilder, WasmLocal};\n\n#[derive(Copy, Clone, Eq, Hash, PartialEq)]\n#[repr(transparent)]\npub struct WasmTableIndex(u16);\nimpl WasmTableIndex {\n    pub fn to_u16(self) -> u16 { self.0 }\n}\n\nmod unsafe_jit {\n    use jit::{CachedStateFlags, WasmTableIndex};\n\n    extern "C" {\n        pub fn codegen_finalize(\n            wasm_table_index: WasmTableIndex,\n            phys_addr: u32,\n            state_flags: CachedStateFlags,\n            ptr: u32,\n            len: u32,\n        );\n        pub fn jit_clear_func(wasm_table_index: WasmTableIndex);\n    }\n}\n\nfn codegen_finalize(\n    wasm_table_index: WasmTableIndex,\n    phys_addr: u32,\n    state_flags: CachedStateFlags,\n    ptr: u32,\n    len: u32,\n) {\n    unsafe { unsafe_jit::codegen_finalize(wasm_table_index, phys_addr, state_flags, ptr, len) }\n}\n\npub fn jit_clear_func(wasm_table_index: WasmTableIndex) {\n    unsafe { unsafe_jit::jit_clear_func(wasm_table_index) }\n}\n\n// less branches will generate if-else, more will generate brtable\npub const BRTABLE_CUTOFF: usize = 10;\n\npub const WASM_TABLE_SIZE: u32 = 900;\n\npub const HASH_PRIME: u32 = 6151;\n\npub const CHECK_JIT_STATE_INVARIANTS: bool = false;\n\npub const JIT_USE_LOOP_SAFETY: bool = true;\n\npub const JIT_THRESHOLD: u32 = 200 * 1000;\n\npub const MAX_EXTRA_BASIC_BLOCKS: usize = 250;\n\nconst MAX_INSTRUCTION_LENGTH: u32 = 16;\n\n#[allow(non_upper_case_globals)]\nstatic mut jit_state: NonNull<JitState> =\n    unsafe { NonNull::new_unchecked(mem::align_of::<JitState>() as *mut _) };\n\npub fn get_jit_state() -> &\'static mut JitState { unsafe { jit_state.as_mut() } }\n\n#[no_mangle]\npub fn rust_init() {\n    let x = Box::new(JitState::create_and_initialise());\n    unsafe {\n        jit_state = NonNull::new(Box::into_raw(x)).unwrap()\n    }\n\n    use std::panic;\n\n    panic::set_hook(Box::new(|panic_info| {\n        console_log!("{}", panic_info.to_string());\n    }));\n}\n\npub struct Entry {\n    #[cfg(any(debug_assertions, feature = "profiler"))]\n    pub len: u32,\n\n    #[cfg(debug_assertions)]\n    pub opcode: u32,\n\n    pub initial_state: u16,\n    pub wasm_table_index: WasmTableIndex,\n    pub state_flags: CachedStateFlags,\n}\n\nenum PageState {\n    Compiling { entries: Vec<(u32, Entry)> },\n    CompilingWritten,\n}\n\npub struct JitState {\n    wasm_builder: WasmBuilder,\n\n    // as an alternative to HashSet, we could use a bitmap of 4096 bits here\n    // (faster, but uses much more memory)\n    // or a compressed bitmap (likely faster)\n    // or HashSet<u32> rather than nested\n    entry_points: HashMap<Page, HashSet<u16>>,\n    hot_pages: [u32; HASH_PRIME as usize],\n\n    wasm_table_index_free_list: Vec<WasmTableIndex>,\n    used_wasm_table_indices: HashMap<WasmTableIndex, HashSet<Page>>,\n    // All pages from used_wasm_table_indices\n    // Used to improve the performance of jit_dirty_page and jit_page_has_code\n    all_pages: HashSet<Page>,\n    cache: HashMap<u32, Entry>,\n    compiling: Option<(WasmTableIndex, PageState)>,\n}\n\npub fn check_jit_state_invariants(ctx: &mut JitState) {\n    if !CHECK_JIT_STATE_INVARIANTS {\n        return;\n    }\n    let mut all_pages = HashSet::new();\n    for pages in ctx.used_wasm_table_indices.values() {\n        all_pages.extend(pages);\n    }\n    dbg_assert!(ctx.all_pages == all_pages);\n}\n\nimpl JitState {\n    pub fn create_and_initialise() -> JitState {\n        // don\'t assign 0 (XXX: Check)\n        let wasm_table_indices = (1..=(WASM_TABLE_SIZE - 1) as u16).map(|x| WasmTableIndex(x));\n\n        JitState {\n            wasm_builder: WasmBuilder::new(),\n\n            entry_points: HashMap::new(),\n            hot_pages: [0; HASH_PRIME as usize],\n\n            wasm_table_index_free_list: Vec::from_iter(wasm_table_indices),\n            used_wasm_table_indices: HashMap::new(),\n            all_pages: HashSet::new(),\n            cache: HashMap::new(),\n            compiling: None,\n        }\n    }\n}\n\n#[derive(PartialEq, Eq)]\npub enum BasicBlockType {\n    Normal {\n        next_block_addr: Option<u32>,\n        jump_offset: i32,\n        jump_offset_is_32: bool,\n    },\n    ConditionalJump {\n        next_block_addr: Option<u32>,\n        next_block_branch_taken_addr: Option<u32>,\n        condition: u8,\n        jump_offset: i32,\n        jump_offset_is_32: bool,\n    },\n    // Set eip to an absolute value (ret, jmp r/m, call r/m)\n    AbsoluteEip,\n    Exit,\n}\n\npub struct BasicBlock {\n    pub addr: u32,\n    pub virt_addr: i32,\n    pub last_instruction_addr: u32,\n    pub end_addr: u32,\n    pub is_entry_block: bool,\n    pub ty: BasicBlockType,\n    pub has_sti: bool,\n    pub number_of_instructions: u32,\n}\n\n#[derive(Copy, Clone, PartialEq)]\npub struct CachedCode {\n    pub wasm_table_index: WasmTableIndex,\n    pub initial_state: u16,\n}\n\nimpl CachedCode {\n    pub const NONE: CachedCode = CachedCode {\n        wasm_table_index: WasmTableIndex(0),\n        initial_state: 0,\n    };\n}\n\n#[derive(PartialEq)]\npub enum InstructionOperand {\n    WasmLocal(WasmLocal),\n    Immediate(i32),\n    Other,\n}\npub enum Instruction {\n    Cmp {\n        dest: InstructionOperand,\n        source: InstructionOperand,\n        opsize: i32,\n    },\n    Sub {\n        opsize: i32,\n    },\n    // Any instruction that sets last_result\n    Arithmetic {\n        opsize: i32,\n    },\n    Other,\n}\n\npub struct JitContext<\'a> {\n    pub cpu: &\'a mut CpuContext,\n    pub builder: &\'a mut WasmBuilder,\n    pub register_locals: &\'a mut Vec<WasmLocal>,\n    pub start_of_current_instruction: u32,\n    pub exit_with_fault_label: Label,\n    pub exit_label: Label,\n    pub current_instruction: Instruction,\n    pub previous_instruction: Instruction,\n    pub instruction_counter: WasmLocal,\n}\nimpl<\'a> JitContext<\'a> {\n    pub fn reg(&self, i: u32) -> WasmLocal { self.register_locals[i as usize].unsafe_clone() }\n}\n\npub const JIT_INSTR_BLOCK_BOUNDARY_FLAG: u32 = 1 << 0;\n\nfn jit_hot_hash_page(page: Page) -> u32 { page.to_u32() % HASH_PRIME }\n\npub fn is_near_end_of_page(address: u32) -> bool {\n    address & 0xFFF >= 0x1000 - MAX_INSTRUCTION_LENGTH\n}\n\npub fn jit_find_cache_entry(phys_address: u32, state_flags: CachedStateFlags) -> CachedCode {\n    if is_near_end_of_page(phys_address) {\n        profiler::stat_increment(stat::RUN_INTERPRETED_NEAR_END_OF_PAGE);\n    }\n\n    let ctx = get_jit_state();\n\n    match ctx.cache.get(&phys_address) {\n        Some(entry) => {\n            if entry.state_flags == state_flags {\n                return CachedCode {\n                    wasm_table_index: entry.wasm_table_index,\n                    initial_state: entry.initial_state,\n                };\n            }\n            else {\n                profiler::stat_increment(stat::RUN_INTERPRETED_DIFFERENT_STATE);\n            }\n        },\n        None => {},\n    }\n\n    return CachedCode::NONE;\n}\n\n#[no_mangle]\npub fn jit_find_cache_entry_in_page(\n    phys_address: u32,\n    wasm_table_index: WasmTableIndex,\n    state_flags: u32,\n) -> i32 {\n    profiler::stat_increment(stat::INDIRECT_JUMP);\n\n    let state_flags = CachedStateFlags::of_u32(state_flags);\n    let ctx = get_jit_state();\n\n    match ctx.cache.get(&phys_address) {\n        Some(entry) => {\n            if entry.state_flags == state_flags && entry.wasm_table_index == wasm_table_index {\n                return entry.initial_state as i32;\n            }\n        },\n        None => {},\n    }\n\n    profiler::stat_increment(stat::INDIRECT_JUMP_NO_ENTRY);\n\n    return -1;\n}\n\npub fn record_entry_point(phys_address: u32) {\n    let ctx = get_jit_state();\n    if is_near_end_of_page(phys_address) {\n        return;\n    }\n    let page = Page::page_of(phys_address);\n    let offset_in_page = phys_address as u16 & 0xFFF;\n    let mut is_new = false;\n    ctx.entry_points\n        .entry(page)\n        .or_insert_with(|| {\n            is_new = true;\n            HashSet::new()\n        })\n        .insert(offset_in_page);\n\n    if is_new {\n        cpu::tlb_set_has_code(page, true);\n    }\n}\n\n// Maximum number of pages per wasm module. Necessary for the following reasons:\n// - There is an upper limit on the size of a single function in wasm (currently ~7MB in all browsers)\n//   See https://github.com/WebAssembly/design/issues/1138\n// - v8 poorly handles large br_table elements and OOMs on modules much smaller than the above limit\n//   See https://bugs.chromium.org/p/v8/issues/detail?id=9697 and https://bugs.chromium.org/p/v8/issues/detail?id=9141\n//   Will hopefully be fixed in the near future by generating direct control flow\nconst MAX_PAGES: usize = 3;\n\nfn jit_find_basic_blocks(\n    ctx: &mut JitState,\n    entry_points: HashSet<i32>,\n    cpu: CpuContext,\n) -> Vec<BasicBlock> {\n    fn follow_jump(\n        virt_target: i32,\n        ctx: &mut JitState,\n        pages: &mut HashSet<Page>,\n        page_blacklist: &mut HashSet<Page>,\n        max_pages: usize,\n        marked_as_entry: &mut HashSet<i32>,\n        to_visit_stack: &mut Vec<i32>,\n    ) -> Option<u32> {\n        if is_near_end_of_page(virt_target as u32) {\n            return None;\n        }\n        let phys_target = match cpu::translate_address_read_no_side_effects(virt_target) {\n            Err(()) => {\n                dbg_log!("Not analysing {:x} (page not mapped)", virt_target);\n                return None;\n            },\n            Ok(t) => t,\n        };\n\n        let phys_page = Page::page_of(phys_target);\n\n        if !pages.contains(&phys_page) && pages.len() == max_pages\n            || page_blacklist.contains(&phys_page)\n        {\n            return None;\n        }\n\n        if !pages.contains(&phys_page) {\n            // page seen for the first time, handle entry points\n            if let Some(entry_points) = ctx.entry_points.get(&phys_page) {\n                if entry_points.iter().all(|&entry_point| {\n                    ctx.cache\n                        .contains_key(&(phys_page.to_address() | u32::from(entry_point)))\n                }) {\n                    profiler::stat_increment(stat::COMPILE_PAGE_SKIPPED_NO_NEW_ENTRY_POINTS);\n                    page_blacklist.insert(phys_page);\n                    return None;\n                }\n\n                let address_hash = jit_hot_hash_page(phys_page) as usize;\n                ctx.hot_pages[address_hash] = 0;\n\n                for &addr_low in entry_points {\n                    let addr = virt_target & !0xFFF | addr_low as i32;\n                    to_visit_stack.push(addr);\n                    marked_as_entry.insert(addr);\n                }\n            }\n            else {\n                // no entry points: ignore this page?\n            }\n\n            pages.insert(phys_page);\n            dbg_assert!(pages.len() <= max_pages);\n        }\n\n        to_visit_stack.push(virt_target);\n        Some(phys_target)\n    }\n\n    let mut to_visit_stack: Vec<i32> = Vec::new();\n    let mut marked_as_entry: HashSet<i32> = HashSet::new();\n    let mut basic_blocks: BTreeMap<u32, BasicBlock> = BTreeMap::new();\n    let mut pages: HashSet<Page> = HashSet::new();\n    let mut page_blacklist = HashSet::new();\n\n    // 16-bit doesn\'t not work correctly, most likely due to instruction pointer wrap-around\n    let max_pages = if cpu.state_flags.is_32() { MAX_PAGES } else { 1 };\n\n    for virt_addr in entry_points {\n        let ok = follow_jump(\n            virt_addr,\n            ctx,\n            &mut pages,\n            &mut page_blacklist,\n            max_pages,\n            &mut marked_as_entry,\n            &mut to_visit_stack,\n        );\n        dbg_assert!(ok.is_some());\n        dbg_assert!(marked_as_entry.contains(&virt_addr));\n    }\n\n    while let Some(to_visit) = to_visit_stack.pop() {\n        let phys_addr = match cpu::translate_address_read_no_side_effects(to_visit) {\n            Err(()) => {\n                dbg_log!("Not analysing {:x} (page not mapped)", to_visit);\n                continue;\n            },\n            Ok(phys_addr) => phys_addr,\n        };\n\n        if basic_blocks.contains_key(&phys_addr) {\n            continue;\n        }\n\n        if is_near_end_of_page(phys_addr) {\n            // Empty basic block, don\'t insert\n            profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);\n            continue;\n        }\n\n        let mut current_address = phys_addr;\n        let mut current_block = BasicBlock {\n            addr: current_address,\n            virt_addr: to_visit,\n            last_instruction_addr: 0,\n            end_addr: 0,\n            ty: BasicBlockType::Exit,\n            is_entry_block: false,\n            has_sti: false,\n            number_of_instructions: 0,\n        };\n        loop {\n            let addr_before_instruction = current_address;\n            let mut cpu = &mut CpuContext {\n                eip: current_address,\n                ..cpu\n            };\n            let analysis = ::analysis::analyze_step(&mut cpu);\n            current_block.number_of_instructions += 1;\n            let has_next_instruction = !analysis.no_next_instruction;\n            current_address = cpu.eip;\n\n            dbg_assert!(Page::page_of(current_address) == Page::page_of(addr_before_instruction));\n            let current_virt_addr = to_visit & !0xFFF | current_address as i32 & 0xFFF;\n\n            match analysis.ty {\n                AnalysisType::Normal | AnalysisType::STI => {\n                    dbg_assert!(has_next_instruction);\n                    dbg_assert!(!analysis.absolute_jump);\n\n                    if current_block.has_sti {\n                        // Convert next instruction after STI (i.e., the current instruction) into block boundary\n\n                        marked_as_entry.insert(current_virt_addr);\n                        to_visit_stack.push(current_virt_addr);\n\n                        current_block.last_instruction_addr = addr_before_instruction;\n                        current_block.end_addr = current_address;\n                        break;\n                    }\n\n                    if analysis.ty == AnalysisType::STI {\n                        current_block.has_sti = true;\n\n                        dbg_assert!(\n                            !is_near_end_of_page(current_address),\n                            "TODO: Handle STI instruction near end of page"\n                        );\n                    }\n                    else {\n                        // Only split non-STI blocks (one instruction needs to run after STI before\n                        // handle_irqs may be called)\n\n                        if basic_blocks.contains_key(&current_address) {\n                            current_block.last_instruction_addr = addr_before_instruction;\n                            current_block.end_addr = current_address;\n                            dbg_assert!(!is_near_end_of_page(current_address));\n                            current_block.ty = BasicBlockType::Normal {\n                                next_block_addr: Some(current_address),\n                                jump_offset: 0,\n                                jump_offset_is_32: true,\n                            };\n                            break;\n                        }\n                    }\n                },\n                AnalysisType::Jump {\n                    offset,\n                    is_32,\n                    condition: Some(condition),\n                } => {\n                    dbg_assert!(!analysis.absolute_jump);\n                    // conditional jump: continue at next and continue at jump target\n\n                    let jump_target = if is_32 {\n                        current_virt_addr + offset\n                    }\n                    else {\n                        cpu.cs_offset as i32\n                            + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)\n                    };\n\n                    dbg_assert!(has_next_instruction);\n                    to_visit_stack.push(current_virt_addr);\n\n                    let next_block_addr = if is_near_end_of_page(current_address) {\n                        None\n                    }\n                    else {\n                        Some(current_address)\n                    };\n\n                    current_block.ty = BasicBlockType::ConditionalJump {\n                        next_block_addr,\n                        next_block_branch_taken_addr: follow_jump(\n                            jump_target,\n                            ctx,\n                            &mut pages,\n                            &mut page_blacklist,\n                            max_pages,\n                            &mut marked_as_entry,\n                            &mut to_visit_stack,\n                        ),\n                        condition,\n                        jump_offset: offset,\n                        jump_offset_is_32: is_32,\n                    };\n\n                    current_block.last_instruction_addr = addr_before_instruction;\n                    current_block.end_addr = current_address;\n\n                    break;\n                },\n                AnalysisType::Jump {\n                    offset,\n                    is_32,\n                    condition: None,\n                } => {\n                    dbg_assert!(!analysis.absolute_jump);\n                    // non-conditional jump: continue at jump target\n\n                    let jump_target = if is_32 {\n                        current_virt_addr + offset\n                    }\n                    else {\n                        cpu.cs_offset as i32\n                            + (current_virt_addr - cpu.cs_offset as i32 + offset & 0xFFFF)\n                    };\n\n                    if has_next_instruction {\n                        // Execution will eventually come back to the next instruction (CALL)\n                        marked_as_entry.insert(current_virt_addr);\n                        to_visit_stack.push(current_virt_addr);\n                    }\n\n                    current_block.ty = BasicBlockType::Normal {\n                        next_block_addr: follow_jump(\n                            jump_target,\n                            ctx,\n                            &mut pages,\n                            &mut page_blacklist,\n                            max_pages,\n                            &mut marked_as_entry,\n                            &mut to_visit_stack,\n                        ),\n                        jump_offset: offset,\n                        jump_offset_is_32: is_32,\n                    };\n                    current_block.last_instruction_addr = addr_before_instruction;\n                    current_block.end_addr = current_address;\n\n                    break;\n                },\n                AnalysisType::BlockBoundary => {\n                    // a block boundary but not a jump, get out\n\n                    if has_next_instruction {\n                        // block boundary, but execution will eventually come back\n                        // to the next instruction. Create a new basic block\n                        // starting at the next instruction and register it as an\n                        // entry point\n                        marked_as_entry.insert(current_virt_addr);\n                        to_visit_stack.push(current_virt_addr);\n                    }\n\n                    if analysis.absolute_jump {\n                        current_block.ty = BasicBlockType::AbsoluteEip;\n                    }\n\n                    current_block.last_instruction_addr = addr_before_instruction;\n                    current_block.end_addr = current_address;\n                    break;\n                },\n            }\n\n            if is_near_end_of_page(current_address) {\n                current_block.last_instruction_addr = addr_before_instruction;\n                current_block.end_addr = current_address;\n                profiler::stat_increment(stat::COMPILE_CUT_OFF_AT_END_OF_PAGE);\n                break;\n            }\n        }\n\n        let previous_block = basic_blocks\n            .range(..current_block.addr)\n            .next_back()\n            .filter(|(_, previous_block)| (!previous_block.has_sti))\n            .map(|(_, previous_block)| previous_block.clone());\n\n        if let Some(previous_block) = previous_block {\n            if current_block.addr < previous_block.end_addr {\n                // If this block overlaps with the previous block, re-analyze the previous block\n                to_visit_stack.push(previous_block.virt_addr);\n\n                let addr = previous_block.addr;\n                let old_block = basic_blocks.remove(&addr);\n                dbg_assert!(old_block.is_some());\n\n                // Note that this does not ensure the invariant that two consecutive blocks don\'t\n                // overlay. For that, we also need to check the following block.\n            }\n        }\n\n        dbg_assert!(current_block.addr < current_block.end_addr);\n        dbg_assert!(current_block.addr <= current_block.last_instruction_addr);\n        dbg_assert!(current_block.last_instruction_addr < current_block.end_addr);\n\n        basic_blocks.insert(current_block.addr, current_block);\n    }\n\n    dbg_assert!(pages.len() <= max_pages);\n\n    for block in basic_blocks.values_mut() {\n        if marked_as_entry.contains(&block.virt_addr) {\n            block.is_entry_block = true;\n        }\n    }\n\n    let basic_blocks: Vec<BasicBlock> = basic_blocks.into_iter().map(|(_, block)| block).collect();\n\n    for i in 0..basic_blocks.len() - 1 {\n        let next_block_addr = basic_blocks[i + 1].addr;\n        let next_block_end_addr = basic_blocks[i + 1].end_addr;\n        let next_block_is_entry = basic_blocks[i + 1].is_entry_block;\n        let block = &basic_blocks[i];\n        dbg_assert!(block.addr < next_block_addr);\n        if next_block_addr < block.end_addr {\n            dbg_log!(\n                "Overlapping first=[from={:x} to={:x} is_entry={}] second=[from={:x} to={:x} is_entry={}]",\n                block.addr,\n                block.end_addr,\n                block.is_entry_block as u8,\n                next_block_addr,\n                next_block_end_addr,\n                next_block_is_entry as u8\n            );\n        }\n    }\n\n    basic_blocks\n}\n\n#[no_mangle]\n#[cfg(debug_assertions)]\npub fn jit_force_generate_unsafe(virt_addr: i32) {\n    let ctx = get_jit_state();\n    let phys_addr = cpu::translate_address_read(virt_addr).unwrap();\n    record_entry_point(phys_addr);\n    let cs_offset = cpu::get_seg_cs() as u32;\n    let state_flags = cpu::pack_current_state_flags();\n    jit_analyze_and_generate(ctx, virt_addr, phys_addr, cs_offset, state_flags);\n}\n\n#[inline(never)]\nfn jit_analyze_and_generate(\n    ctx: &mut JitState,\n    virt_entry_point: i32,\n    phys_entry_point: u32,\n    cs_offset: u32,\n    state_flags: CachedStateFlags,\n) {\n    let page = Page::page_of(phys_entry_point);\n\n    if ctx.compiling.is_some() {\n        return;\n    }\n\n    let entry_points = ctx.entry_points.get(&page);\n\n    let entry_points = match entry_points {\n        None => return,\n        Some(entry_points) => entry_points,\n    };\n\n    if entry_points.iter().all(|&entry_point| {\n        ctx.cache\n            .contains_key(&(page.to_address() | u32::from(entry_point)))\n    }) {\n        profiler::stat_increment(stat::COMPILE_SKIPPED_NO_NEW_ENTRY_POINTS);\n        return;\n    }\n\n    profiler::stat_increment(stat::COMPILE);\n\n    let cpu = CpuContext {\n        eip: 0,\n        prefixes: 0,\n        cs_offset,\n        state_flags,\n    };\n\n    dbg_assert!(\n        cpu::translate_address_read_no_side_effects(virt_entry_point).unwrap() == phys_entry_point\n    );\n    let virt_page = Page::page_of(virt_entry_point as u32);\n    let entry_points: HashSet<i32> = entry_points\n        .iter()\n        .map(|e| virt_page.to_address() as i32 | *e as i32)\n        .collect();\n    let basic_blocks = jit_find_basic_blocks(ctx, entry_points, cpu.clone());\n\n    let mut pages = HashSet::new();\n\n    for b in basic_blocks.iter() {\n        // Remove this assertion once page-crossing jit is enabled\n        dbg_assert!(Page::page_of(b.addr) == Page::page_of(b.end_addr));\n        pages.insert(Page::page_of(b.addr));\n    }\n\n    let print = false;\n\n    for b in basic_blocks.iter() {\n        if !print {\n            break;\n        }\n        let last_instruction_opcode = memory::read32s(b.last_instruction_addr);\n        let op = opstats::decode(last_instruction_opcode as u32);\n        dbg_log!(\n            "BB: 0x{:x} {}{:02x} {} {}",\n            b.addr,\n            if op.is_0f { "0f" } else { "" },\n            op.opcode,\n            if b.is_entry_block { "entry" } else { "noentry" },\n            match &b.ty {\n                BasicBlockType::ConditionalJump {\n                    next_block_addr: Some(next_block_addr),\n                    next_block_branch_taken_addr: Some(next_block_branch_taken_addr),\n                    ..\n                } => format!(\n                    "0x{:x} 0x{:x}",\n                    next_block_addr, next_block_branch_taken_addr\n                ),\n                BasicBlockType::ConditionalJump {\n                    next_block_addr: None,\n                    next_block_branch_taken_addr: Some(next_block_branch_taken_addr),\n                    ..\n                } => format!("0x{:x}", next_block_branch_taken_addr),\n                BasicBlockType::ConditionalJump {\n                    next_block_addr: Some(next_block_addr),\n                    next_block_branch_taken_addr: None,\n                    ..\n                } => format!("0x{:x}", next_block_addr),\n                BasicBlockType::ConditionalJump {\n                    next_block_addr: None,\n                    next_block_branch_taken_addr: None,\n                    ..\n                } => format!(""),\n                BasicBlockType::Normal {\n                    next_block_addr: Some(next_block_addr),\n                    ..\n                } => format!("0x{:x}", next_block_addr),\n                BasicBlockType::Normal {\n                    next_block_addr: None,\n                    ..\n                } => format!(""),\n                BasicBlockType::Exit => format!(""),\n                BasicBlockType::AbsoluteEip => format!(""),\n            }\n        );\n    }\n\n    let graph = control_flow::make_graph(&basic_blocks);\n    let mut structure = control_flow::loopify(&graph);\n\n    if print {\n        dbg_log!("before blockify:");\n        for group in &structure {\n            dbg_log!("=> Group");\n            group.print(0);\n        }\n    }\n\n    control_flow::blockify(&mut structure, &graph);\n\n    if cfg!(debug_assertions) {\n        control_flow::assert_invariants(&structure);\n    }\n\n    if print {\n        dbg_log!("after blockify:");\n        for group in &structure {\n            dbg_log!("=> Group");\n            group.print(0);\n        }\n    }\n\n    if ctx.wasm_table_index_free_list.is_empty() {\n        dbg_log!("wasm_table_index_free_list empty, clearing cache");\n\n        // When no free slots are available, delete all cached modules. We could increase the\n        // size of the table, but this way the initial size acts as an upper bound for the\n        // number of wasm modules that we generate, which we want anyway to avoid getting our\n        // tab killed by browsers due to memory constraints.\n        jit_clear_cache(ctx);\n\n        profiler::stat_increment(stat::INVALIDATE_ALL_MODULES_NO_FREE_WASM_INDICES);\n\n        dbg_log!(\n            "after jit_clear_cache: {} free",\n            ctx.wasm_table_index_free_list.len(),\n        );\n\n        // This assertion can fail if all entries are pending (not possible unless\n        // WASM_TABLE_SIZE is set very low)\n        dbg_assert!(!ctx.wasm_table_index_free_list.is_empty());\n    }\n\n    // allocate an index in the wasm table\n    let wasm_table_index = ctx\n        .wasm_table_index_free_list\n        .pop()\n        .expect("allocate wasm table index");\n    dbg_assert!(wasm_table_index != WasmTableIndex(0));\n\n    dbg_assert!(!pages.is_empty());\n    dbg_assert!(pages.len() <= MAX_PAGES);\n    ctx.used_wasm_table_indices\n        .insert(wasm_table_index, pages.clone());\n    ctx.all_pages.extend(pages.clone());\n\n    let basic_block_by_addr: HashMap<u32, BasicBlock> =\n        basic_blocks.into_iter().map(|b| (b.addr, b)).collect();\n\n    let entries = jit_generate_module(\n        structure,\n        &basic_block_by_addr,\n        cpu.clone(),\n        &mut ctx.wasm_builder,\n        wasm_table_index,\n        state_flags,\n    );\n    dbg_assert!(!entries.is_empty());\n\n    profiler::stat_increment_by(\n        stat::COMPILE_WASM_TOTAL_BYTES,\n        ctx.wasm_builder.get_output_len() as u64,\n    );\n    profiler::stat_increment_by(stat::COMPILE_PAGE, pages.len() as u64);\n\n    cpu::tlb_set_has_code_multiple(&pages, true);\n\n    dbg_assert!(ctx.compiling.is_none());\n    ctx.compiling = Some((wasm_table_index, PageState::Compiling { entries }));\n\n    let phys_addr = page.to_address();\n\n    // will call codegen_finalize_finished asynchronously when finished\n    codegen_finalize(\n        wasm_table_index,\n        phys_addr,\n        state_flags,\n        ctx.wasm_builder.get_output_ptr() as u32,\n        ctx.wasm_builder.get_output_len(),\n    );\n\n    profiler::stat_increment(stat::COMPILE_SUCCESS);\n\n    check_jit_state_invariants(ctx);\n}\n\n#[no_mangle]\npub fn codegen_finalize_finished(\n    wasm_table_index: WasmTableIndex,\n    phys_addr: u32,\n    state_flags: CachedStateFlags,\n) {\n    let ctx = get_jit_state();\n\n    dbg_assert!(wasm_table_index != WasmTableIndex(0));\n\n    dbg_log!(\n        "Finished compiling for page at {:x}",\n        Page::page_of(phys_addr).to_address()\n    );\n\n    let entries = match mem::replace(&mut ctx.compiling, None) {\n        None => {\n            dbg_assert!(false);\n            return;\n        },\n        Some((in_progress_wasm_table_index, PageState::CompilingWritten)) => {\n            dbg_assert!(wasm_table_index == in_progress_wasm_table_index);\n\n            profiler::stat_increment(stat::INVALIDATE_MODULE_WRITTEN_WHILE_COMPILED);\n            free_wasm_table_index(ctx, wasm_table_index);\n            return;\n        },\n        Some((in_progress_wasm_table_index, PageState::Compiling { entries })) => {\n            dbg_assert!(wasm_table_index == in_progress_wasm_table_index);\n            entries\n        },\n    };\n\n    let mut check_for_unused_wasm_table_index = HashSet::new();\n\n    dbg_assert!(!entries.is_empty());\n    for (addr, entry) in entries {\n        let maybe_old_entry = ctx.cache.insert(addr, entry);\n\n        if let Some(old_entry) = maybe_old_entry {\n            check_for_unused_wasm_table_index.insert(old_entry.wasm_table_index);\n\n            profiler::stat_increment(stat::JIT_CACHE_OVERRIDE);\n            if old_entry.state_flags != state_flags {\n                profiler::stat_increment(stat::JIT_CACHE_OVERRIDE_DIFFERENT_STATE_FLAGS)\n            }\n        }\n    }\n\n    for index in check_for_unused_wasm_table_index {\n        let pages = ctx.used_wasm_table_indices.get(&index).unwrap();\n\n        let mut is_used = false;\n        \'outer: for p in pages {\n            for addr in p.address_range() {\n                if let Some(entry) = ctx.cache.get(&addr) {\n                    if entry.wasm_table_index == index {\n                        is_used = true;\n                        break \'outer;\n                    }\n                }\n            }\n        }\n\n        if !is_used {\n            profiler::stat_increment(stat::INVALIDATE_MODULE_UNUSED_AFTER_OVERWRITE);\n            free_wasm_table_index(ctx, index);\n        }\n\n        if !is_used {\n            for (_, entry) in &ctx.cache {\n                dbg_assert!(entry.wasm_table_index != index);\n            }\n        }\n        else {\n            let mut ok = false;\n            for (_, entry) in &ctx.cache {\n                if entry.wasm_table_index == index {\n                    ok = true;\n                    break;\n                }\n            }\n            dbg_assert!(ok);\n        }\n    }\n\n    check_jit_state_invariants(ctx);\n}\n\nfn jit_generate_module(\n    structure: Vec<WasmStructure>,\n    basic_blocks: &HashMap<u32, BasicBlock>,\n    mut cpu: CpuContext,\n    builder: &mut WasmBuilder,\n    wasm_table_index: WasmTableIndex,\n    state_flags: CachedStateFlags,\n) -> Vec<(u32, Entry)> {\n    builder.reset();\n\n    let mut register_locals = (0..8)\n        .map(|i| {\n            builder.load_fixed_i32(global_pointers::get_reg32_offset(i));\n            builder.set_new_local()\n        })\n        .collect();\n\n    builder.const_i32(0);\n    let instruction_counter = builder.set_new_local();\n\n    let exit_label = builder.block_void();\n    let exit_with_fault_label = builder.block_void();\n    let main_loop_label = builder.loop_void();\n    if JIT_USE_LOOP_SAFETY {\n        builder.get_local(&instruction_counter);\n        builder.const_i32(cpu::LOOP_COUNTER);\n        builder.geu_i32();\n        if cfg!(feature = "profiler") {\n            builder.if_void();\n            codegen::gen_debug_track_jit_exit(builder, 0);\n            builder.br(exit_label);\n            builder.block_end();\n        }\n        else {\n            builder.br_if(exit_label);\n        }\n    }\n    let brtable_default = builder.block_void();\n\n    let ctx = &mut JitContext {\n        cpu: &mut cpu,\n        builder,\n        register_locals: &mut register_locals,\n        start_of_current_instruction: 0,\n        exit_with_fault_label,\n        exit_label,\n        current_instruction: Instruction::Other,\n        previous_instruction: Instruction::Other,\n        instruction_counter,\n    };\n\n    let entry_blocks = {\n        let mut nodes = &structure;\n        let result;\n        loop {\n            match &nodes[0] {\n                WasmStructure::Dispatcher(e) => {\n                    result = e.clone();\n                    break;\n                },\n                WasmStructure::Loop { .. } => {\n                    dbg_assert!(false);\n                },\n                WasmStructure::BasicBlock(_) => {\n                    dbg_assert!(false);\n                },\n                // Note: We could use these blocks as entry points, which will yield\n                // more entries for free, but it requires adding those to the dispatcher\n                // It\'s to be investigated if this yields a performance improvement\n                // See also the comment at the bottom of this function when creating entry\n                // points\n                WasmStructure::Block(children) => {\n                    nodes = children;\n                },\n            }\n        }\n        result\n    };\n\n    let mut index_for_addr = HashMap::new();\n    for (i, &addr) in entry_blocks.iter().enumerate() {\n        index_for_addr.insert(addr, i as i32);\n    }\n    for b in basic_blocks.values() {\n        if !index_for_addr.contains_key(&b.addr) {\n            let i = index_for_addr.len();\n            index_for_addr.insert(b.addr, i as i32);\n        }\n    }\n\n    let mut label_for_addr: HashMap<u32, (Label, Option<i32>)> = HashMap::new();\n\n    enum Work {\n        WasmStructure(WasmStructure),\n        BlockEnd {\n            label: Label,\n            targets: Vec<u32>,\n            olds: HashMap<u32, (Label, Option<i32>)>,\n        },\n        LoopEnd {\n            label: Label,\n            entries: Vec<u32>,\n            olds: HashMap<u32, (Label, Option<i32>)>,\n        },\n    }\n    let mut work: VecDeque<Work> = structure\n        .into_iter()\n        .map(|x| Work::WasmStructure(x))\n        .collect();\n\n    while let Some(block) = work.pop_front() {\n        let next_addr: Option<Vec<u32>> = work.iter().find_map(|x| match x {\n            Work::WasmStructure(l) => Some(l.head().collect()),\n            _ => None,\n        });\n        let target_block = &ctx.builder.arg_local_initial_state.unsafe_clone();\n\n        match block {\n            Work::WasmStructure(WasmStructure::BasicBlock(addr)) => {\n                let block = basic_blocks.get(&addr).unwrap();\n                jit_generate_basic_block(ctx, block);\n\n                if block.has_sti {\n                    match block.ty {\n                        BasicBlockType::ConditionalJump {\n                            condition,\n                            jump_offset,\n                            jump_offset_is_32,\n                            ..\n                        } => {\n                            codegen::gen_set_eip_low_bits(\n                                ctx.builder,\n                                block.end_addr as i32 & 0xFFF,\n                            );\n                            codegen::gen_condition_fn(ctx, condition);\n                            ctx.builder.if_void();\n                            if jump_offset_is_32 {\n                                codegen::gen_relative_jump(ctx.builder, jump_offset);\n                            }\n                            else {\n                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                            }\n                            ctx.builder.block_end();\n                        },\n                        BasicBlockType::Normal {\n                            jump_offset,\n                            jump_offset_is_32,\n                            ..\n                        } => {\n        jit_instructions::jit_instruction(&mut cpu, builder, &mut instruction_flags);\n        let end_eip = cpu.eip;\n\n        #[cfg(debug_assertions)]\n        {\n            if ::config::CHECK_CPU_EXCEPTIONS {\n                // only the last instruction in each basic block is allowed to raise\n                if end_eip < stop_addr {\n                    let fn_idx =\n                        builder.get_fn_idx("assert_no_cpu_exception", module_init::FN0_TYPE_INDEX);\n                    builder.instruction_body.call_fn(fn_idx);\n                }\n            }\n        }\n\n        let instruction_length = end_eip - start_eip;\n        let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;\n\n                            }\n                            else {\n                                codegen::gen_set_eip_low_bits(\n                                    ctx.builder,\n                                    block.end_addr as i32 & 0xFFF,\n                                );\n                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                            }\n                        },\n                        BasicBlockType::Exit => {},\n                        BasicBlockType::AbsoluteEip => {},\n                    };\n                    codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);\n                    codegen::gen_move_registers_from_locals_to_memory(ctx);\n                    codegen::gen_fn0_const(ctx.builder, "handle_irqs");\n                    codegen::gen_update_instruction_counter(ctx);\n                    ctx.builder.return_();\n                    continue;\n                }\n\n                match &block.ty {\n                    BasicBlockType::Exit => {\n                        // Exit this function\n                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);\n                        codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);\n                        ctx.builder.br(ctx.exit_label);\n                    },\n                    BasicBlockType::AbsoluteEip => {\n                        // Check if we can stay in this module, if not exit\n                        codegen::gen_get_eip(ctx.builder);\n                        let new_eip = ctx.builder.set_new_local();\n                        codegen::gen_get_phys_eip_plus_mem(ctx, &new_eip);\n                        ctx.builder.const_i32(unsafe { memory::mem8 } as i32);\n                        ctx.builder.sub_i32();\n                        ctx.builder.free_local(new_eip);\n\n                        ctx.builder.const_i32(wasm_table_index.to_u16() as i32);\n                        ctx.builder.const_i32(state_flags.to_u32() as i32);\n                        ctx.builder.call_fn3_ret("jit_find_cache_entry_in_page");\n                        ctx.builder.tee_local(target_block);\n                        ctx.builder.const_i32(0);\n                        ctx.builder.ge_i32();\n                        // TODO: Could make this unconditional by including exit_label in the main br_table\n                        ctx.builder.br_if(main_loop_label);\n\n                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);\n                        ctx.builder.br(ctx.exit_label);\n                    },\n                    &BasicBlockType::Normal {\n                        next_block_addr: None,\n                        jump_offset,\n                        jump_offset_is_32,\n                    } => {\n                        if jump_offset_is_32 {\n                            codegen::gen_set_eip_low_bits_and_jump_rel32(\n                                ctx.builder,\n                                block.end_addr as i32 & 0xFFF,\n                                jump_offset,\n                            );\n                        }\n                        else {\n                            codegen::gen_set_eip_low_bits(\n                                ctx.builder,\n                                block.end_addr as i32 & 0xFFF,\n                            );\n                            codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                        }\n\n                        codegen::gen_debug_track_jit_exit(ctx.builder, block.last_instruction_addr);\n                        codegen::gen_profiler_stat_increment(ctx.builder, stat::DIRECT_EXIT);\n                        ctx.builder.br(ctx.exit_label);\n                    },\n                    &BasicBlockType::Normal {\n                        next_block_addr: Some(next_block_addr),\n                        jump_offset,\n                        jump_offset_is_32,\n                    } => {\n                        // Unconditional jump to next basic block\n                        // - All instructions that don\'t change eip\n                        // - Unconditional jumps\n\n                        if Page::page_of(next_block_addr) != Page::page_of(block.addr) {\n                            if jump_offset_is_32 {\n                                codegen::gen_set_eip_low_bits_and_jump_rel32(\n                                    ctx.builder,\n                                    block.end_addr as i32 & 0xFFF,\n                                    jump_offset,\n                                );\n                            }\n                            else {\n                                codegen::gen_set_eip_low_bits(\n                                    ctx.builder,\n                                    block.end_addr as i32 & 0xFFF,\n                                );\n                                codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                            }\n\n                            codegen::gen_profiler_stat_increment(\n                                ctx.builder,\n                                stat::NORMAL_PAGE_CHANGE,\n                            );\n\n                            codegen::gen_page_switch_check(\n                                ctx,\n                                next_block_addr,\n                                block.last_instruction_addr,\n                            );\n\n                            #[cfg(debug_assertions)]\n                            codegen::gen_fn2_const(\n                                ctx.builder,\n                                "check_page_switch",\n                                block.addr,\n                                next_block_addr,\n                            );\n                        }\n\n                        if next_addr\n                            .as_ref()\n                            .map_or(false, |n| n.contains(&next_block_addr))\n                        {\n                            // Blocks are consecutive\n                            if next_addr.unwrap().len() > 1 {\n                                let target_index = *index_for_addr.get(&next_block_addr).unwrap();\n                                if cfg!(feature = "profiler") {\n                                    ctx.builder.const_i32(target_index);\n                                    ctx.builder.call_fn1("debug_set_dispatcher_target");\n                                }\n                                ctx.builder.const_i32(target_index);\n                                ctx.builder.set_local(target_block);\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::NORMAL_FALLTHRU_WITH_TARGET_BLOCK,\n                                );\n                            }\n                            else {\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::NORMAL_FALLTHRU,\n                                );\n                            }\n                        }\n                        else {\n                            let &(br, target_index) = label_for_addr.get(&next_block_addr).unwrap();\n                            if let Some(target_index) = target_index {\n                                if cfg!(feature = "profiler") {\n                                    ctx.builder.const_i32(target_index);\n                                    ctx.builder.call_fn1("debug_set_dispatcher_target");\n                                }\n                                ctx.builder.const_i32(target_index);\n                                ctx.builder.set_local(target_block);\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::NORMAL_BRANCH_WITH_TARGET_BLOCK,\n                                );\n                            }\n                            else {\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::NORMAL_BRANCH,\n                                );\n                            }\n                            ctx.builder.br(br);\n                        }\n                    },\n                    &BasicBlockType::ConditionalJump {\n                        next_block_addr,\n                        next_block_branch_taken_addr,\n                        condition,\n                        jump_offset,\n                        jump_offset_is_32,\n                    } => {\n                        // Conditional jump to next basic block\n                        // - jnz, jc, loop, jcxz, etc.\n\n                        // Generate:\n                        // (1) condition()\n                        // (2) br_if()\n                        // (3) br()\n                        // Except:\n                        // If we need to update eip in case (2), it\'s replaced by if { update_eip(); br() }\n                        // If case (3) can fall through to the next basic block, the branch is eliminated\n                        // Dispatcher target writes can be generated in either case\n                        // Condition may be inverted if it helps generate a fallthrough instead of the second branch\n\n                        codegen::gen_profiler_stat_increment(ctx.builder, stat::CONDITIONAL_JUMP);\n\n                        #[derive(PartialEq)]\n                        enum Case {\n                            BranchTaken,\n                            BranchNotTaken,\n                        }\n\n                        let mut handle_case = |case: Case, is_first| {\n                            // first case generates condition and *has* to branch away,\n                            // second case branches unconditionally or falls through\n\n                            if is_first {\n                                if case == Case::BranchNotTaken {\n                                    codegen::gen_condition_fn_negated(ctx, condition);\n                                }\n                                else {\n                                    codegen::gen_condition_fn(ctx, condition);\n                                }\n                            }\n\n                            let next_block_addr = if case == Case::BranchTaken {\n                                next_block_branch_taken_addr\n                            }\n                            else {\n                                next_block_addr\n                            };\n\n                            if let Some(next_block_addr) = next_block_addr {\n                                if Page::page_of(next_block_addr) != Page::page_of(block.addr) {\n                                    dbg_assert!(case == Case::BranchTaken); // currently not possible in other case\n                                    if is_first {\n                                        ctx.builder.if_i32();\n                                    }\n                                    if jump_offset_is_32 {\n                                        codegen::gen_set_eip_low_bits_and_jump_rel32(\n                                            ctx.builder,\n                                            block.end_addr as i32 & 0xFFF,\n                                            jump_offset,\n                                        );\n                                    }\n                                    else {\n                                        codegen::gen_set_eip_low_bits(\n                                            ctx.builder,\n                                            block.end_addr as i32 & 0xFFF,\n                                        );\n                                        codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                                    }\n\n                                    codegen::gen_profiler_stat_increment(\n                                        ctx.builder,\n                                        stat::CONDITIONAL_JUMP_PAGE_CHANGE,\n                                    );\n                                    codegen::gen_page_switch_check(\n                                        ctx,\n                                        next_block_addr,\n                                        block.last_instruction_addr,\n                                    );\n\n                                    #[cfg(debug_assertions)]\n                                    codegen::gen_fn2_const(\n                                        ctx.builder,\n                                        "check_page_switch",\n                                        block.addr,\n                                        next_block_addr,\n                                    );\n\n                                    if is_first {\n                                        ctx.builder.const_i32(1);\n                                        ctx.builder.else_();\n                                        ctx.builder.const_i32(0);\n                                        ctx.builder.block_end();\n                                    }\n                                }\n\n                                if next_addr\n                                    .as_ref()\n                                    .map_or(false, |n| n.contains(&next_block_addr))\n                                {\n                                    // blocks are consecutive\n\n                                    // fallthrough, has to be second\n                                    dbg_assert!(!is_first);\n\n                                    if next_addr.as_ref().unwrap().len() > 1 {\n                                        let target_index =\n                                            *index_for_addr.get(&next_block_addr).unwrap();\n                                        if cfg!(feature = "profiler") {\n                                            ctx.builder.const_i32(target_index);\n                                            ctx.builder.call_fn1("debug_set_dispatcher_target");\n                                        }\n                                        ctx.builder.const_i32(target_index);\n                                        ctx.builder.set_local(target_block);\n                                        codegen::gen_profiler_stat_increment(\n                                            ctx.builder,\n                                            stat::CONDITIONAL_JUMP_FALLTHRU_WITH_TARGET_BLOCK,\n                                        );\n                                    }\n                                    else {\n                                        codegen::gen_profiler_stat_increment(\n                                            ctx.builder,\n                                            stat::CONDITIONAL_JUMP_FALLTHRU,\n                                        );\n                                    }\n                                }\n                                else {\n                                    let &(br, target_index) =\n                                        label_for_addr.get(&next_block_addr).unwrap();\n                                    if let Some(target_index) = target_index {\n                                        if cfg!(feature = "profiler") {\n                                            // Note: Currently called unconditionally, even if the\n                                            // br_if below doesn\'t branch\n                                            ctx.builder.const_i32(target_index);\n                                            ctx.builder.call_fn1("debug_set_dispatcher_target");\n                                        }\n                                        ctx.builder.const_i32(target_index);\n                                        ctx.builder.set_local(target_block);\n                                    }\n\n                                    if is_first {\n                                        if cfg!(feature = "profiler") {\n                                            ctx.builder.if_void();\n                                            codegen::gen_profiler_stat_increment(\n                                                ctx.builder,\n                                                if target_index.is_some() {\n                                                    stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK\n                                                }\n                                                else {\n                                                    stat::CONDITIONAL_JUMP_BRANCH\n                                                },\n                                            );\n                                            ctx.builder.br(br);\n                                            ctx.builder.block_end();\n                                        }\n                                        else {\n                                            ctx.builder.br_if(br);\n                                        }\n                                    }\n                                    else {\n                                        codegen::gen_profiler_stat_increment(\n                                            ctx.builder,\n                                            if target_index.is_some() {\n                                                stat::CONDITIONAL_JUMP_BRANCH_WITH_TARGET_BLOCK\n                                            }\n                                            else {\n                                                stat::CONDITIONAL_JUMP_BRANCH\n                                            },\n                                        );\n                                        ctx.builder.br(br);\n                                    }\n                                }\n                            }\n                            else {\n                                // target is outside of this module, update eip and exit\n                                if is_first {\n                                    ctx.builder.if_void();\n                                }\n\n                                if case == Case::BranchTaken {\n                                    if jump_offset_is_32 {\n                                        codegen::gen_set_eip_low_bits_and_jump_rel32(\n                                            ctx.builder,\n                                            block.end_addr as i32 & 0xFFF,\n                                            jump_offset,\n                                        );\n                                    }\n                                    else {\n                                        codegen::gen_set_eip_low_bits(\n                                            ctx.builder,\n                                            block.end_addr as i32 & 0xFFF,\n                                        );\n                                        codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                                    }\n                                }\n                                else {\n                                    codegen::gen_set_eip_low_bits(\n                                        ctx.builder,\n                                        block.end_addr as i32 & 0xFFF,\n                                    );\n                                }\n\n                                codegen::gen_debug_track_jit_exit(\n                                    ctx.builder,\n                                    block.last_instruction_addr,\n                                );\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::CONDITIONAL_JUMP_EXIT,\n                                );\n                                ctx.builder.br(ctx.exit_label);\n\n                                if is_first {\n                                    ctx.builder.block_end();\n                                }\n                            }\n                        };\n\n                        let branch_taken_is_fallthrough = next_block_branch_taken_addr\n                            .map_or(false, |addr| {\n                                next_addr.as_ref().map_or(false, |n| n.contains(&addr))\n                            });\n                        let branch_not_taken_is_fallthrough = next_block_addr\n                            .map_or(false, |addr| {\n                                next_addr.as_ref().map_or(false, |n| n.contains(&addr))\n                            });\n\n                        if branch_not_taken_is_fallthrough && branch_taken_is_fallthrough {\n                            let next_block_addr = next_block_addr.unwrap();\n                            let next_block_branch_taken_addr =\n                                next_block_branch_taken_addr.unwrap();\n\n                            dbg_assert!(\n                                Page::page_of(next_block_addr) == Page::page_of(block.addr)\n                            ); // currently not possible\n\n                            if Page::page_of(next_block_branch_taken_addr)\n                                != Page::page_of(block.addr)\n                            {\n                                if jump_offset_is_32 {\n                                    codegen::gen_set_eip_low_bits_and_jump_rel32(\n                                        ctx.builder,\n                                        block.end_addr as i32 & 0xFFF,\n                                        jump_offset,\n                                    );\n                                }\n                                else {\n                                    codegen::gen_set_eip_low_bits(\n                                        ctx.builder,\n                                        block.end_addr as i32 & 0xFFF,\n                                    );\n                                    codegen::gen_jmp_rel16(ctx.builder, jump_offset as u16);\n                                }\n\n                                codegen::gen_profiler_stat_increment(\n                                    ctx.builder,\n                                    stat::CONDITIONAL_JUMP_PAGE_CHANGE,\n                                );\n                                codegen::gen_page_switch_check(\n                                    ctx,\n                                    next_block_branch_taken_addr,\n                                    block.last_instruction_addr,\n                                );\n\n                                #[cfg(debug_assertions)]\n                                codegen::gen_fn2_const(\n                                    ctx.builder,\n                                    "check_page_switch",\n                                    block.addr,\n                                    next_block_branch_taken_addr,\n                                );\n                            }\n\n                            if next_addr.unwrap().len() > 1 {\n                                let target_index_taken =\n                                    *index_for_addr.get(&next_block_branch_taken_addr).unwrap();\n                                let target_index_not_taken =\n                                    *index_for_addr.get(&next_block_addr).unwrap();\n\n                                codegen::gen_condition_fn(ctx, condition);\n                                ctx.builder.if_i32();\n                                ctx.builder.const_i32(target_index_taken);\n                                ctx.builder.else_();\n                                ctx.builder.const_i32(target_index_not_taken);\n                                ctx.builder.block_end();\n                                ctx.builder.set_local(target_block);\n                            }\n                        }\n                        else if branch_taken_is_fallthrough {\n                            handle_case(Case::BranchNotTaken, true);\n                            handle_case(Case::BranchTaken, false);\n                        }\n                        else {\n                            handle_case(Case::BranchTaken, true);\n                            handle_case(Case::BranchNotTaken, false);\n                        }\n                    },\n                }\n            },\n            Work::WasmStructure(WasmStructure::Dispatcher(entries)) => {\n                profiler::stat_increment(stat::COMPILE_DISPATCHER);\n\n                if cfg!(feature = "profiler") {\n                    ctx.builder.get_local(target_block);\n                    ctx.builder.const_i32(index_for_addr.len() as i32);\n                    ctx.builder.call_fn2("check_dispatcher_target");\n                }\n\n                if entries.len() > BRTABLE_CUTOFF {\n                    // generate a brtable\n                    codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_LARGE);\n                    let mut cases = Vec::new();\n                    for &addr in &entries {\n                        let &(label, target_index) = label_for_addr.get(&addr).unwrap();\n                        let &index = index_for_addr.get(&addr).unwrap();\n                        dbg_assert!(target_index.is_none() || target_index == Some(index));\n                        while index as usize >= cases.len() {\n                            cases.push(brtable_default);\n                        }\n                        cases[index as usize] = label;\n                    }\n                    ctx.builder.get_local(target_block);\n                    ctx.builder.brtable(brtable_default, &mut cases.iter());\n                }\n                else {\n                    // generate a if target == block.addr then br block.label ...\n                    codegen::gen_profiler_stat_increment(ctx.builder, stat::DISPATCHER_SMALL);\n                    let nexts: HashSet<u32> = next_addr\n                        .as_ref()\n                        .map_or(HashSet::new(), |nexts| nexts.iter().copied().collect());\n                    for &addr in &entries {\n                        if nexts.contains(&addr) {\n                            continue;\n                        }\n                        let index = *index_for_addr.get(&addr).unwrap();\n                        let &(label, _) = label_for_addr.get(&addr).unwrap();\n                        ctx.builder.get_local(target_block);\n                        ctx.builder.const_i32(index);\n                        ctx.builder.eq_i32();\n                        ctx.builder.br_if(label);\n                    }\n                }\n            },\n            Work::WasmStructure(WasmStructure::Loop(children)) => {\n                profiler::stat_increment(stat::COMPILE_WASM_LOOP);\n\n                let entries: Vec<u32> = children[0].head().collect();\n                let label = ctx.builder.loop_void();\n                codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP);\n\n                if entries.len() == 1 {\n                    let addr = entries[0];\n                    codegen::gen_set_eip_low_bits(ctx.builder, addr as i32 & 0xFFF);\n                    profiler::stat_increment(stat::COMPILE_WITH_LOOP_SAFETY);\n                    codegen::gen_profiler_stat_increment(ctx.builder, stat::LOOP_SAFETY);\n                    if JIT_USE_LOOP_SAFETY {\n                        ctx.builder.get_local(&ctx.instruction_counter);\n                        ctx.builder.const_i32(cpu::LOOP_COUNTER);\n                        ctx.builder.geu_i32();\n                        if cfg!(feature = "profiler") {\n                            ctx.builder.if_void();\n                            codegen::gen_debug_track_jit_exit(ctx.builder, addr);\n                            ctx.builder.br(exit_label);\n                            ctx.builder.block_end();\n                        }\n                        else {\n                            ctx.builder.br_if(exit_label);\n                        }\n                    }\n                }\n\n                let mut olds = HashMap::new();\n                for &target in entries.iter() {\n                    let index = if entries.len() == 1 {\n                        None\n                    }\n                    else {\n                        Some(*index_for_addr.get(&target).unwrap())\n                    };\n                    let old = label_for_addr.insert(target, (label, index));\n                    if let Some(old) = old {\n                        olds.insert(target, old);\n                    }\n                }\n\n                work.push_front(Work::LoopEnd {\n                    label,\n                    entries,\n                    olds,\n                });\n                for c in children.into_iter().rev() {\n                    work.push_front(Work::WasmStructure(c));\n                }\n            },\n            Work::LoopEnd {\n                label,\n                entries,\n                olds,\n            } => {\n                for target in entries {\n                    let old = label_for_addr.remove(&target);\n                    dbg_assert!(old.map(|(l, _)| l) == Some(label));\n                }\n                for (target, old) in olds {\n                    let old = label_for_addr.insert(target, old);\n                    dbg_assert!(old.is_none());\n                }\n\n                ctx.builder.block_end();\n            },\n            Work::WasmStructure(WasmStructure::Block(children)) => {\n                profiler::stat_increment(stat::COMPILE_WASM_BLOCK);\n\n                let targets = next_addr.clone().unwrap();\n                let label = ctx.builder.block_void();\n                let mut olds = HashMap::new();\n                for &target in targets.iter() {\n                    let index = if targets.len() == 1 {\n                        None\n                    }\n                    else {\n                        Some(*index_for_addr.get(&target).unwrap())\n                    };\n                    let old = label_for_addr.insert(target, (label, index));\n                    if let Some(old) = old {\n                        olds.insert(target, old);\n                    }\n                }\n\n                work.push_front(Work::BlockEnd {\n                    label,\n                    targets,\n                    olds,\n                });\n                for c in children.into_iter().rev() {\n                    work.push_front(Work::WasmStructure(c));\n                }\n            },\n            Work::BlockEnd {\n                label,\n                targets,\n                olds,\n            } => {\n                for target in targets {\n                    let old = label_for_addr.remove(&target);\n                    dbg_assert!(old.map(|(l, _)| l) == Some(label));\n                }\n                for (target, old) in olds {\n                    let old = label_for_addr.insert(target, old);\n                    dbg_assert!(old.is_none());\n                }\n\n                ctx.builder.block_end();\n            },\n        }\n    }\n\n    dbg_assert!(label_for_addr.is_empty());\n\n    {\n        ctx.builder.block_end(); // default case for the brtable\n        ctx.builder.unreachable();\n    }\n    {\n        ctx.builder.block_end(); // main loop\n    }\n    {\n        // exit-with-fault case\n        ctx.builder.block_end();\n        codegen::gen_move_registers_from_locals_to_memory(ctx);\n        codegen::gen_fn0_const(ctx.builder, "trigger_fault_end_jit");\n        codegen::gen_update_instruction_counter(ctx);\n        ctx.builder.return_();\n    }\n    {\n        // exit\n        ctx.builder.block_end();\n        codegen::gen_move_registers_from_locals_to_memory(ctx);\n        codegen::gen_update_instruction_counter(ctx);\n    }\n\n    for local in ctx.register_locals.drain(..) {\n        ctx.builder.free_local(local);\n    }\n    ctx.builder\n        .free_local(ctx.instruction_counter.unsafe_clone());\n\n    ctx.builder.finish();\n\n    let mut entries = Vec::new();\n\n    for &addr in entry_blocks.iter() {\n        let block = basic_blocks.get(&addr).unwrap();\n        let index = *index_for_addr.get(&addr).unwrap();\n\n        profiler::stat_increment(stat::COMPILE_ENTRY_POINT);\n\n        dbg_assert!(block.addr < block.end_addr);\n        // Note: We also insert blocks that weren\'t originally marked as entries here\n        //       This doesn\'t have any downside, besides making the hash table slightly larger\n\n        let initial_state = index.safe_to_u16();\n\n        let entry = Entry {\n            wasm_table_index,\n            initial_state,\n            state_flags,\n\n            #[cfg(any(debug_assertions, feature = "profiler"))]\n            len: block.end_addr - block.addr,\n\n            #[cfg(debug_assertions)]\n            opcode: memory::read32s(block.addr) as u32,\n        };\n        entries.push((block.addr, entry));\n    }\n\n    for b in basic_blocks.values() {\n        if b.is_entry_block {\n            dbg_assert!(entries.iter().find(|(addr, _)| *addr == b.addr).is_some());\n        }\n    }\n\n    return entries;\n}\n\nfn jit_generate_basic_block(ctx: &mut JitContext, block: &BasicBlock) {\n    let needs_eip_updated = match block.ty {\n        BasicBlockType::Exit => true,\n        _ => false,\n    };\n\n    profiler::stat_increment(stat::COMPILE_BASIC_BLOCK);\n\n    let start_addr = block.addr;\n    let last_instruction_addr = block.last_instruction_addr;\n    let stop_addr = block.end_addr;\n\n    // First iteration of do-while assumes the caller confirms this condition\n    dbg_assert!(!is_near_end_of_page(start_addr));\n\n    if cfg!(feature = "profiler") {\n        ctx.builder.const_i32(start_addr as i32);\n        ctx.builder.call_fn1("enter_basic_block");\n    }\n\n    ctx.builder.get_local(&ctx.instruction_counter);\n    ctx.builder.const_i32(block.number_of_instructions as i32);\n    ctx.builder.add_i32();\n    ctx.builder.set_local(&ctx.instruction_counter);\n\n    ctx.cpu.eip = start_addr;\n    ctx.current_instruction = Instruction::Other;\n    ctx.previous_instruction = Instruction::Other;\n\n    loop {\n        let mut instruction = 0;\n        if cfg!(feature = "profiler") {\n            instruction = memory::read32s(ctx.cpu.eip) as u32;\n            opstats::gen_opstats(ctx.builder, instruction);\n            opstats::record_opstat_compiled(instruction);\n        }\n\n        if ctx.cpu.eip == last_instruction_addr {\n            // Before the last instruction:\n            // - Set eip to *after* the instruction\n            // - Set previous_eip to *before* the instruction\n            if needs_eip_updated {\n                codegen::gen_set_previous_eip_offset_from_eip_with_low_bits(\n                    ctx.builder,\n                    last_instruction_addr as i32 & 0xFFF,\n                );\n                codegen::gen_set_eip_low_bits(ctx.builder, stop_addr as i32 & 0xFFF);\n            }\n        }\n\n        let wasm_length_before = ctx.builder.instruction_body_length();\n\n        ctx.start_of_current_instruction = ctx.cpu.eip;\n        let start_eip = ctx.cpu.eip;\n        let mut instruction_flags = 0;\n        jit_instructions::jit_instruction(ctx, &mut instruction_flags);\n        let end_eip = ctx.cpu.eip;\n\n        let instruction_length = end_eip - start_eip;\n        let was_block_boundary = instruction_flags & JIT_INSTR_BLOCK_BOUNDARY_FLAG != 0;\n\n        let wasm_length = ctx.builder.instruction_body_length() - wasm_length_before;\n        opstats::record_opstat_size_wasm(instruction, wasm_length as u64);\n\n        dbg_assert!((end_eip == stop_addr) == (start_eip == last_instruction_addr));\n        dbg_assert!(instruction_length < MAX_INSTRUCTION_LENGTH);\n\n        let end_addr = ctx.cpu.eip;\n\n        if end_addr == stop_addr {\n            // no page was crossed\n            dbg_assert!(Page::page_of(end_addr) == Page::page_of(start_addr));\n            break;\n        }\n\n        if was_block_boundary || is_near_end_of_page(end_addr) || end_addr > stop_addr {\n            dbg_log!(\n                "Overlapping basic blocks start={:x} expected_end={:x} end={:x} was_block_boundary={} near_end_of_page={}",\n                start_addr,\n                stop_addr,\n                end_addr,\n                was_block_boundary,\n                is_near_end_of_page(end_addr)\n            );\n            dbg_assert!(false);\n            break;\n        }\n\n        ctx.previous_instruction = mem::replace(&mut ctx.current_instruction, Instruction::Other);\n    }\n}\n\npub fn jit_increase_hotness_and_maybe_compile(\n    virt_address: i32,\n    phys_address: u32,\n    cs_offset: u32,\n    state_flags: CachedStateFlags,\n    hotness: u32,\n) {\n    let ctx = get_jit_state();\n    let page = Page::page_of(phys_address);\n    let address_hash = jit_hot_hash_page(page) as usize;\n    ctx.hot_pages[address_hash] += hotness;\n    if ctx.hot_pages[address_hash] >= JIT_THRESHOLD {\n        if ctx.compiling.is_some() {\n            return;\n        }\n        // only try generating if we\'re in the correct address space\n        if cpu::translate_address_read_no_side_effects(virt_address) == Ok(phys_address) {\n            ctx.hot_pages[address_hash] = 0;\n            jit_analyze_and_generate(ctx, virt_address, phys_address, cs_offset, state_flags)\n        }\n        else {\n            profiler::stat_increment(stat::COMPILE_WRONG_ADDRESS_SPACE);\n        }\n    };\n}\n\nfn free_wasm_table_index(ctx: &mut JitState, wasm_table_index: WasmTableIndex) {\n    if CHECK_JIT_STATE_INVARIANTS {\n        dbg_assert!(!ctx.wasm_table_index_free_list.contains(&wasm_table_index));\n\n        match &ctx.compiling {\n            Some((wasm_table_index_compiling, _)) => {\n                dbg_assert!(\n                    *wasm_table_index_compiling != wasm_table_index,\n                    "Attempt to free wasm table index that is currently being compiled"\n                );\n            },\n            _ => {},\n        }\n    }\n\n    match ctx.used_wasm_table_indices.remove(&wasm_table_index) {\n        None => {\n            dbg_assert!(false);\n        },\n        Some(_pages) => {\n            //dbg_assert!(!pages.is_empty()); // only if CompilingWritten\n        },\n    }\n    ctx.wasm_table_index_free_list.push(wasm_table_index);\n\n    dbg_assert!(\n        ctx.wasm_table_index_free_list.len() + ctx.used_wasm_table_indices.len()\n            == WASM_TABLE_SIZE as usize - 1\n    );\n\n    // It is not strictly necessary to clear the function, but it will fail more predictably if we\n    // accidentally use the function and may garbage collect unused modules earlier\n    jit_clear_func(wasm_table_index);\n\n    rebuild_all_pages(ctx);\n\n    check_jit_state_invariants(ctx);\n}\n\npub fn rebuild_all_pages(ctx: &mut JitState) {\n    // rebuild ctx.all_pages\n    let mut all_pages = HashSet::new();\n    for pages in ctx.used_wasm_table_indices.values() {\n        all_pages.extend(pages);\n    }\n    ctx.all_pages = all_pages;\n}\n\n/// Register a write in this page: Delete all present code\npub fn jit_dirty_page(ctx: &mut JitState, page: Page) {\n    let mut did_have_code = false;\n\n    if ctx.all_pages.contains(&page) {\n        profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_CODE);\n        did_have_code = true;\n        let mut index_to_free = HashSet::new();\n\n        let compiling = match &ctx.compiling {\n            Some((wasm_table_index, _)) => Some(*wasm_table_index),\n            None => None,\n        };\n\n        for (&wasm_table_index, pages) in &ctx.used_wasm_table_indices {\n            if Some(wasm_table_index) != compiling && pages.contains(&page) {\n                index_to_free.insert(wasm_table_index);\n            }\n        }\n\n        match &ctx.compiling {\n            None => {},\n            Some((_, PageState::CompilingWritten)) => {},\n            Some((wasm_table_index, PageState::Compiling { .. })) => {\n                let pages = ctx\n                    .used_wasm_table_indices\n                    .get_mut(wasm_table_index)\n                    .unwrap();\n                if pages.contains(&page) {\n                    pages.clear();\n                    ctx.compiling = Some((*wasm_table_index, PageState::CompilingWritten));\n                    rebuild_all_pages(ctx);\n                }\n            },\n        }\n\n        for index in &index_to_free {\n            match ctx.used_wasm_table_indices.get(&index) {\n                None => {\n                    dbg_assert!(false);\n                },\n                Some(pages) => {\n                    for &p in pages {\n                        for addr in p.address_range() {\n                            if let Some(e) = ctx.cache.get(&addr) {\n                                if index_to_free.contains(&e.wasm_table_index) {\n                                    ctx.cache.remove(&addr);\n                                }\n                            }\n                        }\n                    }\n                },\n            }\n        }\n\n        for index in index_to_free {\n            profiler::stat_increment(stat::INVALIDATE_MODULE_DIRTY_PAGE);\n            free_wasm_table_index(ctx, index)\n        }\n    }\n\n    match ctx.entry_points.remove(&page) {\n        None => {},\n        Some(_entry_points) => {\n            profiler::stat_increment(stat::INVALIDATE_PAGE_HAD_ENTRY_POINTS);\n            did_have_code = true;\n\n            // don\'t try to compile code in this page anymore until it\'s hot again\n            ctx.hot_pages[jit_hot_hash_page(page) as usize] = 0;\n        },\n    }\n\n    for pages in ctx.used_wasm_table_indices.values() {\n        dbg_assert!(!pages.contains(&page));\n    }\n\n    check_jit_state_invariants(ctx);\n\n    dbg_assert!(!ctx.all_pages.contains(&page));\n    dbg_assert!(!jit_page_has_code_ctx(ctx, page));\n\n    if did_have_code {\n        cpu::tlb_set_has_code(page, false);\n    }\n\n    if !did_have_code {\n        profiler::stat_increment(stat::DIRTY_PAGE_DID_NOT_HAVE_CODE);\n    }\n}\n\n#[no_mangle]\npub fn jit_dirty_cache(start_addr: u32, end_addr: u32) {\n    dbg_assert!(start_addr < end_addr);\n\n    let start_page = Page::page_of(start_addr);\n    let end_page = Page::page_of(end_addr - 1);\n\n    for page in start_page.to_u32()..end_page.to_u32() + 1 {\n        jit_dirty_page(get_jit_state(), Page::page_of(page << 12));\n    }\n}\n\n/// dirty pages in the range of start_addr and end_addr, which must span at most two pages\npub fn jit_dirty_cache_small(start_addr: u32, end_addr: u32) {\n    dbg_assert!(start_addr < end_addr);\n\n    let start_page = Page::page_of(start_addr);\n    let end_page = Page::page_of(end_addr - 1);\n\n    let ctx = get_jit_state();\n    jit_dirty_page(ctx, start_page);\n\n    // Note: This can\'t happen when paging is enabled, as writes across\n    //       boundaries are split up on two pages\n    if start_page != end_page {\n        dbg_assert!(start_page.to_u32() + 1 == end_page.to_u32());\n        jit_dirty_page(ctx, end_page);\n    }\n}\n\n#[no_mangle]\npub fn jit_clear_cache_js() { jit_clear_cache(get_jit_state()) }\n\npub fn jit_clear_cache(ctx: &mut JitState) {\n    let mut pages_with_code = HashSet::new();\n\n    for page in ctx.entry_points.keys() {\n        pages_with_code.insert(*page);\n    }\n    for &p in &ctx.all_pages {\n        pages_with_code.insert(p);\n    }\n    for addr in ctx.cache.keys() {\n        dbg_assert!(pages_with_code.contains(&Page::page_of(*addr)));\n    }\n    for pages in ctx.used_wasm_table_indices.values() {\n        dbg_assert!(pages_with_code.is_superset(pages));\n    }\n\n    for page in pages_with_code {\n        jit_dirty_page(ctx, page);\n    }\n}\n\npub fn jit_page_has_code(page: Page) -> bool { jit_page_has_code_ctx(get_jit_state(), page) }\n\npub fn jit_page_has_code_ctx(ctx: &mut JitState, page: Page) -> bool {\n    ctx.all_pages.contains(&page) || ctx.entry_points.contains_key(&page)\n}\n\n#[no_mangle]\npub fn jit_get_wasm_table_index_free_list_count() -> u32 {\n    if cfg!(feature = "profiler") {\n        get_jit_state().wasm_table_index_free_list.len() as u32\n    }\n    else {\n        0\n    }\n}\n#[no_mangle]\npub fn jit_get_cache_size() -> u32 {\n    if cfg!(feature = "profiler") { get_jit_state().cache.len() as u32 } else { 0 }\n}\n\n#[cfg(feature = "profiler")]\npub fn check_missed_entry_points(phys_address: u32, state_flags: CachedStateFlags) {\n    let ctx = get_jit_state();\n\n    // backwards until beginning of page\n    for offset in 0..=(phys_address & 0xFFF) {\n        let addr = phys_address - offset;\n        dbg_assert!(phys_address >= addr);\n\n        if let Some(entry) = ctx.cache.get(&addr) {\n            if entry.state_flags != state_flags || phys_address >= addr + entry.len {\n                // give up search on first entry that is not a match\n                break;\n            }\n\n            profiler::stat_increment(stat::RUN_INTERPRETED_MISSED_COMPILED_ENTRY_LOOKUP);\n\n            let last_jump_type = unsafe { cpu::debug_last_jump.name() };\n            let last_jump_addr = unsafe { cpu::debug_last_jump.phys_address() }.unwrap_or(0);\n            let last_jump_opcode =\n                if last_jump_addr != 0 { memory::read32s(last_jump_addr) } else { 0 };\n\n            let opcode = memory::read32s(phys_address);\n            dbg_log!(\n                "Compiled exists, but no entry point, \\\n                 start={:x} end={:x} phys_addr={:x} opcode={:02x} {:02x} {:02x} {:02x}. \\\n                 Last jump at {:x} ({}) opcode={:02x} {:02x} {:02x} {:02x}",\n                addr,\n                addr + entry.len,\n                phys_address,\n                opcode & 0xFF,\n                opcode >> 8 & 0xFF,\n                opcode >> 16 & 0xFF,\n                opcode >> 16 & 0xFF,\n                last_jump_addr,\n                last_jump_type,\n                last_jump_opcode & 0xFF,\n                last_jump_opcode >> 8 & 0xFF,\n                last_jump_opcode >> 16 & 0xFF,\n                last_jump_opcode >> 16 & 0xFF,\n            );\n        }\n    }\n}\n\n#[no_mangle]\n#[cfg(feature = "profiler")]\npub fn debug_set_dispatcher_target(_target_index: i32) {\n    //dbg_log!("About to call dispatcher target_index={}", target_index);\n}\n\n#[no_mangle]\n#[cfg(feature = "profiler")]\npub fn check_dispatcher_target(target_index: i32, max: i32) {\n    //dbg_log!("Dispatcher called target={}", target_index);\n    dbg_assert!(target_index >= 0);\n    dbg_assert!(target_index < max);\n}\n\n#[no_mangle]\n#[cfg(feature = "profiler")]\npub fn enter_basic_block(phys_eip: u32) {\n    let eip =\n        unsafe { cpu::translate_address_read(*global_pointers::instruction_pointer).unwrap() };\n    if Page::page_of(eip) != Page::page_of(phys_eip) {\n        dbg_log!(\n            "enter basic block failed block=0x{:x} actual eip=0x{:x}",\n            phys_eip,\n            eip\n        );\n        panic!();\n    }\n}\n\n#[no_mangle]\n#[cfg(feature = "profiler")]\npub fn get_config(index: u32) -> u32 {\n    match index {\n        0 => MAX_PAGES as u32,\n        1 => JIT_USE_LOOP_SAFETY as u32,\n        2 => MAX_EXTRA_BASIC_BLOCKS as u32,\n        _ => 0,\n    }\n}\n', 'message': 'Clean up\n', 'author_name': 'Fabian', 'lang': 'rs'}