diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 5175f760d1..31169b5de1 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -192,6 +192,14 @@ pub const Segment = struct { pub fn isPassive(segment: Segment) bool { return segment.flags & @enumToInt(Flag.WASM_DATA_SEGMENT_IS_PASSIVE) != 0; } + + /// For a given segment, determines if it needs passive initialization + fn needsPassiveInitialization(segment: Segment, import_mem: bool, name: []const u8) bool { + if (import_mem and !std.mem.eql(u8, name, ".bss")) { + return true; + } + return segment.isPassive(); + } }; pub const Export = struct { @@ -824,6 +832,178 @@ fn resolveSymbolsInArchives(wasm: *Wasm) !void { } } +fn setupInitMemoryFunction(wasm: *Wasm) !void { + // Passive segments are used to avoid memory being reinitialized on each + // thread's instantiation. These passive segments are initialized and + // dropped in __wasm_init_memory, which is registered as the start function + // We also initialize bss segments (using memory.fill) as part of this + // function. + if (!wasm.hasPassiveInitializationSegments()) { + return; + } + + const flag_address: u32 = if (wasm.base.options.shared_memory) address: { + // when we have passive initialization segments and shared memory + // `setupMemory` will create this symbol and set its virtual address. + const loc = wasm.findGlobalSymbol("__wasm_init_memory_flag").?; + break :address loc.getSymbol(wasm).virtual_address; + } else 0; + + var function_body = std.ArrayList(u8).init(wasm.base.allocator); + defer function_body.deinit(); + const writer = function_body.writer(); + + // we have 0 locals + try leb.writeULEB128(writer, @as(u32, 0)); + + if (wasm.base.options.shared_memory) { + // destination blocks + // based on values we jump to corresponding label + try writer.writeByte(std.wasm.opcode(.block)); // $drop + try writer.writeByte(std.wasm.block_empty); // block type + + try writer.writeByte(std.wasm.opcode(.block)); // $wait + try writer.writeByte(std.wasm.block_empty); // block type + + try writer.writeByte(std.wasm.opcode(.block)); // $init + try writer.writeByte(std.wasm.block_empty); // block type + + // atomically check + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, flag_address); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, @as(u32, 0)); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, @as(u32, 1)); + try writer.writeByte(std.wasm.opcode(.atomics_prefix)); + try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_rmw_cmpxchg)); + try leb.writeULEB128(writer, @as(u32, 2)); // alignment + try leb.writeULEB128(writer, @as(u32, 0)); // offset + + // based on the value from the atomic check, jump to the label. + try writer.writeByte(std.wasm.opcode(.br_table)); + try leb.writeULEB128(writer, @as(u32, 2)); // length of the table (we have 3 blocks but because of the mandatory default the length is 2). + try leb.writeULEB128(writer, @as(u32, 0)); // $init + try leb.writeULEB128(writer, @as(u32, 1)); // $wait + try leb.writeULEB128(writer, @as(u32, 2)); // $drop + try writer.writeByte(std.wasm.opcode(.end)); + } + + var it = wasm.data_segments.iterator(); + var segment_index: u32 = 0; + while (it.next()) |entry| : (segment_index += 1) { + const segment: Segment = wasm.segments.items[entry.value_ptr.*]; + if (segment.needsPassiveInitialization(wasm.base.options.import_memory, entry.key_ptr.*)) { + // For passive BSS segments we can simple issue a memory.fill(0). + // For non-BSS segments we do a memory.init. Both these + // instructions take as their first argument the destination + // address. + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, segment.offset); + + if (wasm.base.options.shared_memory and std.mem.eql(u8, entry.key_ptr.*, ".tdata")) { + // When we initialize the TLS segment we also set the `__tls_base` + // global. This allows the runtime to use this static copy of the + // TLS data for the first/main thread. + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, segment.offset); + try writer.writeByte(std.wasm.opcode(.global_set)); + const loc = wasm.findGlobalSymbol("__tls_base").?; + try leb.writeULEB128(writer, loc.getSymbol(wasm).index); + } + + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, @as(u32, 0)); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, segment.size); + try writer.writeByte(std.wasm.opcode(.misc_prefix)); + if (std.mem.eql(u8, entry.key_ptr.*, ".bss")) { + // fill bss segment with zeroes + try leb.writeULEB128(writer, std.wasm.miscOpcode(.memory_fill)); + } else { + // initialize the segment + try leb.writeULEB128(writer, std.wasm.miscOpcode(.memory_init)); + try leb.writeULEB128(writer, segment_index); + } + try writer.writeByte(0); // memory index immediate + } + } + + if (wasm.base.options.shared_memory) { + // we set the init memory flag to value '2' + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, flag_address); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, @as(u32, 2)); + try writer.writeByte(std.wasm.opcode(.atomics_prefix)); + try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_store)); + try leb.writeULEB128(writer, @as(u32, 2)); // alignment + try leb.writeULEB128(writer, @as(u32, 0)); // offset + + // notify any waiters for segment initialization completion + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, flag_address); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeILEB128(writer, @as(i32, -1)); // number of waiters + try writer.writeByte(std.wasm.opcode(.atomics_prefix)); + try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.memory_atomic_notify)); + try leb.writeULEB128(writer, @as(u32, 2)); // alignment + try leb.writeULEB128(writer, @as(u32, 0)); // offset + try writer.writeByte(std.wasm.opcode(.drop)); + + // branch and drop segments + try writer.writeByte(std.wasm.opcode(.br)); + try leb.writeULEB128(writer, @as(u32, 1)); + + // wait for thread to initialize memory segments + try writer.writeByte(std.wasm.opcode(.end)); // end $wait + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, flag_address); + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeULEB128(writer, @as(u32, 1)); // expected flag value + try writer.writeByte(std.wasm.opcode(.i32_const)); + try leb.writeILEB128(writer, @as(i32, -1)); // timeout + try writer.writeByte(std.wasm.opcode(.atomics_prefix)); + try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.memory_atomic_wait32)); + try leb.writeULEB128(writer, @as(u32, 2)); // alignment + try leb.writeULEB128(writer, @as(u32, 0)); // offset + try writer.writeByte(std.wasm.opcode(.drop)); + + try writer.writeByte(std.wasm.opcode(.end)); // end $drop + } + + it.reset(); + segment_index = 0; + while (it.next()) |entry| : (segment_index += 1) { + const name = entry.key_ptr.*; + const segment: Segment = wasm.segments.items[entry.value_ptr.*]; + if (segment.needsPassiveInitialization(wasm.base.options.import_memory, name) and + !std.mem.eql(u8, name, ".bss")) + { + // The TLS region should not be dropped since its is needed + // during the initialization of each thread (__wasm_init_tls). + if (wasm.base.options.shared_memory and std.mem.eql(u8, name, ".tdata")) { + continue; + } + + try writer.writeByte(std.wasm.opcode(.misc_prefix)); + try leb.writeULEB128(writer, std.wasm.miscOpcode(.data_drop)); + try leb.writeULEB128(writer, segment_index); + } + } + + // End of the function body + try writer.writeByte(std.wasm.opcode(.end)); + + try wasm.createSyntheticFunction( + "__wasm_init_memory", + std.wasm.Type{ .params = &.{}, .returns = &.{} }, + &function_body, + ); +} + +/// Constructs a synthetic function that performs runtime relocations for +/// TLS symbols. This function is called by `__wasm_init_tls`. fn setupTLSRelocationsFunction(wasm: *Wasm) !void { // When we have TLS GOT entries and shared memory is enabled, // we must perform runtime relocations or else we don't create the function. @@ -2469,6 +2649,16 @@ fn setupMemory(wasm: *Wasm) !void { offset += segment.size; } + // create the memory init flag which is used by the init memory function + if (wasm.base.options.shared_memory and wasm.hasPassiveInitializationSegments()) { + // align to pointer size + memory_ptr = mem.alignForwardGeneric(u64, memory_ptr, 4); + const loc = try wasm.createSyntheticSymbol("__wasm_init_memory_flag", .data); + const sym = loc.getSymbol(wasm); + sym.virtual_address = @intCast(u32, memory_ptr); + memory_ptr += 4; + } + if (!place_stack_first and !is_obj) { memory_ptr = std.mem.alignForwardGeneric(u64, memory_ptr, stack_alignment); memory_ptr += stack_size; @@ -3000,6 +3190,7 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l try wasm.mergeSections(); try wasm.mergeTypes(); try wasm.initializeCallCtorsFunction(); + try wasm.setupInitMemoryFunction(); try wasm.setupTLSRelocationsFunction(); try wasm.initializeTLSFunction(); try wasm.setupExports(); @@ -3121,6 +3312,7 @@ pub fn flushModule(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod try wasm.mergeSections(); try wasm.mergeTypes(); try wasm.initializeCallCtorsFunction(); + try wasm.setupInitMemoryFunction(); try wasm.setupTLSRelocationsFunction(); try wasm.initializeTLSFunction(); try wasm.setupExports(); @@ -4473,6 +4665,17 @@ fn emitDataRelocations( try writeCustomSectionHeader(binary_bytes.items, header_offset, size); } +fn hasPassiveInitializationSegments(wasm: *const Wasm) bool { + var it = wasm.data_segments.iterator(); + while (it.next()) |entry| { + const segment: Segment = wasm.segments.items[entry.value_ptr.*]; + if (segment.needsPassiveInitialization(wasm.base.options.import_memory, entry.key_ptr.*)) { + return true; + } + } + return false; +} + pub fn getTypeIndex(wasm: *const Wasm, func_type: std.wasm.Type) ?u32 { var index: u32 = 0; while (index < wasm.func_types.items.len) : (index += 1) {