diff --git a/doc/langref.html.in b/doc/langref.html.in index 29e9e0cc1d..d8887cd51a 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -8674,6 +8674,50 @@ test "@wasmMemoryGrow" { {#see_also|@ctz|@clz#} {#header_close#} + {#header_open|@prefetch#} + 
{#syntax#}@prefetch(ptr: anytype, comptime options: std.builtin.PrefetchOptions){#endsyntax#}
+

+ This builtin tells the compiler to emit a prefetch instruction if supported by the + target CPU. If the target CPU does not support the requested prefetch instruction, + this builtin is a noop. This function has no effect on the behavior of the program, + only on the performance characteristics. +

+

+ The {#syntax#}ptr{#endsyntax#} argument may be any pointer type and determines the memory + address to prefetch. This function does not dereference the pointer, it is perfectly legal + to pass a pointer to invalid memory to this function and no illegal behavior will result. +

+

+ The {#syntax#}options{#endsyntax#} argument is the following struct: +

+ {#code_begin|syntax|builtin#} +/// This data structure is used by the Zig language code generation and +/// therefore must be kept in sync with the compiler implementation. +pub const PrefetchOptions = struct { + /// Whether the prefetch should prepare for a read or a write. + rw: Rw = .read, + /// 0 means no temporal locality. That is, the data can be immediately + /// dropped from the cache after it is accessed. + /// + /// 3 means high temporal locality. That is, the data should be kept in + /// the cache as it is likely to be accessed again soon. + locality: u2 = 3, + /// The cache that the prefetch should be preformed on. + cache: Cache = .data, + + pub const Rw = enum { + read, + write, + }; + + pub const Cache = enum { + instruction, + data, + }; +}; + {#code_end#} + {#header_close#} + {#header_open|@ptrCast#} 
{#syntax#}@ptrCast(comptime DestType: type, value: anytype) DestType{#endsyntax#}

diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig index 9ce8c1c38e..2ffbdaa5ad 100644 --- a/lib/std/builtin.zig +++ b/lib/std/builtin.zig @@ -651,6 +651,31 @@ pub const CallOptions = struct { }; }; +/// This data structure is used by the Zig language code generation and +/// therefore must be kept in sync with the compiler implementation. +pub const PrefetchOptions = struct { + /// Whether the prefetch should prepare for a read or a write. + rw: Rw = .read, + /// 0 means no temporal locality. That is, the data can be immediately + /// dropped from the cache after it is accessed. + /// + /// 3 means high temporal locality. That is, the data should be kept in + /// the cache as it is likely to be accessed again soon. + locality: u2 = 3, + /// The cache that the prefetch should be preformed on. + cache: Cache = .data, + + pub const Rw = enum { + read, + write, + }; + + pub const Cache = enum { + instruction, + data, + }; +}; + /// This data structure is used by the Zig language code generation and /// therefore must be kept in sync with the compiler implementation. pub const ExportOptions = struct { diff --git a/src/AstGen.zig b/src/AstGen.zig index 54682f9341..30efa47f4c 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -6739,7 +6739,6 @@ fn builtinCall( } } - // zig fmt: off switch (info.tag) { .import => { const node_tags = tree.nodes.items(.tag); @@ -6768,7 +6767,7 @@ fn builtinCall( astgen.extra.items[extra_index] = @enumToInt(param_ref); extra_index += 1; } - const result = try gz.addExtendedMultiOpPayloadIndex(.compile_log,payload_index, params.len); + const result = try gz.addExtendedMultiOpPayloadIndex(.compile_log, payload_index, params.len); return rvalue(gz, rl, result, node); }, .field => { @@ -6784,11 +6783,14 @@ fn builtinCall( }); return rvalue(gz, rl, result, node); }, + + // zig fmt: off .as => return as( gz, scope, rl, node, params[0], params[1]), .bit_cast => return bitCast( gz, scope, rl, node, params[0], params[1]), .TypeOf => return typeOf( gz, scope, rl, node, params), .union_init => return unionInit(gz, scope, rl, node, params), .c_import => return cImport( gz, scope, node, params[0]), + // zig fmt: on .@"export" => { const node_tags = tree.nodes.items(.tag); @@ -6858,9 +6860,7 @@ fn builtinCall( const field_ident = dot_token + 1; decl_name = try astgen.identAsString(field_ident); }, - else => return astgen.failNode( - params[0], "symbol to export must identify a declaration", .{}, - ), + else => return astgen.failNode(params[0], "symbol to export must identify a declaration", .{}), } const options = try comptimeExpr(gz, scope, .{ .ty = .export_options_type }, params[1]); _ = try gz.addPlNode(.@"export", node, Zir.Inst.Export{ @@ -6888,6 +6888,7 @@ fn builtinCall( .breakpoint => return simpleNoOpVoid(gz, rl, node, .breakpoint), + // zig fmt: off .This => return rvalue(gz, rl, try gz.addNodeExtended(.this, node), node), .return_address => return rvalue(gz, rl, try gz.addNodeExtended(.ret_addr, node), node), .src => return rvalue(gz, rl, try gz.addNodeExtended(.builtin_src, node), node), @@ -6942,6 +6943,8 @@ fn builtinCall( .err_set_cast => return typeCast(gz, scope, rl, node, params[0], params[1], .err_set_cast), .ptr_cast => return typeCast(gz, scope, rl, node, params[0], params[1], .ptr_cast), .truncate => return typeCast(gz, scope, rl, node, params[0], params[1], .truncate), + // zig fmt: on + .align_cast => { const dest_align = try comptimeExpr(gz, scope, align_rl, params[0]); const rhs = try expr(gz, scope, .none, params[1]); @@ -6952,6 +6955,7 @@ fn builtinCall( return rvalue(gz, rl, result, node); }, + // zig fmt: off .has_decl => return hasDeclOrField(gz, scope, rl, node, params[0], params[1], .has_decl), .has_field => return hasDeclOrField(gz, scope, rl, node, params[0], params[1], .has_field), @@ -6978,6 +6982,7 @@ fn builtinCall( .cmpxchg_strong => return cmpxchg(gz, scope, rl, node, params, .cmpxchg_strong), .cmpxchg_weak => return cmpxchg(gz, scope, rl, node, params, .cmpxchg_weak), + // zig fmt: on .wasm_memory_size => { const operand = try expr(gz, scope, .{ .ty = .u32_type }, params[0]); @@ -7221,8 +7226,17 @@ fn builtinCall( }); return rvalue(gz, rl, result, node); }, + .prefetch => { + const ptr = try expr(gz, scope, .none, params[0]); + const options = try comptimeExpr(gz, scope, .{ .ty = .prefetch_options_type }, params[1]); + const result = try gz.addExtendedPayload(.prefetch, Zir.Inst.BinNode{ + .node = gz.nodeIndexToRelative(node), + .lhs = ptr, + .rhs = options, + }); + return rvalue(gz, rl, result, node); + }, } - // zig fmt: on } fn simpleNoOpVoid( diff --git a/src/BuiltinFn.zig b/src/BuiltinFn.zig index 7c5dde03d1..3bf7224fab 100644 --- a/src/BuiltinFn.zig +++ b/src/BuiltinFn.zig @@ -67,6 +67,7 @@ pub const Tag = enum { mul_with_overflow, panic, pop_count, + prefetch, ptr_cast, ptr_to_int, rem, @@ -615,6 +616,13 @@ pub const list = list: { .param_count = 2, }, }, + .{ + "@prefetch", + .{ + .tag = .prefetch, + .param_count = 2, + }, + }, .{ "@ptrCast", .{ diff --git a/src/Sema.zig b/src/Sema.zig index 89c6de2e0b..12578c56c5 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -1042,6 +1042,7 @@ fn zirExtended(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai .c_define => return sema.zirCDefine( block, extended), .wasm_memory_size => return sema.zirWasmMemorySize( block, extended), .wasm_memory_grow => return sema.zirWasmMemoryGrow( block, extended), + .prefetch => return sema.zirPrefetch( block, extended), // zig fmt: on } } @@ -11104,6 +11105,16 @@ fn zirWasmMemoryGrow( return sema.fail(block, src, "TODO: implement Sema.zirWasmMemoryGrow", .{}); } +fn zirPrefetch( + sema: *Sema, + block: *Block, + extended: Zir.Inst.Extended.InstData, +) CompileError!Air.Inst.Ref { + const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data; + const src: LazySrcLoc = .{ .node_offset = extra.node }; + return sema.fail(block, src, "TODO: implement Sema.zirPrefetch", .{}); +} + fn zirBuiltinExtern( sema: *Sema, block: *Block, @@ -14231,6 +14242,7 @@ fn resolveTypeFields(sema: *Sema, block: *Block, src: LazySrcLoc, ty: Type) Comp .float_mode => return sema.resolveBuiltinTypeFields(block, src, "FloatMode"), .reduce_op => return sema.resolveBuiltinTypeFields(block, src, "ReduceOp"), .call_options => return sema.resolveBuiltinTypeFields(block, src, "CallOptions"), + .prefetch_options => return sema.resolveBuiltinTypeFields(block, src, "PrefetchOptions"), .@"union", .union_tagged => { const union_obj = ty.cast(Type.Payload.Union).?.data; @@ -14819,6 +14831,7 @@ fn typeHasOnePossibleValue( .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -15032,6 +15045,7 @@ pub fn addType(sema: *Sema, ty: Type) !Air.Inst.Ref { .float_mode => return .float_mode_type, .reduce_op => return .reduce_op_type, .call_options => return .call_options_type, + .prefetch_options => return .prefetch_options_type, .export_options => return .export_options_type, .extern_options => return .extern_options_type, .type_info => return .type_info_type, diff --git a/src/Zir.zig b/src/Zir.zig index e32200f78c..d568cb2e8c 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -1572,6 +1572,9 @@ pub const Inst = struct { wasm_memory_size, /// `operand` is payload index to `BinNode`. wasm_memory_grow, + /// The `@prefetch` builtin. + /// `operand` is payload index to `BinNode`. + prefetch, pub const InstData = struct { opcode: Extended, @@ -1648,6 +1651,7 @@ pub const Inst = struct { float_mode_type, reduce_op_type, call_options_type, + prefetch_options_type, export_options_type, extern_options_type, type_info_type, @@ -1917,6 +1921,10 @@ pub const Inst = struct { .ty = Type.initTag(.type), .val = Value.initTag(.call_options_type), }, + .prefetch_options_type = .{ + .ty = Type.initTag(.type), + .val = Value.initTag(.prefetch_options_type), + }, .export_options_type = .{ .ty = Type.initTag(.type), .val = Value.initTag(.export_options_type), diff --git a/src/print_zir.zig b/src/print_zir.zig index 401d41cd50..680ca55d09 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -477,7 +477,7 @@ const Writer = struct { try self.writeSrc(stream, src); }, - .builtin_extern, .c_define, .wasm_memory_grow => { + .builtin_extern, .c_define, .wasm_memory_grow, .prefetch => { const inst_data = self.code.extraData(Zir.Inst.BinNode, extended.operand).data; const src: LazySrcLoc = .{ .node_offset = inst_data.node }; try self.writeInstRef(stream, inst_data.lhs); diff --git a/src/stage1/all_types.hpp b/src/stage1/all_types.hpp index 0a3e52f4f4..1c50134194 100644 --- a/src/stage1/all_types.hpp +++ b/src/stage1/all_types.hpp @@ -898,6 +898,18 @@ struct AstNodeFnCallExpr { bool seen; // used by @compileLog }; +// Must be kept in sync with std.builtin.PrefetchOptions.Rw +enum PrefetchRw { + PrefetchRwRead, + PrefetchRwWrite, +}; + +// Must be kept in sync with std.builtin.PrefetchOptions.Cache +enum PrefetchCache { + PrefetchCacheInstruction, + PrefetchCacheData, +}; + struct AstNodeArrayAccessExpr { AstNode *array_ref_expr; AstNode *subscript; @@ -1818,6 +1830,7 @@ enum BuiltinFnId { BuiltinFnIdReduce, BuiltinFnIdMaximum, BuiltinFnIdMinimum, + BuiltinFnIdPrefetch, }; struct BuiltinFnEntry { @@ -2021,6 +2034,7 @@ struct CodeGen { LLVMValueRef return_err_fn; LLVMValueRef wasm_memory_size; LLVMValueRef wasm_memory_grow; + LLVMValueRef prefetch; LLVMTypeRef anyframe_fn_type; // reminder: hash tables must be initialized before use @@ -2647,6 +2661,7 @@ enum Stage1ZirInstId : uint8_t { Stage1ZirInstIdWasmMemorySize, Stage1ZirInstIdWasmMemoryGrow, Stage1ZirInstIdSrc, + Stage1ZirInstIdPrefetch, }; // ir_render_* functions in codegen.cpp consume Gen instructions and produce LLVM IR. @@ -2743,6 +2758,7 @@ enum Stage1AirInstId : uint8_t { Stage1AirInstIdWasmMemorySize, Stage1AirInstIdWasmMemoryGrow, Stage1AirInstIdExtern, + Stage1AirInstIdPrefetch, }; struct Stage1ZirInst { @@ -3683,6 +3699,24 @@ struct Stage1ZirInstSrc { Stage1ZirInst base; }; +struct Stage1ZirInstPrefetch { + Stage1ZirInst base; + + Stage1ZirInst *ptr; + Stage1ZirInst *options; +}; + +struct Stage1AirInstPrefetch { + Stage1AirInst base; + + Stage1AirInst *ptr; + PrefetchRw rw; + // Must be in the range 0-3 inclusive + uint8_t locality; + PrefetchCache cache; +}; + + struct Stage1ZirInstSlice { Stage1ZirInst base; diff --git a/src/stage1/astgen.cpp b/src/stage1/astgen.cpp index 8fbd02c688..ee59aef04a 100644 --- a/src/stage1/astgen.cpp +++ b/src/stage1/astgen.cpp @@ -349,6 +349,8 @@ void destroy_instruction_src(Stage1ZirInst *inst) { return heap::c_allocator.destroy(reinterpret_cast(inst)); case Stage1ZirInstIdSrc: return heap::c_allocator.destroy(reinterpret_cast(inst)); + case Stage1ZirInstIdPrefetch: + return heap::c_allocator.destroy(reinterpret_cast(inst)); } zig_unreachable(); } @@ -941,6 +943,10 @@ static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstSrc *) { return Stage1ZirInstIdSrc; } +static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstPrefetch *) { + return Stage1ZirInstIdPrefetch; +} + template static T *ir_create_instruction(Stage1AstGen *ag, Scope *scope, AstNode *source_node) { T *special_instruction = heap::c_allocator.create(); @@ -2870,6 +2876,21 @@ static Stage1ZirInst *ir_build_src(Stage1AstGen *ag, Scope *scope, AstNode *sour return &instruction->base; } +static Stage1ZirInst *ir_build_prefetch(Stage1AstGen *ag, Scope *scope, AstNode *source_node, + Stage1ZirInst *ptr, Stage1ZirInst *options) +{ + Stage1ZirInstPrefetch *prefetch_instruction = ir_build_instruction( + ag, scope, source_node); + prefetch_instruction->ptr = ptr; + prefetch_instruction->options = options; + + ir_ref_instruction(ptr, ag->current_basic_block); + ir_ref_instruction(options, ag->current_basic_block); + + return &prefetch_instruction->base; +} + + static void ir_count_defers(Stage1AstGen *ag, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -5416,6 +5437,29 @@ static Stage1ZirInst *astgen_builtin_fn_call(Stage1AstGen *ag, Scope *scope, Ast Stage1ZirInst *src_inst = ir_build_src(ag, scope, node); return ir_lval_wrap(ag, scope, src_inst, lval, result_loc); } + case BuiltinFnIdPrefetch: + { + ZigType *options_type = get_builtin_type(ag->codegen, "PrefetchOptions"); + Stage1ZirInst *options_type_inst = ir_build_const_type(ag, scope, node, options_type); + ResultLocCast *result_loc_cast = ir_build_cast_result_loc(ag, options_type_inst, no_result_loc()); + + AstNode *ptr_node = node->data.fn_call_expr.params.at(0); + Stage1ZirInst *ptr_value = astgen_node(ag, ptr_node, scope); + if (ptr_value == ag->codegen->invalid_inst_src) + return ptr_value; + + AstNode *options_node = node->data.fn_call_expr.params.at(1); + Stage1ZirInst *options_value = astgen_node_extra(ag, options_node, + scope, LValNone, &result_loc_cast->base); + if (options_value == ag->codegen->invalid_inst_src) + return options_value; + + Stage1ZirInst *casted_options_value = ir_build_implicit_cast( + ag, scope, options_node, options_value, result_loc_cast); + + Stage1ZirInst *ir_extern = ir_build_prefetch(ag, scope, node, ptr_value, casted_options_value); + return ir_lval_wrap(ag, scope, ir_extern, lval, result_loc); + } } zig_unreachable(); } diff --git a/src/stage1/codegen.cpp b/src/stage1/codegen.cpp index cba820037e..44600ffa45 100644 --- a/src/stage1/codegen.cpp +++ b/src/stage1/codegen.cpp @@ -1139,6 +1139,24 @@ static LLVMValueRef gen_wasm_memory_grow(CodeGen *g) { return g->wasm_memory_grow; } +static LLVMValueRef gen_prefetch(CodeGen *g) { + if (g->prefetch) + return g->prefetch; + + // declare void @llvm.prefetch(i8*, i32, i32, i32) + LLVMTypeRef param_types[] = { + LLVMPointerType(LLVMInt8Type(), 0), + LLVMInt32Type(), + LLVMInt32Type(), + LLVMInt32Type(), + }; + LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false); + g->prefetch = LLVMAddFunction(g->module, "llvm.prefetch.p0i8", fn_type); + assert(LLVMGetIntrinsicID(g->prefetch)); + + return g->prefetch; +} + static LLVMValueRef get_stacksave_fn_val(CodeGen *g) { if (g->stacksave_fn_val) return g->stacksave_fn_val; @@ -5899,6 +5917,52 @@ static LLVMValueRef ir_render_wasm_memory_grow(CodeGen *g, Stage1Air *executable return val; } +static LLVMValueRef ir_render_prefetch(CodeGen *g, Stage1Air *executable, Stage1AirInstPrefetch *instruction) { + static_assert(PrefetchRwRead == 0, ""); + static_assert(PrefetchRwWrite == 1, ""); + assert(instruction->rw == PrefetchRwRead || instruction->rw == PrefetchRwWrite); + + assert(instruction->locality >= 0 && instruction->locality <= 3); + + static_assert(PrefetchCacheInstruction == 0, ""); + static_assert(PrefetchCacheData == 1, ""); + assert(instruction->cache == PrefetchCacheData || instruction->cache == PrefetchCacheInstruction); + + // LLVM fails during codegen of instruction cache prefetchs for these architectures. + // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported by the target. + // To work around this, simply don't emit llvm.prefetch in this case. + // See https://bugs.llvm.org/show_bug.cgi?id=21037 + if (instruction->cache == PrefetchCacheInstruction) { + switch (g->zig_target->arch) { + case ZigLLVM_x86: + case ZigLLVM_x86_64: + return nullptr; + default: + break; + } + } + + // Another case of the same LLVM bug described above + if (instruction->rw == PrefetchRwWrite && instruction->cache == PrefetchCacheInstruction) { + switch (g->zig_target->arch) { + case ZigLLVM_arm: + return nullptr; + default: + break; + } + + } + + LLVMValueRef params[] = { + LLVMBuildBitCast(g->builder, ir_llvm_value(g, instruction->ptr), LLVMPointerType(LLVMInt8Type(), 0), ""), + LLVMConstInt(LLVMInt32Type(), instruction->rw, false), + LLVMConstInt(LLVMInt32Type(), instruction->locality, false), + LLVMConstInt(LLVMInt32Type(), instruction->cache, false), + }; + LLVMValueRef val = LLVMBuildCall(g->builder, gen_prefetch(g), params, 4, ""); + return val; +} + static LLVMValueRef ir_render_slice(CodeGen *g, Stage1Air *executable, Stage1AirInstSlice *instruction) { Error err; @@ -7150,6 +7214,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, Stage1Air *executable, Sta return ir_render_wasm_memory_grow(g, executable, (Stage1AirInstWasmMemoryGrow *) instruction); case Stage1AirInstIdExtern: return ir_render_extern(g, executable, (Stage1AirInstExtern *) instruction); + case Stage1AirInstIdPrefetch: + return ir_render_prefetch(g, executable, (Stage1AirInstPrefetch *) instruction); } zig_unreachable(); } @@ -9120,6 +9186,7 @@ static void define_builtin_fns(CodeGen *g) { create_builtin_fn(g, BuiltinFnIdReduce, "reduce", 2); create_builtin_fn(g, BuiltinFnIdMaximum, "maximum", 2); create_builtin_fn(g, BuiltinFnIdMinimum, "minimum", 2); + create_builtin_fn(g, BuiltinFnIdPrefetch, "prefetch", 2); } static const char *bool_to_str(bool b) { diff --git a/src/stage1/ir.cpp b/src/stage1/ir.cpp index 34ff4a4f04..a78b8a3018 100644 --- a/src/stage1/ir.cpp +++ b/src/stage1/ir.cpp @@ -467,6 +467,8 @@ void destroy_instruction_gen(Stage1AirInst *inst) { return heap::c_allocator.destroy(reinterpret_cast(inst)); case Stage1AirInstIdExtern: return heap::c_allocator.destroy(reinterpret_cast(inst)); + case Stage1AirInstIdPrefetch: + return heap::c_allocator.destroy(reinterpret_cast(inst)); } zig_unreachable(); } @@ -1115,6 +1117,10 @@ static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstExtern *) { return Stage1AirInstIdExtern; } +static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstPrefetch *) { + return Stage1AirInstIdPrefetch; +} + template static T *ir_create_inst_gen(IrBuilderGen *irb, Scope *scope, AstNode *source_node) { T *special_instruction = heap::c_allocator.create(); @@ -24853,6 +24859,52 @@ static Stage1AirInst *ir_analyze_instruction_src(IrAnalyze *ira, Stage1ZirInstSr return ir_const_move(ira, instruction->base.scope, instruction->base.source_node, result); } +static Stage1AirInst *ir_analyze_instruction_prefetch(IrAnalyze *ira, Stage1ZirInstPrefetch *instruction) { + Stage1AirInst *ptr = instruction->ptr->child; + if (type_is_invalid(ptr->value->type)) + return ira->codegen->invalid_inst_gen; + + Stage1AirInst *raw_options_inst = instruction->options->child; + if (type_is_invalid(raw_options_inst->value->type)) + return ira->codegen->invalid_inst_gen; + + ZigType *options_type = get_builtin_type(ira->codegen, "PrefetchOptions"); + Stage1AirInst *options_inst = ir_implicit_cast(ira, raw_options_inst, options_type); + if (type_is_invalid(options_inst->value->type)) + return ira->codegen->invalid_inst_gen; + + ZigValue *options_val = ir_resolve_const(ira, options_inst, UndefBad); + if (options_val == nullptr) + return ira->codegen->invalid_inst_gen; + + ZigValue *rw_val = get_const_field(ira, options_inst->source_node, options_val, "rw", 0); + if (rw_val == nullptr) + return ira->codegen->invalid_inst_gen; + PrefetchRw rw = (PrefetchRw)bigint_as_u8(&rw_val->data.x_enum_tag); + + ZigValue *locality_val = get_const_field(ira, options_inst->source_node, options_val, "locality", 1); + if (locality_val == nullptr) + return ira->codegen->invalid_inst_gen; + uint8_t locality = bigint_as_u8(&locality_val->data.x_bigint); + assert(locality <= 3); + + ZigValue *cache_val = get_const_field(ira, options_inst->source_node, options_val, "cache", 2); + if (cache_val == nullptr) + return ira->codegen->invalid_inst_gen; + PrefetchCache cache = (PrefetchCache)bigint_as_u8(&cache_val->data.x_enum_tag); + + Stage1AirInstPrefetch *air_instruction = ir_build_inst_void(&ira->new_irb, + instruction->base.scope, instruction->base.source_node); + air_instruction->ptr = ptr; + air_instruction->rw = rw; + air_instruction->locality = locality; + air_instruction->cache = cache; + + ir_ref_inst_gen(ptr); + + return &air_instruction->base; +} + static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst *instruction) { switch (instruction->id) { case Stage1ZirInstIdInvalid: @@ -25138,6 +25190,8 @@ static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst return ir_analyze_instruction_wasm_memory_grow(ira, (Stage1ZirInstWasmMemoryGrow *)instruction); case Stage1ZirInstIdSrc: return ir_analyze_instruction_src(ira, (Stage1ZirInstSrc *)instruction); + case Stage1ZirInstIdPrefetch: + return ir_analyze_instruction_prefetch(ira, (Stage1ZirInstPrefetch *)instruction); } zig_unreachable(); } @@ -25305,6 +25359,7 @@ bool ir_inst_gen_has_side_effects(Stage1AirInst *instruction) { case Stage1AirInstIdSpillBegin: case Stage1AirInstIdWasmMemoryGrow: case Stage1AirInstIdExtern: + case Stage1AirInstIdPrefetch: return true; case Stage1AirInstIdPhi: @@ -25444,6 +25499,7 @@ bool ir_inst_src_has_side_effects(Stage1ZirInst *instruction) { case Stage1ZirInstIdAwait: case Stage1ZirInstIdSpillBegin: case Stage1ZirInstIdWasmMemoryGrow: + case Stage1ZirInstIdPrefetch: return true; case Stage1ZirInstIdPhi: diff --git a/src/stage1/ir_print.cpp b/src/stage1/ir_print.cpp index f92f146d84..5c7727da0c 100644 --- a/src/stage1/ir_print.cpp +++ b/src/stage1/ir_print.cpp @@ -371,6 +371,8 @@ const char* ir_inst_src_type_str(Stage1ZirInstId id) { return "SrcWasmMemoryGrow"; case Stage1ZirInstIdSrc: return "SrcSrc"; + case Stage1ZirInstIdPrefetch: + return "SrcPrefetch"; } zig_unreachable(); } @@ -559,6 +561,8 @@ const char* ir_inst_gen_type_str(Stage1AirInstId id) { return "GenWasmMemoryGrow"; case Stage1AirInstIdExtern: return "GenExtern"; + case Stage1AirInstIdPrefetch: + return "GenPrefetch"; } zig_unreachable(); } @@ -2436,6 +2440,18 @@ static void ir_print_extern(IrPrintSrc *irp, Stage1ZirInstExtern *instruction) { fprintf(irp->f, ")"); } +static void ir_print_prefetch(IrPrintSrc *irp, Stage1ZirInstPrefetch *instruction) { + fprintf(irp->f, "@prefetch("); + ir_print_other_inst_src(irp, instruction->ptr); + fprintf(irp->f, ","); + ir_print_other_inst_src(irp, instruction->options); + fprintf(irp->f, ")"); +} + +static void ir_print_prefetch(IrPrintGen *irp, Stage1AirInstPrefetch *instruction) { + fprintf(irp->f, "@prefetch(...)"); +} + static void ir_print_error_return_trace(IrPrintSrc *irp, Stage1ZirInstErrorReturnTrace *instruction) { fprintf(irp->f, "@errorReturnTrace("); switch (instruction->optional) { @@ -3108,6 +3124,9 @@ static void ir_print_inst_src(IrPrintSrc *irp, Stage1ZirInst *instruction, bool case Stage1ZirInstIdSrc: ir_print_builtin_src(irp, (Stage1ZirInstSrc *)instruction); break; + case Stage1ZirInstIdPrefetch: + ir_print_prefetch(irp, (Stage1ZirInstPrefetch *)instruction); + break; } fprintf(irp->f, "\n"); } @@ -3387,6 +3406,9 @@ static void ir_print_inst_gen(IrPrintGen *irp, Stage1AirInst *instruction, bool case Stage1AirInstIdExtern: ir_print_extern(irp, (Stage1AirInstExtern *)instruction); break; + case Stage1AirInstIdPrefetch: + ir_print_prefetch(irp, (Stage1AirInstPrefetch *)instruction); + break; } fprintf(irp->f, "\n"); diff --git a/src/type.zig b/src/type.zig index a293848b89..0a2b6f4675 100644 --- a/src/type.zig +++ b/src/type.zig @@ -123,6 +123,7 @@ pub const Type = extern union { .empty_struct_literal, .@"struct", .call_options, + .prefetch_options, .export_options, .extern_options, => return .Struct, @@ -798,6 +799,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -1027,6 +1029,7 @@ pub const Type = extern union { .float_mode => return writer.writeAll("std.builtin.FloatMode"), .reduce_op => return writer.writeAll("std.builtin.ReduceOp"), .call_options => return writer.writeAll("std.builtin.CallOptions"), + .prefetch_options => return writer.writeAll("std.builtin.PrefetchOptions"), .export_options => return writer.writeAll("std.builtin.ExportOptions"), .extern_options => return writer.writeAll("std.builtin.ExternOptions"), .type_info => return writer.writeAll("std.builtin.TypeInfo"), @@ -1318,6 +1321,7 @@ pub const Type = extern union { .float_mode => return "FloatMode", .reduce_op => return "ReduceOp", .call_options => return "CallOptions", + .prefetch_options => return "PrefetchOptions", .export_options => return "ExportOptions", .extern_options => return "ExternOptions", .type_info => return "TypeInfo", @@ -1376,6 +1380,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .manyptr_u8, @@ -1502,6 +1507,7 @@ pub const Type = extern union { .float_mode => return Value.initTag(.float_mode_type), .reduce_op => return Value.initTag(.reduce_op_type), .call_options => return Value.initTag(.call_options_type), + .prefetch_options => return Value.initTag(.prefetch_options_type), .export_options => return Value.initTag(.export_options_type), .extern_options => return Value.initTag(.extern_options_type), .type_info => return Value.initTag(.type_info_type), @@ -1563,6 +1569,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .@"anyframe", @@ -1750,6 +1757,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, => return 1, @@ -1929,6 +1937,7 @@ pub const Type = extern union { .var_args_param => unreachable, .generic_poison => unreachable, .call_options => unreachable, // missing call to resolveTypeFields + .prefetch_options => unreachable, // missing call to resolveTypeFields .export_options => unreachable, // missing call to resolveTypeFields .extern_options => unreachable, // missing call to resolveTypeFields .type_info => unreachable, // missing call to resolveTypeFields @@ -2269,6 +2278,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -2794,6 +2804,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -3294,6 +3305,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -3502,6 +3514,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, => @panic("TODO resolve std.builtin types"), @@ -3577,6 +3590,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, => @panic("TODO resolve std.builtin types"), @@ -3701,6 +3715,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -3741,6 +3756,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, @@ -3801,6 +3817,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, => @panic("TODO resolve std.builtin types"), @@ -3862,6 +3879,7 @@ pub const Type = extern union { float_mode, reduce_op, call_options, + prefetch_options, export_options, extern_options, type_info, @@ -3989,6 +4007,7 @@ pub const Type = extern union { .float_mode, .reduce_op, .call_options, + .prefetch_options, .export_options, .extern_options, .type_info, diff --git a/src/value.zig b/src/value.zig index 02e27cd498..5df194d511 100644 --- a/src/value.zig +++ b/src/value.zig @@ -67,6 +67,7 @@ pub const Value = extern union { float_mode_type, reduce_op_type, call_options_type, + prefetch_options_type, export_options_type, extern_options_type, type_info_type, @@ -244,6 +245,7 @@ pub const Value = extern union { .float_mode_type, .reduce_op_type, .call_options_type, + .prefetch_options_type, .export_options_type, .extern_options_type, .type_info_type, @@ -434,6 +436,7 @@ pub const Value = extern union { .float_mode_type, .reduce_op_type, .call_options_type, + .prefetch_options_type, .export_options_type, .extern_options_type, .type_info_type, @@ -652,6 +655,7 @@ pub const Value = extern union { .float_mode_type => return out_stream.writeAll("std.builtin.FloatMode"), .reduce_op_type => return out_stream.writeAll("std.builtin.ReduceOp"), .call_options_type => return out_stream.writeAll("std.builtin.CallOptions"), + .prefetch_options_type => return out_stream.writeAll("std.builtin.PrefetchOptions"), .export_options_type => return out_stream.writeAll("std.builtin.ExportOptions"), .extern_options_type => return out_stream.writeAll("std.builtin.ExternOptions"), .type_info_type => return out_stream.writeAll("std.builtin.TypeInfo"), @@ -829,6 +833,7 @@ pub const Value = extern union { .float_mode_type => Type.initTag(.float_mode), .reduce_op_type => Type.initTag(.reduce_op), .call_options_type => Type.initTag(.call_options), + .prefetch_options_type => Type.initTag(.prefetch_options), .export_options_type => Type.initTag(.export_options), .extern_options_type => Type.initTag(.extern_options), .type_info_type => Type.initTag(.type_info), diff --git a/test/behavior.zig b/test/behavior.zig index 6c793fad53..4a28e1b07a 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -169,6 +169,7 @@ test { _ = @import("behavior/optional_stage1.zig"); _ = @import("behavior/pointers_stage1.zig"); _ = @import("behavior/popcount_stage1.zig"); + _ = @import("behavior/prefetch.zig"); _ = @import("behavior/ptrcast_stage1.zig"); _ = @import("behavior/reflection.zig"); _ = @import("behavior/saturating_arithmetic_stage1.zig"); diff --git a/test/behavior/prefetch.zig b/test/behavior/prefetch.zig new file mode 100644 index 0000000000..98dc72a976 --- /dev/null +++ b/test/behavior/prefetch.zig @@ -0,0 +1,27 @@ +const std = @import("std"); + +test "@prefetch()" { + var a: u32 = 42; + + @prefetch(&a, .{}); + + @prefetch(&a, .{ .rw = .read, .locality = 3, .cache = .data }); + @prefetch(&a, .{ .rw = .read, .locality = 2, .cache = .data }); + @prefetch(&a, .{ .rw = .read, .locality = 1, .cache = .data }); + @prefetch(&a, .{ .rw = .read, .locality = 0, .cache = .data }); + + @prefetch(&a, .{ .rw = .write, .locality = 3, .cache = .data }); + @prefetch(&a, .{ .rw = .write, .locality = 2, .cache = .data }); + @prefetch(&a, .{ .rw = .write, .locality = 1, .cache = .data }); + @prefetch(&a, .{ .rw = .write, .locality = 0, .cache = .data }); + + @prefetch(&a, .{ .rw = .read, .locality = 3, .cache = .instruction }); + @prefetch(&a, .{ .rw = .read, .locality = 2, .cache = .instruction }); + @prefetch(&a, .{ .rw = .read, .locality = 1, .cache = .instruction }); + @prefetch(&a, .{ .rw = .read, .locality = 0, .cache = .instruction }); + + @prefetch(&a, .{ .rw = .write, .locality = 3, .cache = .instruction }); + @prefetch(&a, .{ .rw = .write, .locality = 2, .cache = .instruction }); + @prefetch(&a, .{ .rw = .write, .locality = 1, .cache = .instruction }); + @prefetch(&a, .{ .rw = .write, .locality = 0, .cache = .instruction }); +}