diff --git a/lib/compiler_rt.zig b/lib/compiler_rt.zig index 98c9e2074b..9487ac01e6 100644 --- a/lib/compiler_rt.zig +++ b/lib/compiler_rt.zig @@ -1,8 +1,14 @@ +const builtin = @import("builtin"); + pub const panic = @import("compiler_rt/common.zig").panic; comptime { _ = @import("compiler_rt/atomics.zig"); - _ = @import("compiler_rt/lse_atomics.zig"); + + // macOS has these functions inside libSystem. + if (builtin.cpu.arch.isAARCH64() and !builtin.os.tag.isDarwin()) { + _ = @import("compiler_rt/lse_atomics.zig"); + } _ = @import("compiler_rt/addf3.zig"); _ = @import("compiler_rt/addhf3.zig"); diff --git a/lib/compiler_rt/lse_atomics.zig b/lib/compiler_rt/lse_atomics.zig index ee7e725f25..c87d80c857 100644 --- a/lib/compiler_rt/lse_atomics.zig +++ b/lib/compiler_rt/lse_atomics.zig @@ -1,46 +1,17 @@ -const std = @import("std"); +//! This file is generated by tools/gen_outline_atomics.zig. const builtin = @import("builtin"); -const arch = builtin.cpu.arch; -const is_test = builtin.is_test; -const target = std.Target; -const os_tag = builtin.os.tag; -const is_darwin = target.Os.Tag.isDarwin(os_tag); -const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse); -const linkage = if (is_test) - std.builtin.GlobalLinkage.Internal -else - std.builtin.GlobalLinkage.Strong; +const std = @import("std"); +const linkage = @import("./common.zig").linkage; +const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse); -fn cas1RelaxDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { +/// This default is overridden at runtime after inspecting CPU properties. +/// It is intentionally not exported in order to make the machine code that +/// uses it a statically predicted direct branch rather than using the PLT, +/// which ARM is concerned would have too much overhead. +var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse); + +fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1RelaxDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x00000000 + 0x000000 @@ -63,93 +34,8 @@ fn cas1RelaxDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn cas1RelaxNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1RelaxNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x00000000 + 0x000000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas1_relax = if (is_darwin) - if (has_lse) - cas1RelaxDarwinLse - else - cas1RelaxDarwinNolse -else if (has_lse) - cas1RelaxNondarwinLse -else - cas1RelaxNondarwinNolse; -fn swp1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x00000000 + 0x000000 @@ -168,86 +54,8 @@ fn swp1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn swp1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x00000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp1_relax = if (is_darwin) - if (has_lse) - swp1RelaxDarwinLse - else - swp1RelaxDarwinNolse -else if (has_lse) - swp1RelaxNondarwinLse -else - swp1RelaxNondarwinNolse; -fn ldadd1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000 @@ -267,88 +75,8 @@ fn ldadd1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldadd1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd1_relax = if (is_darwin) - if (has_lse) - ldadd1RelaxDarwinLse - else - ldadd1RelaxDarwinNolse -else if (has_lse) - ldadd1RelaxNondarwinLse -else - ldadd1RelaxNondarwinNolse; -fn ldclr1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000 @@ -368,88 +96,8 @@ fn ldclr1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldclr1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr1_relax = if (is_darwin) - if (has_lse) - ldclr1RelaxDarwinLse - else - ldclr1RelaxDarwinNolse -else if (has_lse) - ldclr1RelaxNondarwinLse -else - ldclr1RelaxNondarwinNolse; -fn ldeor1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000 @@ -469,88 +117,8 @@ fn ldeor1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldeor1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor1_relax = if (is_darwin) - if (has_lse) - ldeor1RelaxDarwinLse - else - ldeor1RelaxDarwinNolse -else if (has_lse) - ldeor1RelaxNondarwinLse -else - ldeor1RelaxNondarwinNolse; -fn ldset1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000 @@ -570,91 +138,8 @@ fn ldset1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldset1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset1_relax = if (is_darwin) - if (has_lse) - ldset1RelaxDarwinLse - else - ldset1RelaxDarwinNolse -else if (has_lse) - ldset1RelaxNondarwinLse -else - ldset1RelaxNondarwinNolse; -fn cas1AcqDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casab w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1AcqDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x00000000 + 0x400000 @@ -677,93 +162,8 @@ fn cas1AcqDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn cas1AcqNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casab w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1AcqNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x00000000 + 0x400000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas1_acq = if (is_darwin) - if (has_lse) - cas1AcqDarwinLse - else - cas1AcqDarwinNolse -else if (has_lse) - cas1AcqNondarwinLse -else - cas1AcqNondarwinNolse; -fn swp1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x00000000 + 0x800000 @@ -782,86 +182,8 @@ fn swp1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn swp1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x00000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp1_acq = if (is_darwin) - if (has_lse) - swp1AcqDarwinLse - else - swp1AcqDarwinNolse -else if (has_lse) - swp1AcqNondarwinLse -else - swp1AcqNondarwinNolse; -fn ldadd1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000 @@ -881,88 +203,8 @@ fn ldadd1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldadd1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd1_acq = if (is_darwin) - if (has_lse) - ldadd1AcqDarwinLse - else - ldadd1AcqDarwinNolse -else if (has_lse) - ldadd1AcqNondarwinLse -else - ldadd1AcqNondarwinNolse; -fn ldclr1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000 @@ -982,88 +224,8 @@ fn ldclr1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldclr1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr1_acq = if (is_darwin) - if (has_lse) - ldclr1AcqDarwinLse - else - ldclr1AcqDarwinNolse -else if (has_lse) - ldclr1AcqNondarwinLse -else - ldclr1AcqNondarwinNolse; -fn ldeor1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000 @@ -1083,88 +245,8 @@ fn ldeor1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldeor1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor1_acq = if (is_darwin) - if (has_lse) - ldeor1AcqDarwinLse - else - ldeor1AcqDarwinNolse -else if (has_lse) - ldeor1AcqNondarwinLse -else - ldeor1AcqNondarwinNolse; -fn ldset1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000 @@ -1184,91 +266,8 @@ fn ldset1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldset1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetab w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset1_acq = if (is_darwin) - if (has_lse) - ldset1AcqDarwinLse - else - ldset1AcqDarwinNolse -else if (has_lse) - ldset1AcqNondarwinLse -else - ldset1AcqNondarwinNolse; -fn cas1RelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caslb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1RelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x00000000 + 0x008000 @@ -1291,93 +290,8 @@ fn cas1RelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn cas1RelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caslb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1RelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x00000000 + 0x008000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas1_rel = if (is_darwin) - if (has_lse) - cas1RelDarwinLse - else - cas1RelDarwinNolse -else if (has_lse) - cas1RelNondarwinLse -else - cas1RelNondarwinNolse; -fn swp1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swplb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x00000000 + 0x400000 @@ -1396,86 +310,8 @@ fn swp1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn swp1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swplb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x00000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp1_rel = if (is_darwin) - if (has_lse) - swp1RelDarwinLse - else - swp1RelDarwinNolse -else if (has_lse) - swp1RelNondarwinLse -else - swp1RelNondarwinNolse; -fn ldadd1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000 @@ -1495,88 +331,8 @@ fn ldadd1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldadd1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd1_rel = if (is_darwin) - if (has_lse) - ldadd1RelDarwinLse - else - ldadd1RelDarwinNolse -else if (has_lse) - ldadd1RelNondarwinLse -else - ldadd1RelNondarwinNolse; -fn ldclr1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000 @@ -1596,88 +352,8 @@ fn ldclr1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldclr1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr1_rel = if (is_darwin) - if (has_lse) - ldclr1RelDarwinLse - else - ldclr1RelDarwinNolse -else if (has_lse) - ldclr1RelNondarwinLse -else - ldclr1RelNondarwinNolse; -fn ldeor1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000 @@ -1697,88 +373,8 @@ fn ldeor1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldeor1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor1_rel = if (is_darwin) - if (has_lse) - ldeor1RelDarwinLse - else - ldeor1RelDarwinNolse -else if (has_lse) - ldeor1RelNondarwinLse -else - ldeor1RelNondarwinNolse; -fn ldset1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000 @@ -1798,91 +394,8 @@ fn ldset1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldset1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetlb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset1_rel = if (is_darwin) - if (has_lse) - ldset1RelDarwinLse - else - ldset1RelDarwinNolse -else if (has_lse) - ldset1RelNondarwinLse -else - ldset1RelNondarwinNolse; -fn cas1AcqRelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casalb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1AcqRelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x00000000 + 0x408000 @@ -1905,93 +418,8 @@ fn cas1AcqRelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn cas1AcqRelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casalb w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas1AcqRelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x00000000 + 0x408000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxtb w16, w0 - \\0: - \\ ldaxrb w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrb w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas1_acq_rel = if (is_darwin) - if (has_lse) - cas1AcqRelDarwinLse - else - cas1AcqRelDarwinNolse -else if (has_lse) - cas1AcqRelNondarwinLse -else - cas1AcqRelNondarwinNolse; -fn swp1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x00000000 + 0xc00000 @@ -2010,86 +438,8 @@ fn swp1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn swp1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x00000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ stlxrb w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp1_acq_rel = if (is_darwin) - if (has_lse) - swp1AcqRelDarwinLse - else - swp1AcqRelDarwinNolse -else if (has_lse) - swp1AcqRelNondarwinLse -else - swp1AcqRelNondarwinNolse; -fn ldadd1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000 @@ -2109,88 +459,8 @@ fn ldadd1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldadd1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ add w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd1_acq_rel = if (is_darwin) - if (has_lse) - ldadd1AcqRelDarwinLse - else - ldadd1AcqRelDarwinNolse -else if (has_lse) - ldadd1AcqRelNondarwinLse -else - ldadd1AcqRelNondarwinNolse; -fn ldclr1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclralb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000 @@ -2210,88 +480,8 @@ fn ldclr1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldclr1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclralb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr1_acq_rel = if (is_darwin) - if (has_lse) - ldclr1AcqRelDarwinLse - else - ldclr1AcqRelDarwinNolse -else if (has_lse) - ldclr1AcqRelNondarwinLse -else - ldclr1AcqRelNondarwinNolse; -fn ldeor1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoralb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000 @@ -2311,88 +501,8 @@ fn ldeor1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldeor1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoralb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor1_acq_rel = if (is_darwin) - if (has_lse) - ldeor1AcqRelDarwinLse - else - ldeor1AcqRelDarwinNolse -else if (has_lse) - ldeor1AcqRelNondarwinLse -else - ldeor1AcqRelNondarwinNolse; -fn ldset1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000 @@ -2412,91 +522,8 @@ fn ldset1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { : "w15", "w16", "w17", "memory" ); } -fn ldset1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 { +fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetalb w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrb w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrb w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u8), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset1_acq_rel = if (is_darwin) - if (has_lse) - ldset1AcqRelDarwinLse - else - ldset1AcqRelDarwinNolse -else if (has_lse) - ldset1AcqRelNondarwinLse -else - ldset1AcqRelNondarwinNolse; -fn cas2RelaxDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cash w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2RelaxDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x40000000 + 0x000000 @@ -2519,93 +546,8 @@ fn cas2RelaxDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 : "w15", "w16", "w17", "memory" ); } -fn cas2RelaxNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cash w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2RelaxNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x40000000 + 0x000000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas2_relax = if (is_darwin) - if (has_lse) - cas2RelaxDarwinLse - else - cas2RelaxDarwinNolse -else if (has_lse) - cas2RelaxNondarwinLse -else - cas2RelaxNondarwinNolse; -fn swp2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swph w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x40000000 + 0x000000 @@ -2624,86 +566,8 @@ fn swp2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn swp2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swph w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x40000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp2_relax = if (is_darwin) - if (has_lse) - swp2RelaxDarwinLse - else - swp2RelaxDarwinNolse -else if (has_lse) - swp2RelaxNondarwinLse -else - swp2RelaxNondarwinNolse; -fn ldadd2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000 @@ -2723,88 +587,8 @@ fn ldadd2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldadd2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd2_relax = if (is_darwin) - if (has_lse) - ldadd2RelaxDarwinLse - else - ldadd2RelaxDarwinNolse -else if (has_lse) - ldadd2RelaxNondarwinLse -else - ldadd2RelaxNondarwinNolse; -fn ldclr2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000 @@ -2824,88 +608,8 @@ fn ldclr2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldclr2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr2_relax = if (is_darwin) - if (has_lse) - ldclr2RelaxDarwinLse - else - ldclr2RelaxDarwinNolse -else if (has_lse) - ldclr2RelaxNondarwinLse -else - ldclr2RelaxNondarwinNolse; -fn ldeor2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000 @@ -2925,88 +629,8 @@ fn ldeor2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldeor2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor2_relax = if (is_darwin) - if (has_lse) - ldeor2RelaxDarwinLse - else - ldeor2RelaxDarwinNolse -else if (has_lse) - ldeor2RelaxNondarwinLse -else - ldeor2RelaxNondarwinNolse; -fn ldset2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseth w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000 @@ -3026,91 +650,8 @@ fn ldset2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldset2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseth w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset2_relax = if (is_darwin) - if (has_lse) - ldset2RelaxDarwinLse - else - ldset2RelaxDarwinNolse -else if (has_lse) - ldset2RelaxNondarwinLse -else - ldset2RelaxNondarwinNolse; -fn cas2AcqDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casah w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2AcqDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x40000000 + 0x400000 @@ -3133,93 +674,8 @@ fn cas2AcqDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn cas2AcqNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casah w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2AcqNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x40000000 + 0x400000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas2_acq = if (is_darwin) - if (has_lse) - cas2AcqDarwinLse - else - cas2AcqDarwinNolse -else if (has_lse) - cas2AcqNondarwinLse -else - cas2AcqNondarwinNolse; -fn swp2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x40000000 + 0x800000 @@ -3238,86 +694,8 @@ fn swp2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn swp2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x40000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp2_acq = if (is_darwin) - if (has_lse) - swp2AcqDarwinLse - else - swp2AcqDarwinNolse -else if (has_lse) - swp2AcqNondarwinLse -else - swp2AcqNondarwinNolse; -fn ldadd2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000 @@ -3337,88 +715,8 @@ fn ldadd2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldadd2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd2_acq = if (is_darwin) - if (has_lse) - ldadd2AcqDarwinLse - else - ldadd2AcqDarwinNolse -else if (has_lse) - ldadd2AcqNondarwinLse -else - ldadd2AcqNondarwinNolse; -fn ldclr2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000 @@ -3438,88 +736,8 @@ fn ldclr2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldclr2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr2_acq = if (is_darwin) - if (has_lse) - ldclr2AcqDarwinLse - else - ldclr2AcqDarwinNolse -else if (has_lse) - ldclr2AcqNondarwinLse -else - ldclr2AcqNondarwinNolse; -fn ldeor2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000 @@ -3539,88 +757,8 @@ fn ldeor2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldeor2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor2_acq = if (is_darwin) - if (has_lse) - ldeor2AcqDarwinLse - else - ldeor2AcqDarwinNolse -else if (has_lse) - ldeor2AcqNondarwinLse -else - ldeor2AcqNondarwinNolse; -fn ldset2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000 @@ -3640,91 +778,8 @@ fn ldset2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldset2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetah w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset2_acq = if (is_darwin) - if (has_lse) - ldset2AcqDarwinLse - else - ldset2AcqDarwinNolse -else if (has_lse) - ldset2AcqNondarwinLse -else - ldset2AcqNondarwinNolse; -fn cas2RelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caslh w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2RelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x40000000 + 0x008000 @@ -3747,93 +802,8 @@ fn cas2RelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn cas2RelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caslh w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2RelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x40000000 + 0x008000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas2_rel = if (is_darwin) - if (has_lse) - cas2RelDarwinLse - else - cas2RelDarwinNolse -else if (has_lse) - cas2RelNondarwinLse -else - cas2RelNondarwinNolse; -fn swp2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swplh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x40000000 + 0x400000 @@ -3852,86 +822,8 @@ fn swp2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn swp2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swplh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x40000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp2_rel = if (is_darwin) - if (has_lse) - swp2RelDarwinLse - else - swp2RelDarwinNolse -else if (has_lse) - swp2RelNondarwinLse -else - swp2RelNondarwinNolse; -fn ldadd2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000 @@ -3951,88 +843,8 @@ fn ldadd2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldadd2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd2_rel = if (is_darwin) - if (has_lse) - ldadd2RelDarwinLse - else - ldadd2RelDarwinNolse -else if (has_lse) - ldadd2RelNondarwinLse -else - ldadd2RelNondarwinNolse; -fn ldclr2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000 @@ -4052,88 +864,8 @@ fn ldclr2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldclr2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr2_rel = if (is_darwin) - if (has_lse) - ldclr2RelDarwinLse - else - ldclr2RelDarwinNolse -else if (has_lse) - ldclr2RelNondarwinLse -else - ldclr2RelNondarwinNolse; -fn ldeor2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000 @@ -4153,88 +885,8 @@ fn ldeor2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldeor2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor2_rel = if (is_darwin) - if (has_lse) - ldeor2RelDarwinLse - else - ldeor2RelDarwinNolse -else if (has_lse) - ldeor2RelNondarwinLse -else - ldeor2RelNondarwinNolse; -fn ldset2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000 @@ -4254,91 +906,8 @@ fn ldset2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldset2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetlh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset2_rel = if (is_darwin) - if (has_lse) - ldset2RelDarwinLse - else - ldset2RelDarwinNolse -else if (has_lse) - ldset2RelNondarwinLse -else - ldset2RelNondarwinNolse; -fn cas2AcqRelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casalh w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2AcqRelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x40000000 + 0x408000 @@ -4361,93 +930,8 @@ fn cas2AcqRelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u1 : "w15", "w16", "w17", "memory" ); } -fn cas2AcqRelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casalh w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas2AcqRelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x40000000 + 0x408000 - \\ - \\ cbz wzr, 1f - \\8: - \\ uxth w16, w0 - \\0: - \\ ldaxrh w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxrh w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas2_acq_rel = if (is_darwin) - if (has_lse) - cas2AcqRelDarwinLse - else - cas2AcqRelDarwinNolse -else if (has_lse) - cas2AcqRelNondarwinLse -else - cas2AcqRelNondarwinNolse; -fn swp2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x40000000 + 0xc00000 @@ -4466,86 +950,8 @@ fn swp2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn swp2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x40000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ stlxrh w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp2_acq_rel = if (is_darwin) - if (has_lse) - swp2AcqRelDarwinLse - else - swp2AcqRelDarwinNolse -else if (has_lse) - swp2AcqRelNondarwinLse -else - swp2AcqRelNondarwinNolse; -fn ldadd2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000 @@ -4565,88 +971,8 @@ fn ldadd2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldadd2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ add w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd2_acq_rel = if (is_darwin) - if (has_lse) - ldadd2AcqRelDarwinLse - else - ldadd2AcqRelDarwinNolse -else if (has_lse) - ldadd2AcqRelNondarwinLse -else - ldadd2AcqRelNondarwinNolse; -fn ldclr2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclralh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000 @@ -4666,88 +992,8 @@ fn ldclr2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldclr2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclralh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ bic w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr2_acq_rel = if (is_darwin) - if (has_lse) - ldclr2AcqRelDarwinLse - else - ldclr2AcqRelDarwinNolse -else if (has_lse) - ldclr2AcqRelNondarwinLse -else - ldclr2AcqRelNondarwinNolse; -fn ldeor2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoralh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000 @@ -4767,88 +1013,8 @@ fn ldeor2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldeor2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoralh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ eor w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor2_acq_rel = if (is_darwin) - if (has_lse) - ldeor2AcqRelDarwinLse - else - ldeor2AcqRelDarwinNolse -else if (has_lse) - ldeor2AcqRelNondarwinLse -else - ldeor2AcqRelNondarwinNolse; -fn ldset2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000 @@ -4868,91 +1034,8 @@ fn ldset2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { : "w15", "w16", "w17", "memory" ); } -fn ldset2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 { +fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetalh w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxrh w0, [x1] - \\ orr w17, w0, w16 - \\ stlxrh w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u16), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset2_acq_rel = if (is_darwin) - if (has_lse) - ldset2AcqRelDarwinLse - else - ldset2AcqRelDarwinNolse -else if (has_lse) - ldset2AcqRelNondarwinLse -else - ldset2AcqRelNondarwinNolse; -fn cas4RelaxDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cas w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4RelaxDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x80000000 + 0x000000 @@ -4975,93 +1058,8 @@ fn cas4RelaxDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 : "w15", "w16", "w17", "memory" ); } -fn cas4RelaxNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cas w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4RelaxNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x80000000 + 0x000000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas4_relax = if (is_darwin) - if (has_lse) - cas4RelaxDarwinLse - else - cas4RelaxDarwinNolse -else if (has_lse) - cas4RelaxNondarwinLse -else - cas4RelaxNondarwinNolse; -fn swp4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swp w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x80000000 + 0x000000 @@ -5080,86 +1078,8 @@ fn swp4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn swp4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swp w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x80000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp4_relax = if (is_darwin) - if (has_lse) - swp4RelaxDarwinLse - else - swp4RelaxDarwinNolse -else if (has_lse) - swp4RelaxNondarwinLse -else - swp4RelaxNondarwinNolse; -fn ldadd4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadd w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000 @@ -5179,88 +1099,8 @@ fn ldadd4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldadd4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadd w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd4_relax = if (is_darwin) - if (has_lse) - ldadd4RelaxDarwinLse - else - ldadd4RelaxDarwinNolse -else if (has_lse) - ldadd4RelaxNondarwinLse -else - ldadd4RelaxNondarwinNolse; -fn ldclr4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclr w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000 @@ -5280,88 +1120,8 @@ fn ldclr4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldclr4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclr w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr4_relax = if (is_darwin) - if (has_lse) - ldclr4RelaxDarwinLse - else - ldclr4RelaxDarwinNolse -else if (has_lse) - ldclr4RelaxNondarwinLse -else - ldclr4RelaxNondarwinNolse; -fn ldeor4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeor w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000 @@ -5381,88 +1141,8 @@ fn ldeor4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldeor4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeor w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor4_relax = if (is_darwin) - if (has_lse) - ldeor4RelaxDarwinLse - else - ldeor4RelaxDarwinNolse -else if (has_lse) - ldeor4RelaxNondarwinLse -else - ldeor4RelaxNondarwinNolse; -fn ldset4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldset w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000 @@ -5482,91 +1162,8 @@ fn ldset4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldset4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldset w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset4_relax = if (is_darwin) - if (has_lse) - ldset4RelaxDarwinLse - else - ldset4RelaxDarwinNolse -else if (has_lse) - ldset4RelaxNondarwinLse -else - ldset4RelaxNondarwinNolse; -fn cas4AcqDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casa w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4AcqDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x80000000 + 0x400000 @@ -5589,93 +1186,8 @@ fn cas4AcqDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn cas4AcqNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casa w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4AcqNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x80000000 + 0x400000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas4_acq = if (is_darwin) - if (has_lse) - cas4AcqDarwinLse - else - cas4AcqDarwinNolse -else if (has_lse) - cas4AcqNondarwinLse -else - cas4AcqNondarwinNolse; -fn swp4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpa w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x80000000 + 0x800000 @@ -5694,86 +1206,8 @@ fn swp4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn swp4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpa w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x80000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp4_acq = if (is_darwin) - if (has_lse) - swp4AcqDarwinLse - else - swp4AcqDarwinNolse -else if (has_lse) - swp4AcqNondarwinLse -else - swp4AcqNondarwinNolse; -fn ldadd4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadda w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000 @@ -5793,88 +1227,8 @@ fn ldadd4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldadd4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadda w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd4_acq = if (is_darwin) - if (has_lse) - ldadd4AcqDarwinLse - else - ldadd4AcqDarwinNolse -else if (has_lse) - ldadd4AcqNondarwinLse -else - ldadd4AcqNondarwinNolse; -fn ldclr4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclra w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000 @@ -5894,88 +1248,8 @@ fn ldclr4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldclr4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclra w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr4_acq = if (is_darwin) - if (has_lse) - ldclr4AcqDarwinLse - else - ldclr4AcqDarwinNolse -else if (has_lse) - ldclr4AcqNondarwinLse -else - ldclr4AcqNondarwinNolse; -fn ldeor4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeora w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000 @@ -5995,88 +1269,8 @@ fn ldeor4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldeor4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeora w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor4_acq = if (is_darwin) - if (has_lse) - ldeor4AcqDarwinLse - else - ldeor4AcqDarwinNolse -else if (has_lse) - ldeor4AcqNondarwinLse -else - ldeor4AcqNondarwinNolse; -fn ldset4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseta w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000 @@ -6096,91 +1290,8 @@ fn ldset4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldset4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseta w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset4_acq = if (is_darwin) - if (has_lse) - ldset4AcqDarwinLse - else - ldset4AcqDarwinNolse -else if (has_lse) - ldset4AcqNondarwinLse -else - ldset4AcqNondarwinNolse; -fn cas4RelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casl w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4RelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x80000000 + 0x008000 @@ -6203,93 +1314,8 @@ fn cas4RelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn cas4RelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casl w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4RelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x80000000 + 0x008000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas4_rel = if (is_darwin) - if (has_lse) - cas4RelDarwinLse - else - cas4RelDarwinNolse -else if (has_lse) - cas4RelNondarwinLse -else - cas4RelNondarwinNolse; -fn swp4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x80000000 + 0x400000 @@ -6308,86 +1334,8 @@ fn swp4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn swp4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x80000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp4_rel = if (is_darwin) - if (has_lse) - swp4RelDarwinLse - else - swp4RelDarwinNolse -else if (has_lse) - swp4RelNondarwinLse -else - swp4RelNondarwinNolse; -fn ldadd4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000 @@ -6407,88 +1355,8 @@ fn ldadd4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldadd4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd4_rel = if (is_darwin) - if (has_lse) - ldadd4RelDarwinLse - else - ldadd4RelDarwinNolse -else if (has_lse) - ldadd4RelNondarwinLse -else - ldadd4RelNondarwinNolse; -fn ldclr4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000 @@ -6508,88 +1376,8 @@ fn ldclr4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldclr4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr4_rel = if (is_darwin) - if (has_lse) - ldclr4RelDarwinLse - else - ldclr4RelDarwinNolse -else if (has_lse) - ldclr4RelNondarwinLse -else - ldclr4RelNondarwinNolse; -fn ldeor4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000 @@ -6609,88 +1397,8 @@ fn ldeor4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldeor4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor4_rel = if (is_darwin) - if (has_lse) - ldeor4RelDarwinLse - else - ldeor4RelDarwinNolse -else if (has_lse) - ldeor4RelNondarwinLse -else - ldeor4RelNondarwinNolse; -fn ldset4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000 @@ -6710,91 +1418,8 @@ fn ldset4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldset4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetl w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset4_rel = if (is_darwin) - if (has_lse) - ldset4RelDarwinLse - else - ldset4RelDarwinNolse -else if (has_lse) - ldset4RelNondarwinLse -else - ldset4RelNondarwinNolse; -fn cas4AcqRelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casal w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4AcqRelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0x80000000 + 0x408000 @@ -6817,93 +1442,8 @@ fn cas4AcqRelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u3 : "w15", "w16", "w17", "memory" ); } -fn cas4AcqRelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casal w0, w1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas4AcqRelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0x80000000 + 0x408000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x2] - \\ cmp w0, w16 - \\ bne 1f - \\ stlxr w17, w1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [expected] "{w0}" (expected), - [desired] "{w1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas4_acq_rel = if (is_darwin) - if (has_lse) - cas4AcqRelDarwinLse - else - cas4AcqRelDarwinNolse -else if (has_lse) - cas4AcqRelNondarwinLse -else - cas4AcqRelNondarwinNolse; -fn swp4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0x80000000 + 0xc00000 @@ -6922,86 +1462,8 @@ fn swp4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn swp4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0x80000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ stlxr w17, w16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp4_acq_rel = if (is_darwin) - if (has_lse) - swp4AcqRelDarwinLse - else - swp4AcqRelDarwinNolse -else if (has_lse) - swp4AcqRelNondarwinLse -else - swp4AcqRelNondarwinNolse; -fn ldadd4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000 @@ -7021,88 +1483,8 @@ fn ldadd4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldadd4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ add w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd4_acq_rel = if (is_darwin) - if (has_lse) - ldadd4AcqRelDarwinLse - else - ldadd4AcqRelDarwinNolse -else if (has_lse) - ldadd4AcqRelNondarwinLse -else - ldadd4AcqRelNondarwinNolse; -fn ldclr4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclral w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000 @@ -7122,88 +1504,8 @@ fn ldclr4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldclr4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclral w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ bic w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr4_acq_rel = if (is_darwin) - if (has_lse) - ldclr4AcqRelDarwinLse - else - ldclr4AcqRelDarwinNolse -else if (has_lse) - ldclr4AcqRelNondarwinLse -else - ldclr4AcqRelNondarwinNolse; -fn ldeor4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoral w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000 @@ -7223,88 +1525,8 @@ fn ldeor4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldeor4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoral w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ eor w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor4_acq_rel = if (is_darwin) - if (has_lse) - ldeor4AcqRelDarwinLse - else - ldeor4AcqRelDarwinNolse -else if (has_lse) - ldeor4AcqRelNondarwinLse -else - ldeor4AcqRelNondarwinNolse; -fn ldset4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000 @@ -7324,91 +1546,8 @@ fn ldset4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { : "w15", "w16", "w17", "memory" ); } -fn ldset4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 { +fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetal w0, w0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov w16, w0 - \\0: - \\ ldaxr w0, [x1] - \\ orr w17, w0, w16 - \\ stlxr w15, w17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={w0}" (-> u32), - : [val] "{w0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset4_acq_rel = if (is_darwin) - if (has_lse) - ldset4AcqRelDarwinLse - else - ldset4AcqRelDarwinNolse -else if (has_lse) - ldset4AcqRelNondarwinLse -else - ldset4AcqRelNondarwinNolse; -fn cas8RelaxDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cas x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8RelaxDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000 @@ -7431,93 +1570,8 @@ fn cas8RelaxDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 : "w15", "w16", "w17", "memory" ); } -fn cas8RelaxNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ cas x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8RelaxNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas8_relax = if (is_darwin) - if (has_lse) - cas8RelaxDarwinLse - else - cas8RelaxDarwinNolse -else if (has_lse) - cas8RelaxNondarwinLse -else - cas8RelaxNondarwinNolse; -fn swp8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swp x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0xc0000000 + 0x000000 @@ -7536,86 +1590,8 @@ fn swp8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn swp8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swp x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0xc0000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp8_relax = if (is_darwin) - if (has_lse) - swp8RelaxDarwinLse - else - swp8RelaxDarwinNolse -else if (has_lse) - swp8RelaxNondarwinLse -else - swp8RelaxNondarwinNolse; -fn ldadd8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadd x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000 @@ -7635,88 +1611,8 @@ fn ldadd8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldadd8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadd x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd8_relax = if (is_darwin) - if (has_lse) - ldadd8RelaxDarwinLse - else - ldadd8RelaxDarwinNolse -else if (has_lse) - ldadd8RelaxNondarwinLse -else - ldadd8RelaxNondarwinNolse; -fn ldclr8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclr x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000 @@ -7736,88 +1632,8 @@ fn ldclr8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldclr8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclr x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr8_relax = if (is_darwin) - if (has_lse) - ldclr8RelaxDarwinLse - else - ldclr8RelaxDarwinNolse -else if (has_lse) - ldclr8RelaxNondarwinLse -else - ldclr8RelaxNondarwinNolse; -fn ldeor8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeor x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000 @@ -7837,88 +1653,8 @@ fn ldeor8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldeor8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeor x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor8_relax = if (is_darwin) - if (has_lse) - ldeor8RelaxDarwinLse - else - ldeor8RelaxDarwinNolse -else if (has_lse) - ldeor8RelaxNondarwinLse -else - ldeor8RelaxNondarwinNolse; -fn ldset8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldset x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000 @@ -7938,91 +1674,8 @@ fn ldset8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldset8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldset x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset8_relax = if (is_darwin) - if (has_lse) - ldset8RelaxDarwinLse - else - ldset8RelaxDarwinNolse -else if (has_lse) - ldset8RelaxNondarwinLse -else - ldset8RelaxNondarwinNolse; -fn cas8AcqDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casa x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8AcqDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000 @@ -8045,93 +1698,8 @@ fn cas8AcqDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn cas8AcqNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casa x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8AcqNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas8_acq = if (is_darwin) - if (has_lse) - cas8AcqDarwinLse - else - cas8AcqDarwinNolse -else if (has_lse) - cas8AcqNondarwinLse -else - cas8AcqNondarwinNolse; -fn swp8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpa x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0xc0000000 + 0x800000 @@ -8150,86 +1718,8 @@ fn swp8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn swp8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpa x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0xc0000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp8_acq = if (is_darwin) - if (has_lse) - swp8AcqDarwinLse - else - swp8AcqDarwinNolse -else if (has_lse) - swp8AcqNondarwinLse -else - swp8AcqNondarwinNolse; -fn ldadd8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadda x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000 @@ -8249,88 +1739,8 @@ fn ldadd8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldadd8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldadda x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd8_acq = if (is_darwin) - if (has_lse) - ldadd8AcqDarwinLse - else - ldadd8AcqDarwinNolse -else if (has_lse) - ldadd8AcqNondarwinLse -else - ldadd8AcqNondarwinNolse; -fn ldclr8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclra x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000 @@ -8350,88 +1760,8 @@ fn ldclr8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldclr8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclra x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr8_acq = if (is_darwin) - if (has_lse) - ldclr8AcqDarwinLse - else - ldclr8AcqDarwinNolse -else if (has_lse) - ldclr8AcqNondarwinLse -else - ldclr8AcqNondarwinNolse; -fn ldeor8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeora x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000 @@ -8451,88 +1781,8 @@ fn ldeor8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldeor8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeora x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor8_acq = if (is_darwin) - if (has_lse) - ldeor8AcqDarwinLse - else - ldeor8AcqDarwinNolse -else if (has_lse) - ldeor8AcqNondarwinLse -else - ldeor8AcqNondarwinNolse; -fn ldset8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseta x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000 @@ -8552,91 +1802,8 @@ fn ldset8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldset8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldseta x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset8_acq = if (is_darwin) - if (has_lse) - ldset8AcqDarwinLse - else - ldset8AcqDarwinNolse -else if (has_lse) - ldset8AcqNondarwinLse -else - ldset8AcqNondarwinNolse; -fn cas8RelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casl x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8RelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000 @@ -8659,93 +1826,8 @@ fn cas8RelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn cas8RelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casl x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8RelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas8_rel = if (is_darwin) - if (has_lse) - cas8RelDarwinLse - else - cas8RelDarwinNolse -else if (has_lse) - cas8RelNondarwinLse -else - cas8RelNondarwinNolse; -fn swp8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0xc0000000 + 0x400000 @@ -8764,86 +1846,8 @@ fn swp8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn swp8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0xc0000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp8_rel = if (is_darwin) - if (has_lse) - swp8RelDarwinLse - else - swp8RelDarwinNolse -else if (has_lse) - swp8RelNondarwinLse -else - swp8RelNondarwinNolse; -fn ldadd8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000 @@ -8863,88 +1867,8 @@ fn ldadd8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldadd8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd8_rel = if (is_darwin) - if (has_lse) - ldadd8RelDarwinLse - else - ldadd8RelDarwinNolse -else if (has_lse) - ldadd8RelNondarwinLse -else - ldadd8RelNondarwinNolse; -fn ldclr8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000 @@ -8964,88 +1888,8 @@ fn ldclr8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldclr8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclrl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr8_rel = if (is_darwin) - if (has_lse) - ldclr8RelDarwinLse - else - ldclr8RelDarwinNolse -else if (has_lse) - ldclr8RelNondarwinLse -else - ldclr8RelNondarwinNolse; -fn ldeor8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000 @@ -9065,88 +1909,8 @@ fn ldeor8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldeor8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeorl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor8_rel = if (is_darwin) - if (has_lse) - ldeor8RelDarwinLse - else - ldeor8RelDarwinNolse -else if (has_lse) - ldeor8RelNondarwinLse -else - ldeor8RelNondarwinNolse; -fn ldset8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000 @@ -9166,91 +1930,8 @@ fn ldset8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldset8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetl x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset8_rel = if (is_darwin) - if (has_lse) - ldset8RelDarwinLse - else - ldset8RelDarwinNolse -else if (has_lse) - ldset8RelNondarwinLse -else - ldset8RelNondarwinNolse; -fn cas8AcqRelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casal x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8AcqRelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000 @@ -9273,93 +1954,8 @@ fn cas8AcqRelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u6 : "w15", "w16", "w17", "memory" ); } -fn cas8AcqRelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casal x0, x1, [x2] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas8AcqRelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x2] - \\ cmp x0, x16 - \\ bne 1f - \\ stlxr w17, x1, [x2] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas8_acq_rel = if (is_darwin) - if (has_lse) - cas8AcqRelDarwinLse - else - cas8AcqRelDarwinNolse -else if (has_lse) - cas8AcqRelNondarwinLse -else - cas8AcqRelNondarwinNolse; -fn swp8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38208020 + 0xc0000000 + 0xc00000 @@ -9378,86 +1974,8 @@ fn swp8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn swp8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ swpal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn swp8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38208020 + 0xc0000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ stlxr w17, x16, [x1] - \\ cbnz w17, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_swp8_acq_rel = if (is_darwin) - if (has_lse) - swp8AcqRelDarwinLse - else - swp8AcqRelDarwinNolse -else if (has_lse) - swp8AcqRelNondarwinLse -else - swp8AcqRelNondarwinNolse; -fn ldadd8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000 @@ -9477,88 +1995,8 @@ fn ldadd8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldadd8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldaddal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldadd8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ add x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldadd8_acq_rel = if (is_darwin) - if (has_lse) - ldadd8AcqRelDarwinLse - else - ldadd8AcqRelDarwinNolse -else if (has_lse) - ldadd8AcqRelNondarwinLse -else - ldadd8AcqRelNondarwinNolse; -fn ldclr8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclral x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000 @@ -9578,88 +2016,8 @@ fn ldclr8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldclr8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldclral x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldclr8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ bic x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldclr8_acq_rel = if (is_darwin) - if (has_lse) - ldclr8AcqRelDarwinLse - else - ldclr8AcqRelDarwinNolse -else if (has_lse) - ldclr8AcqRelNondarwinLse -else - ldclr8AcqRelNondarwinNolse; -fn ldeor8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoral x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000 @@ -9679,88 +2037,8 @@ fn ldeor8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldeor8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldeoral x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldeor8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ eor x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldeor8_acq_rel = if (is_darwin) - if (has_lse) - ldeor8AcqRelDarwinLse - else - ldeor8AcqRelDarwinNolse -else if (has_lse) - ldeor8AcqRelNondarwinLse -else - ldeor8AcqRelNondarwinNolse; -fn ldset8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000 @@ -9780,93 +2058,8 @@ fn ldset8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { : "w15", "w16", "w17", "memory" ); } -fn ldset8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 { +fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ ldsetal x0, x0, [x1] - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn ldset8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000 - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\0: - \\ ldaxr x0, [x1] - \\ orr x17, x0, x16 - \\ stlxr w15, x17, [x1] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u64), - : [val] "{x0}" (val), - [ptr] "{x1}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_ldset8_acq_rel = if (is_darwin) - if (has_lse) - ldset8AcqRelDarwinLse - else - ldset8AcqRelDarwinNolse -else if (has_lse) - ldset8AcqRelNondarwinLse -else - ldset8AcqRelNondarwinNolse; -fn cas16RelaxDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casp x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16RelaxDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x48207c82 + 0x000000 @@ -9891,103 +2084,8 @@ fn cas16RelaxDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) : "w15", "w16", "w17", "memory" ); } -fn cas16RelaxNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { +fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ casp x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16RelaxNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x48207c82 + 0x000000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas16_relax = if (is_darwin) - if (has_lse) - cas16RelaxDarwinLse - else - cas16RelaxDarwinNolse -else if (has_lse) - cas16RelaxNondarwinLse -else - cas16RelaxNondarwinNolse; -fn cas16AcqDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspa x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16AcqDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x48207c82 + 0x400000 @@ -10012,103 +2110,8 @@ fn cas16AcqDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u : "w15", "w16", "w17", "memory" ); } -fn cas16AcqNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { +fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspa x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16AcqNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x48207c82 + 0x400000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas16_acq = if (is_darwin) - if (has_lse) - cas16AcqDarwinLse - else - cas16AcqDarwinNolse -else if (has_lse) - cas16AcqNondarwinLse -else - cas16AcqNondarwinNolse; -fn cas16RelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspl x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16RelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x48207c82 + 0x008000 @@ -10133,103 +2136,8 @@ fn cas16RelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u : "w15", "w16", "w17", "memory" ); } -fn cas16RelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { +fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspl x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16RelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x48207c82 + 0x008000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas16_rel = if (is_darwin) - if (has_lse) - cas16RelDarwinLse - else - cas16RelDarwinNolse -else if (has_lse) - cas16RelNondarwinLse -else - cas16RelNondarwinNolse; -fn cas16AcqRelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspal x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - return asm volatile ( \\ cbz w16, 8f \\ .inst 0x48207c82 + 0x408000 @@ -10254,182 +2162,106 @@ fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C : "w15", "w16", "w17", "memory" ); } -fn cas16AcqRelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ caspal x0, x1, x2, x3, [x4] - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -fn cas16AcqRelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 { - @setRuntimeSafety(false); - __init_aarch64_have_lse_atomics(); - - return asm volatile ( - \\ cbz w16, 8f - \\ .inst 0x48207c82 + 0x408000 - \\ - \\ cbz wzr, 1f - \\8: - \\ mov x16, x0 - \\ mov x17, x1 - \\0: - \\ ldaxp x0, x1, [x4] - \\ cmp x0, x16 - \\ ccmp x1, x17, #0, eq - \\ bne 1f - \\ stlxp w15, x2, x3, [x4] - \\ cbnz w15, 0b - \\1: - : [ret] "={x0}" (-> u128), - : [expected] "{x0}" (expected), - [desired] "{x1}" (desired), - [ptr] "{x2}" (ptr), - [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics), - : "w15", "w16", "w17", "memory" - ); -} -const __aarch64_cas16_acq_rel = if (is_darwin) - if (has_lse) - cas16AcqRelDarwinLse - else - cas16AcqRelDarwinNolse -else if (has_lse) - cas16AcqRelNondarwinLse -else - cas16AcqRelNondarwinNolse; -//TODO: Add linksection once implemented and remove init at writeFunction -fn __init_aarch64_have_lse_atomics() callconv(.C) void { - const AT_HWCAP = 16; - const HWCAP_ATOMICS = 1 << 8; - const hwcap = std.os.linux.getauxval(AT_HWCAP); - __aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0); -} - -var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse); comptime { - if (arch.isAARCH64()) { - @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage }); - @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage }); - @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage }); - @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage }); - @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage }); - @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage }); - @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage }); - @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage }); - @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage }); - @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage }); - @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage }); - @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage }); - @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage }); - @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage }); - @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage }); - @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage }); - @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage }); - @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage }); - @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage }); - @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage }); - @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage }); - @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage }); - @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage }); - @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage }); - @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage }); - @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage }); - @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage }); - @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage }); - @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage }); - @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage }); - @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage }); - @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage }); - @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage }); - @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage }); - @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage }); - @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage }); - @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage }); - @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage }); - @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage }); - @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage }); - @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage }); - @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage }); - @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage }); - @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage }); - @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage }); - @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage }); - @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage }); - @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage }); - @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage }); - @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage }); - @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage }); - @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage }); - @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage }); - @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage }); - @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage }); - @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage }); - @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage }); - @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage }); - @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage }); - @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage }); - @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage }); - @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage }); - @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage }); - @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage }); - @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage }); - @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage }); - @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage }); - @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage }); - @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage }); - @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage }); - @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage }); - @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage }); - @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage }); - @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage }); - @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage }); - @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage }); - @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage }); - @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage }); - @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage }); - @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage }); - @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage }); - @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage }); - @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage }); - @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage }); - @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage }); - @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage }); - @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage }); - @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage }); - @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage }); - @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage }); - @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage }); - @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage }); - @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage }); - @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage }); - @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage }); - @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage }); - @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage }); - @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage }); - @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage }); - @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage }); - } + @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage }); + @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage }); + @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage }); + @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage }); + @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage }); + @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage }); + @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage }); + @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage }); + @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage }); + @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage }); + @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage }); + @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage }); + @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage }); + @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage }); + @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage }); + @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage }); + @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage }); + @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage }); + @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage }); + @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage }); + @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage }); + @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage }); + @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage }); + @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage }); + @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage }); + @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage }); + @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage }); + @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage }); + @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage }); + @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage }); + @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage }); + @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage }); + @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage }); + @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage }); + @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage }); + @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage }); + @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage }); + @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage }); + @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage }); + @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage }); + @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage }); + @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage }); + @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage }); + @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage }); + @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage }); + @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage }); + @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage }); + @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage }); + @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage }); + @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage }); + @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage }); + @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage }); + @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage }); + @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage }); + @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage }); + @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage }); + @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage }); + @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage }); + @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage }); + @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage }); + @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage }); + @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage }); + @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage }); + @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage }); + @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage }); + @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage }); + @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage }); + @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage }); + @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage }); + @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage }); + @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage }); + @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage }); + @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage }); + @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage }); + @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage }); + @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage }); + @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage }); + @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage }); + @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage }); + @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage }); + @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage }); + @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage }); + @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage }); + @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage }); + @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage }); + @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage }); + @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage }); + @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage }); + @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage }); + @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage }); + @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage }); + @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage }); + @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage }); + @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage }); + @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage }); + @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage }); + @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage }); + @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage }); + @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage }); + @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage }); } diff --git a/tools/gen_outline_atomics.zig b/tools/gen_outline_atomics.zig index 7e851df331..6ea1d31883 100644 --- a/tools/gen_outline_atomics.zig +++ b/tools/gen_outline_atomics.zig @@ -11,217 +11,97 @@ const AtomicOp = enum { }; pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; - defer std.debug.assert(!gpa.deinit()); - var allocator = gpa.allocator(); + var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); - if (args.len != 2) { - usageAndExit(std.io.getStdErr(), args[0], 1); - } + //const args = try std.process.argsAlloc(arena); - var file = try std.fs.cwd().createFile(args[1], .{ .truncate = true }); + var bw = std.io.bufferedWriter(std.io.getStdOut().writer()); + const w = bw.writer(); - try file.writeAll( - \\const std = @import("std"); + try w.writeAll( + \\//! This file is generated by tools/gen_outline_atomics.zig. \\const builtin = @import("builtin"); - \\const arch = builtin.cpu.arch; - \\const is_test = builtin.is_test; - \\const target = std.Target; - \\const os_tag = builtin.os.tag; - \\const is_darwin = target.Os.Tag.isDarwin(os_tag); - \\const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse); - \\const linkage = if (is_test) - \\ std.builtin.GlobalLinkage.Internal - \\else - \\ std.builtin.GlobalLinkage.Strong; + \\const std = @import("std"); + \\const linkage = @import("./common.zig").linkage; + \\const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse); + \\ + \\/// This default is overridden at runtime after inspecting CPU properties. + \\/// It is intentionally not exported in order to make the machine code that + \\/// uses it a statically predicted direct branch rather than using the PLT, + \\/// which ARM is concerned would have too much overhead. + \\var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse); \\ \\ ); + var footer = std.ArrayList(u8).init(arena); + try footer.appendSlice("\ncomptime {\n"); + for ([_]N{ .one, .two, .four, .eight, .sixteen }) |n| { for ([_]Ordering{ .relax, .acq, .rel, .acq_rel }) |order| { - for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) |pat| { - if (pat == .cas or n != .sixteen) { - for ([_]bool{ true, false }) |darwin| { - for ([_]bool{ true, false }) |lse| { - const darwin_name = if (darwin) "Darwin" else "Nondarwin"; - const lse_name = if (lse) "Lse" else "Nolse"; - var buf: [100:0]u8 = undefined; - const name = try std.fmt.bufPrintZ(&buf, "{s}{s}{s}{s}{s}", .{ @tagName(pat), n.toBytes(), order.capName(), darwin_name, lse_name }); - const body = switch (pat) { - .cas => try generateCas(&allocator, n, order, lse), - .swp => try generateSwp(&allocator, n, order, lse), - .ldadd => try generateLd(&allocator, n, order, .ldadd, lse), - .ldclr => try generateLd(&allocator, n, order, .ldclr, lse), - .ldeor => try generateLd(&allocator, n, order, .ldeor, lse), - .ldset => try generateLd(&allocator, n, order, .ldset, lse), - }; - defer allocator.destroy(body.ptr); - try writeFunction(&file, name, pat, n, body); - } - } - try writeExport(&file, @tagName(pat), n.toBytes(), order); - } + for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) |op| { + if (n == .sixteen and op != .cas) continue; + + const name = try std.fmt.allocPrint(arena, "__aarch64_{s}{d}_{s}", .{ + @tagName(op), n.toBytes(), @tagName(order), + }); + try writeFunction(arena, w, name, op, n, order); + try footer.writer().print(" @export({s}, .{{ .name = \"{s}\", .linkage = linkage }});\n", .{ + name, name, + }); } } } - try file.writeAll( - \\//TODO: Add linksection once implemented and remove init at writeFunction - \\fn __init_aarch64_have_lse_atomics() callconv(.C) void { - \\ const AT_HWCAP = 16; - \\ const HWCAP_ATOMICS = 1 << 8; - \\ const hwcap = std.os.linux.getauxval(AT_HWCAP); - \\ __aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0); - \\} - \\ - \\var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse); - \\ - \\comptime { - \\ if (arch.isAARCH64()) { - \\ @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage }); - \\ @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage }); - \\ @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage }); - \\ @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage }); - \\ @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage }); - \\ @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage }); - \\ @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage }); - \\ @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage }); - \\ @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage }); - \\ @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage }); - \\ @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage }); - \\ @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage }); - \\ @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage }); - \\ @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage }); - \\ @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage }); - \\ @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage }); - \\ @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage }); - \\ @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage }); - \\ @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage }); - \\ @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage }); - \\ @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage }); - \\ @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage }); - \\ @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage }); - \\ @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage }); - \\ @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage }); - \\ @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage }); - \\ @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage }); - \\ @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage }); - \\ @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage }); - \\ @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage }); - \\ @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage }); - \\ @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage }); - \\ @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage }); - \\ @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage }); - \\ @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage }); - \\ @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage }); - \\ @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage }); - \\ @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage }); - \\ @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage }); - \\ @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage }); - \\ @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage }); - \\ @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage }); - \\ @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage }); - \\ @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage }); - \\ @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage }); - \\ @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage }); - \\ @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage }); - \\ @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage }); - \\ @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage }); - \\ @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage }); - \\ @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage }); - \\ @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage }); - \\ @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage }); - \\ @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage }); - \\ @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage }); - \\ @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage }); - \\ @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage }); - \\ @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage }); - \\ @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage }); - \\ @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage }); - \\ @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage }); - \\ @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage }); - \\ @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage }); - \\ @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage }); - \\ } - \\} - \\ - ); + try w.writeAll(footer.items); + try w.writeAll("}\n"); + try bw.flush(); } -fn usageAndExit(file: std.fs.File, arg0: []const u8, code: u8) noreturn { - file.writer().print( - \\Usage: {s} /path/to/lib/compiler_rt/lse_atomics.zig - \\ - \\Generates outline atomics for compiler-rt. - \\ - , .{arg0}) catch std.process.exit(1); - std.process.exit(code); -} - -fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, body: [:0]const u8) !void { - var fn_buf: [100:0]u8 = undefined; +fn writeFunction( + arena: Allocator, + w: anytype, + name: []const u8, + op: AtomicOp, + n: N, + order: Ordering, +) !void { + const body = switch (op) { + .cas => try generateCas(arena, n, order), + .swp => try generateSwp(arena, n, order), + .ldadd => try generateLd(arena, n, order, .ldadd), + .ldclr => try generateLd(arena, n, order, .ldclr), + .ldeor => try generateLd(arena, n, order, .ldeor), + .ldset => try generateLd(arena, n, order, .ldset), + }; const fn_sig = if (op != .cas) - try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(val: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() }) + try std.fmt.allocPrint(arena, "fn {[name]s}(val: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{ + .name = name, + .n = n.toBits(), + }) else - try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(expected: u{[n]s}, desired: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() }); - try file.writeAll(fn_sig); - try file.writeAll( + try std.fmt.allocPrint(arena, "fn {[name]s}(expected: u{[n]d}, desired: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{ + .name = name, + .n = n.toBits(), + }); + try w.writeAll(fn_sig); + try w.writeAll( \\ \\ @setRuntimeSafety(false); - \\ __init_aarch64_have_lse_atomics(); - \\ \\ return asm volatile ( \\ ); var iter = std.mem.split(u8, body, "\n"); while (iter.next()) |line| { - try file.writeAll(" \\\\"); - try file.writeAll(line); - try file.writeAll("\n"); + try w.writeAll(" \\\\"); + try w.writeAll(line); + try w.writeAll("\n"); } - var constraint_buf: [500:0]u8 = undefined; const constraints = if (op != .cas) - try std.fmt.bufPrintZ(&constraint_buf, - \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}), + try std.fmt.allocPrint(arena, + \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}), \\ : [val] "{{{[reg]s}0}}" (val), \\ [ptr] "{{x1}}" (ptr), \\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics), @@ -229,8 +109,8 @@ fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, bod \\ , .{ .reg = n.register(), .ty = n.toBits() }) else - try std.fmt.bufPrintZ(&constraint_buf, - \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}), + try std.fmt.allocPrint(arena, + \\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}), \\ : [expected] "{{{[reg]s}0}}" (expected), \\ [desired] "{{{[reg]s}1}}" (desired), \\ [ptr] "{{x2}}" (ptr), @@ -239,55 +119,12 @@ fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, bod \\ , .{ .reg = n.register(), .ty = n.toBits() }); - try file.writeAll(constraints); - try file.writeAll( + try w.writeAll(constraints); + try w.writeAll( \\ ); \\ ); - try file.writeAll("}\n"); -} - -fn writeExport(file: *std.fs.File, pat: [:0]const u8, n: [:0]const u8, order: Ordering) !void { - var darwin_lse_buf: [100:0]u8 = undefined; - var darwin_nolse_buf: [100:0]u8 = undefined; - var nodarwin_lse_buf: [100:0]u8 = undefined; - var nodarwin_nolse_buf: [100:0]u8 = undefined; - var name_buf: [100:0]u8 = undefined; - const darwin_lse = try std.fmt.bufPrintZ(&darwin_lse_buf, "{s}{s}{s}DarwinLse", .{ pat, n, order.capName() }); - const darwin_nolse = try std.fmt.bufPrintZ(&darwin_nolse_buf, "{s}{s}{s}DarwinNolse", .{ pat, n, order.capName() }); - const nodarwin_lse = try std.fmt.bufPrintZ(&nodarwin_lse_buf, "{s}{s}{s}NondarwinLse", .{ pat, n, order.capName() }); - const nodarwin_nolse = try std.fmt.bufPrintZ(&nodarwin_nolse_buf, "{s}{s}{s}NondarwinNolse", .{ pat, n, order.capName() }); - const name = try std.fmt.bufPrintZ(&name_buf, "__aarch64_{s}{s}_{s}", .{ pat, n, @tagName(order) }); - try file.writeAll("const "); - try file.writeAll(name); - try file.writeAll( - \\ = if (is_darwin) - \\ if (has_lse) - \\ - ); - try file.writeAll(darwin_lse); - try file.writeAll( - \\ - \\ else - \\ - ); - try file.writeAll(darwin_nolse); - try file.writeAll( - \\ - \\else if (has_lse) - \\ - ); - try file.writeAll(nodarwin_lse); - try file.writeAll( - \\ - \\else - \\ - ); - try file.writeAll(nodarwin_nolse); - try file.writeAll( - \\; - \\ - ); + try w.writeAll("}\n"); } const N = enum(u8) { @@ -298,22 +135,23 @@ const N = enum(u8) { sixteen = 16, const Defines = struct { - s: [:0]const u8, - uxt: [:0]const u8, - b: [:0]const u8, + s: []const u8, + uxt: []const u8, + b: []const u8, }; - fn defines(self: @This()) Defines { - const s = switch (self) { + + fn defines(n: N) Defines { + const s = switch (n) { .one => "b", .two => "h", else => "", }; - const uxt = switch (self) { + const uxt = switch (n) { .one => "uxtb", .two => "uxth", .four, .eight, .sixteen => "mov", }; - const b = switch (self) { + const b = switch (n) { .one => "0x00000000", .two => "0x40000000", .four => "0x80000000", @@ -327,28 +165,16 @@ const N = enum(u8) { }; } - fn register(self: @This()) [:0]const u8 { - return if (@enumToInt(self) < 8) "w" else "x"; + fn register(n: N) []const u8 { + return if (@enumToInt(n) < 8) "w" else "x"; } - fn toBytes(self: @This()) [:0]const u8 { - return switch (self) { - .one => "1", - .two => "2", - .four => "4", - .eight => "8", - .sixteen => "16", - }; + fn toBytes(n: N) u8 { + return @enumToInt(n); } - fn toBits(self: @This()) [:0]const u8 { - return switch (self) { - .one => "8", - .two => "16", - .four => "32", - .eight => "64", - .sixteen => "128", - }; + fn toBits(n: N) u8 { + return n.toBytes() * 8; } }; @@ -411,31 +237,18 @@ const Ordering = enum { const LdName = enum { ldadd, ldclr, ldeor, ldset }; -fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 { +fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 { const s_def = n.defines(); const o_def = order.defines(); - var cas_buf = try alloc.create([200:0]u8); - var ldxr_buf = try alloc.create([200:0]u8); - var stxr_buf = try alloc.create([200:0]u8); - defer alloc.destroy(cas_buf); - defer alloc.destroy(ldxr_buf); - defer alloc.destroy(stxr_buf); - var instr_buf = try alloc.create([1000:0]u8); - errdefer alloc.destroy(instr_buf); const reg = n.register(); if (@enumToInt(n) < 16) { - const cas = if (lse) blk: { - break :blk try std.fmt.bufPrintZ(cas_buf, - \\cas{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}1, [x2] - \\ - , .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg }); - } else try std.fmt.bufPrintZ(cas_buf, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m }); - const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s }); - const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s }); + const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m }); + const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s }); + const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s }); - return try std.fmt.bufPrintZ(instr_buf, + return try std.fmt.allocPrint(arena, \\ cbz w16, 8f \\ {[cas]s} \\ cbz wzr, 1f @@ -456,15 +269,11 @@ fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u .reg = reg, }); } else { - const casp = if (lse) - try std.fmt.bufPrintZ(cas_buf, "casp{s}{s} x0, x1, x2, x3, [x4]\n", .{ o_def.a, o_def.l }) - else - try std.fmt.bufPrintZ(cas_buf, ".inst 0x48207c82 + {s}\n", .{o_def.m}); + const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}\n", .{o_def.m}); + const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a}); + const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l}); - const ldxp = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xp", .{o_def.a}); - const stxp = try std.fmt.bufPrintZ(stxr_buf, "st{s}xp", .{o_def.l}); - - return try std.fmt.bufPrintZ(instr_buf, + return try std.fmt.allocPrint(arena, \\ cbz w16, 8f \\ {[casp]s} \\ cbz wzr, 1f @@ -487,53 +296,35 @@ fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u } } -fn generateSwp(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 { +fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 { const s_def = n.defines(); const o_def = order.defines(); - - var swp_buf = try alloc.create([200:0]u8); - var ldxr_buf = try alloc.create([200:0]u8); - var stxr_buf = try alloc.create([200:0]u8); - defer alloc.destroy(swp_buf); - defer alloc.destroy(ldxr_buf); - defer alloc.destroy(stxr_buf); - const reg = n.register(); - const swp = if (lse) blk: { - break :blk try std.fmt.bufPrintZ(swp_buf, - \\swp{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1] - , .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg }); - } else std.fmt.bufPrintZ(swp_buf, ".inst 0x38208020 + {s} + {s}", .{ .b = s_def.b, .n = o_def.n }); - - const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s }); - const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s }); - - var instr_buf = try alloc.create([1000:0]u8); - errdefer alloc.destroy(instr_buf); - return try std.fmt.bufPrintZ(instr_buf, + return try std.fmt.allocPrint(arena, \\ cbz w16, 8f - \\ {[swp]s} + \\ .inst 0x38208020 + {[b]s} + {[n]s} \\ cbz wzr, 1f \\8: \\ mov {[reg]s}16, {[reg]s}0 \\0: - \\ {[ldxr]s} {[reg]s}0, [x1] - \\ {[stxr]s} w17, {[reg]s}16, [x1] + \\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1] + \\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1] \\ cbnz w17, 0b \\1: , .{ - .swp = swp, - .ldxr = ldxr, - .stxr = stxr, + .b = s_def.b, + .n = o_def.n, .reg = reg, + .s = s_def.s, + .a = o_def.a, + .l = o_def.l, }); } -fn generateLd(alloc: *Allocator, n: N, order: Ordering, ld: LdName, lse: bool) ![:0]const u8 { +fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 { const s_def = n.defines(); const o_def = order.defines(); - const ldname = @tagName(ld); const op = switch (ld) { .ldadd => "add", .ldclr => "bic", @@ -547,47 +338,27 @@ fn generateLd(alloc: *Allocator, n: N, order: Ordering, ld: LdName, lse: bool) ! .ldset => "0x3000", }; - var swp_buf = try alloc.create([200:0]u8); - var ldop_buf = try alloc.create([200:0]u8); - var ldxr_buf = try alloc.create([200:0]u8); - var stxr_buf = try alloc.create([200:0]u8); - defer alloc.destroy(swp_buf); - defer alloc.destroy(ldop_buf); - defer alloc.destroy(ldxr_buf); - defer alloc.destroy(stxr_buf); - const reg = n.register(); - const ldop = if (lse) - std.fmt.bufPrintZ(ldop_buf, - \\{[ldnm]s}{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1] - , .{ .ldnm = ldname, .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg }) - else - std.fmt.bufPrintZ(ldop_buf, - \\.inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s} - , .{ .op_n = op_n, .b = s_def.b, .n = o_def.n }); - - const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s }); - const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s }); - - var instr_buf = try alloc.create([1000:0]u8); - errdefer alloc.destroy(instr_buf); - return try std.fmt.bufPrintZ(instr_buf, + return try std.fmt.allocPrint(arena, \\ cbz w16, 8f - \\ {[ldop]s} + \\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s} \\ cbz wzr, 1f \\8: \\ mov {[reg]s}16, {[reg]s}0 \\0: - \\ {[ldxr]s} {[reg]s}0, [x1] + \\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1] \\ {[op]s} {[reg]s}17, {[reg]s}0, {[reg]s}16 - \\ {[stxr]s} w15, {[reg]s}17, [x1] + \\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1] \\ cbnz w15, 0b \\1: , .{ - .ldop = ldop, - .ldxr = ldxr, - .stxr = stxr, + .op_n = op_n, + .b = s_def.b, + .n = o_def.n, + .s = s_def.s, + .a = o_def.a, + .l = o_def.l, .op = op, .reg = reg, });