From f597e0d0923e914ec4690016a7aab8ebdaaab4db Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 10 Oct 2019 18:03:57 +0300 Subject: [PATCH 01/31] Change layout and styles --- lib/std/special/docs/index.html | 655 +++++++++++++++++++++----------- 1 file changed, 433 insertions(+), 222 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index ec226f220b..f0d68ffdab 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -5,28 +5,28 @@ Documentation - Zig - - -
- -

Loading...

- - -

- - - - - - -
From a34b697c0c6655407d08aee1b7f56ce5ca5205a2 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 10 Oct 2019 18:55:14 +0300 Subject: [PATCH 02/31] Add transtition to search field --- lib/std/special/docs/index.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index f0d68ffdab..c26468be17 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -428,10 +428,13 @@ border-bottom-style: solid; border-bottom-color: #000; outline: none; + transition: border-bottom-color 0.35s, background 0.35s, box-shadow 0.35s; } .search:focus { border-bottom-color: #ffbb4d; + background: white; + box-shadow: 0 0.3em 1em 0.125em rgba(0, 0, 0, 0.18); } .search::placeholder { From 238ea85b7efcef797d7cb88c97dc677ecacd4938 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Fri, 11 Oct 2019 00:39:08 +0300 Subject: [PATCH 03/31] Fix dark mode Additionally introduces css variables --- lib/std/special/docs/index.html | 781 +++++++++++++++----------------- 1 file changed, 367 insertions(+), 414 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index c26468be17..81668fd11e 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -5,350 +5,48 @@ Documentation - Zig - -
+ +
- -
-
-
- +
+ +
+
+
+
+

Loading...

From c95a9e978582ca96bf5462c6bfdd40b934e9ba92 Mon Sep 17 00:00:00 2001 From: Rocknest <35231115+Rocknest@users.noreply.github.com> Date: Thu, 17 Oct 2019 01:54:39 +0300 Subject: [PATCH 08/31] Cleanup --- lib/std/special/docs/index.html | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index 8bb6ad414d..8ce1033235 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -13,6 +13,13 @@ --bg-color: #ffffff; --link-color: #2A6286; --sidebar-sh-color: rgba(0, 0, 0, 0.09); + --sidebar-pkg-bg-color: #f1f1f1; + --sidebar-pkglnk-tx-color: #141414; + --sidebar-pkglnk-tx-color-hover: #fff; + --sidebar-pkglnk-tx-color-active: #000; + --sidebar-pkglnk-bg-color: transparent; + --sidebar-pkglnk-bg-color-hover: #555; + --sidebar-pkglnk-bg-color-active: #FFBB4D; --search-bg-color: #f3f3f3; --search-bg-color-focus: #ffffff; --search-sh-color: rgba(0, 0, 0, 0.18); @@ -118,24 +125,25 @@ list-style-type: none; margin: 0; padding: 0; - background-color: #f1f1f1; + background-color: var(--sidebar-pkg-bg-color); } .sidebar .packages > li > a { display: block; padding: 0.5rem 1rem; - color: var(--tx-color); + color: var(--sidebar-pkglnk-tx-color); + background-color: var(--sidebar-pkglnk-bg-color); text-decoration: none; } .sidebar .packages > li > a:hover { - color: #fff; - background-color: #555; + color: var(--sidebar-pkglnk-tx-color-hover); + background-color: var(--sidebar-pkglnk-bg-color-hover); } .sidebar .packages > li > a.active { - color: #000; - background-color: #FFBB4D; + color: var(--sidebar-pkglnk-tx-color-active); + background-color: var(--sidebar-pkglnk-bg-color-active); } .sidebar p.str { @@ -367,6 +375,13 @@ --bg-color: #111; --link-color: #88f; --sidebar-sh-color: rgba(128, 128, 128, 0.09); + --sidebar-pkg-bg-color: #333; + --sidebar-pkglnk-tx-color: #fff; + --sidebar-pkglnk-tx-color-hover: #fff; + --sidebar-pkglnk-tx-color-active: #000; + --sidebar-pkglnk-bg-color: transparent; + --sidebar-pkglnk-bg-color-hover: #555; + --sidebar-pkglnk-bg-color-active: #FFBB4D; --search-bg-color: #3c3c3c; --search-bg-color-focus: #000; --search-sh-color: rgba(255, 255, 255, 0.28); @@ -390,20 +405,6 @@ background-color: #FFBB4D; color: #000; } - #listPkgs { - background-color: #333; - } - #listPkgs li a { - color: #fff; - } - #listPkgs li a:hover { - background-color: #555; - color: #fff; - } - #listPkgs li a.active { - background-color: #FFBB4D; - color: #000; - } #listSearchResults li.selected { background-color: #000; } From 8774b1d9478139a8da0f43c49fc33e01c74a6a3f Mon Sep 17 00:00:00 2001 From: Nick Erdmann Date: Fri, 18 Oct 2019 19:49:02 +0200 Subject: [PATCH 09/31] std/os/windows: remove subsystem detection for uefi --- lib/std/os/windows.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8..c35d9d592b 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -42,7 +42,6 @@ pub const subsystem: ?builtin.SubSystem = blk: { break :blk builtin.SubSystem.Console; } }, - .uefi => break :blk builtin.SubSystem.EfiApplication, else => break :blk null, } }; From c8dec5729cabb90fa56fad2b6d77171189e01703 Mon Sep 17 00:00:00 2001 From: Nick Erdmann Date: Fri, 18 Oct 2019 20:11:28 +0200 Subject: [PATCH 10/31] std/build: make subsystem configurable --- lib/std/build.zig | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a315..cb02c8f131 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1491,6 +1491,8 @@ pub const LibExeObjStep = struct { /// Position Independent Code force_pic: ?bool = null, + subsystem: ?builtin.SubSystem = null, + const LinkObject = union(enum) { StaticPath: []const u8, OtherStep: *LibExeObjStep, @@ -2325,6 +2327,20 @@ pub const LibExeObjStep = struct { } } + if (self.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(switch (subsystem) { + .Console => "console", + .Windows => "windows", + .Posix => "posix", + .Native => "native", + .EfiApplication => "efi_application", + .EfiBootServiceDriver => "efi_boot_service_driver", + .EfiRom => "efi_rom", + .EfiRuntimeDriver => "efi_runtime_driver", + }); + } + if (self.kind == Kind.Test) { try builder.spawnChild(zig_args.toSliceConst()); } else { From fa9f1d23968d3b9fe3ae35ed1306aec0cbcfd908 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Oct 2019 03:23:23 -0400 Subject: [PATCH 11/31] add -DZIG_ENABLE_MEM_PROFILE option and -fmem-report flag This can be used to find what's taking up memory in the compiler. Here's an example of how to use it: ``` ./zig test ../lib/std/std.zig --cache off -fmem-report ``` And here's the output I get for this on x86_64-linux-gnu today: ``` Const: 6462833 items, 152 bytes each, total 936.84 MiB ConstGlobalRefs: 17236534 items, 24 bytes each, total 394.51 MiB ResetResult: 1698108 items, 160 bytes each, total 259.11 MiB ConstExprValue: 3118299 items, 80 bytes each, total 237.91 MiB EndExpr: 1345395 items, 168 bytes each, total 215.56 MiB Unknown_8: 27370821 items, 8 bytes each, total 208.82 MiB VarPtr: 1127866 items, 168 bytes each, total 180.70 MiB IrBasicBlock: 794834 items, 120 bytes each, total 90.96 MiB LoadPtr: 554024 items, 160 bytes each, total 84.54 MiB Unknown_64: 1245715 items, 64 bytes each, total 76.03 MiB Unknown_40: 1879218 items, 40 bytes each, total 71.69 MiB Unknown_72: 989117 items, 72 bytes each, total 67.92 MiB Return: 423783 items, 160 bytes each, total 64.66 MiB Unknown_168: 332480 items, 168 bytes each, total 53.27 MiB Unknown_152: 336890 items, 152 bytes each, total 48.84 MiB AddImplicitReturnType: 230819 items, 168 bytes each, total 36.98 MiB Br: 217835 items, 168 bytes each, total 34.90 MiB Unknown_184: 179529 items, 184 bytes each, total 31.50 MiB FieldPtr: 179388 items, 184 bytes each, total 31.48 MiB BinOp: 171004 items, 176 bytes each, total 28.70 MiB LoadPtrGen: 173287 items, 168 bytes each, total 27.76 MiB CondBr: 137864 items, 192 bytes each, total 25.24 MiB Unknown_720: 30918 items, 720 bytes each, total 21.23 MiB CallSrc: 99320 items, 216 bytes each, total 20.46 MiB Unknown_160: 129243 items, 160 bytes each, total 19.72 MiB Unknown_1: 19339456 items, 1 bytes each, total 18.44 MiB CheckStatementIsVoid: 119838 items, 160 bytes each, total 18.29 MiB Unknown_48: 371178 items, 48 bytes each, total 16.99 MiB TestComptime: 101443 items, 160 bytes each, total 15.48 MiB DeclVarSrc: 72578 items, 184 bytes each, total 12.74 MiB StorePtr: 72776 items, 176 bytes each, total 12.22 MiB ZigVar: 79201 items, 160 bytes each, total 12.09 MiB Unknown_16: 770643 items, 16 bytes each, total 11.76 MiB Phi: 60482 items, 184 bytes each, total 10.61 MiB TestErrSrc: 66177 items, 168 bytes each, total 10.60 MiB Unknown_240: 45164 items, 240 bytes each, total 10.34 MiB ElemPtr: 58232 items, 184 bytes each, total 10.22 MiB AllocaSrc: 60053 items, 176 bytes each, total 10.08 MiB CallGen: 44873 items, 224 bytes each, total 9.59 MiB SaveErrRetAddr: 63787 items, 152 bytes each, total 9.25 MiB Unknown_112: 82283 items, 112 bytes each, total 8.79 MiB AllocaGen: 51909 items, 176 bytes each, total 8.71 MiB Unknown_24: 373599 items, 24 bytes each, total 8.55 MiB ResultLocPeer: 113683 items, 72 bytes each, total 7.81 MiB DeclRef: 36343 items, 168 bytes each, total 5.82 MiB UnwrapErrPayload: 34603 items, 168 bytes each, total 5.54 MiB Ref: 33414 items, 168 bytes each, total 5.35 MiB Unknown_104: 53882 items, 104 bytes each, total 5.34 MiB DeclVarGen: 32540 items, 168 bytes each, total 5.21 MiB StructFieldPtr: 30449 items, 176 bytes each, total 5.11 MiB UnwrapErrCode: 31508 items, 168 bytes each, total 5.05 MiB Unknown_56: 90256 items, 56 bytes each, total 4.82 MiB SpillBegin: 28722 items, 168 bytes each, total 4.60 MiB SpillEnd: 28722 items, 160 bytes each, total 4.38 MiB ResultLocReturn: 64573 items, 48 bytes each, total 2.96 MiB PtrType: 14702 items, 184 bytes each, total 2.58 MiB SliceType: 15005 items, 176 bytes each, total 2.52 MiB Unknown_176: 13326 items, 176 bytes each, total 2.24 MiB RefGen: 12881 items, 168 bytes each, total 2.06 MiB UnOp: 12102 items, 176 bytes each, total 2.03 MiB SwitchBr: 9453 items, 200 bytes each, total 1.80 MiB TestErrGen: 11143 items, 160 bytes each, total 1.70 MiB Unknown_32: 52359 items, 32 bytes each, total 1.60 MiB CheckSwitchProngs: 9094 items, 184 bytes each, total 1.60 MiB TypeOf: 9259 items, 160 bytes each, total 1.41 MiB IntCast: 8772 items, 168 bytes each, total 1.41 MiB OptionalUnwrapPtr: 8755 items, 168 bytes each, total 1.40 MiB SwitchTarget: 9094 items, 160 bytes each, total 1.39 MiB Cast: 8198 items, 176 bytes each, total 1.38 MiB WidenOrShorten: 8448 items, 160 bytes each, total 1.29 MiB ErrorUnion: 7613 items, 176 bytes each, total 1.28 MiB SliceSrc: 6249 items, 192 bytes each, total 1.14 MiB ErrWrapCode: 7133 items, 168 bytes each, total 1.14 MiB TypeName: 7328 items, 160 bytes each, total 1.12 MiB ImplicitCast: 5480 items, 176 bytes each, total 941.88 KiB ResolveResult: 5638 items, 168 bytes each, total 924.98 KiB ResultLocInstruction: 22696 items, 40 bytes each, total 886.56 KiB BitCastSrc: 4947 items, 168 bytes each, total 811.62 KiB CompileErr: 5148 items, 160 bytes each, total 804.38 KiB ReturnPtr: 5305 items, 152 bytes each, total 787.46 KiB Unreachable: 5038 items, 152 bytes each, total 747.83 KiB TestNonNull: 4716 items, 160 bytes each, total 736.88 KiB BitCastGen: 4431 items, 160 bytes each, total 692.34 KiB PtrToInt: 4289 items, 160 bytes each, total 670.16 KiB SliceGen: 3573 items, 192 bytes each, total 669.94 KiB ArrayType: 4081 items, 168 bytes each, total 669.54 KiB IntType: 3868 items, 168 bytes each, total 634.59 KiB Unknown_88: 7213 items, 88 bytes each, total 619.87 KiB Truncate: 3771 items, 168 bytes each, total 618.68 KiB TypeInfo: 3740 items, 160 bytes each, total 584.38 KiB SwitchVar: 3385 items, 176 bytes each, total 581.80 KiB ContainerInitFields: 3223 items, 184 bytes each, total 579.13 KiB ContainerInitList: 2309 items, 192 bytes each, total 432.94 KiB PtrCastGen: 2626 items, 168 bytes each, total 430.83 KiB BoolNot: 2457 items, 160 bytes each, total 383.91 KiB FnProto: 2054 items, 184 bytes each, total 369.08 KiB MergeErrSets: 1927 items, 176 bytes each, total 331.20 KiB Unknown_136: 2486 items, 136 bytes each, total 330.17 KiB Unknown_80: 4059 items, 80 bytes each, total 317.11 KiB Bswap: 1670 items, 168 bytes each, total 273.98 KiB TypeId: 1680 items, 160 bytes each, total 262.50 KiB PtrCastSrc: 1371 items, 176 bytes each, total 235.64 KiB ErrName: 1193 items, 160 bytes each, total 186.41 KiB UnionTag: 1120 items, 160 bytes each, total 175.00 KiB TagName: 1050 items, 160 bytes each, total 164.06 KiB SizeOf: 942 items, 160 bytes each, total 147.19 KiB MemberName: 871 items, 168 bytes each, total 142.90 KiB Import: 881 items, 160 bytes each, total 137.66 KiB PtrOfArrayToSlice: 758 items, 168 bytes each, total 124.36 KiB UnionFieldPtr: 710 items, 176 bytes each, total 122.03 KiB EnumToInt: 778 items, 160 bytes each, total 121.56 KiB CheckRuntimeScope: 700 items, 168 bytes each, total 114.84 KiB FieldParentPtr: 632 items, 184 bytes each, total 113.56 KiB BoolToInt: 719 items, 160 bytes each, total 112.34 KiB ResultLocPeerParent: 904 items, 104 bytes each, total 91.81 KiB IntToPtr: 537 items, 168 bytes each, total 88.10 KiB AlignOf: 561 items, 160 bytes each, total 87.66 KiB AtomicRmw: 356 items, 208 bytes each, total 72.31 KiB MemberCount: 441 items, 160 bytes each, total 68.91 KiB Memset: 342 items, 176 bytes each, total 58.78 KiB PopCount: 321 items, 168 bytes each, total 52.66 KiB AlignCast: 251 items, 168 bytes each, total 41.18 KiB IrInstruction *: 5230 items, 8 bytes each, total 40.86 KiB IrBasicBlock *: 5230 items, 8 bytes each, total 40.86 KiB TagType: 261 items, 160 bytes each, total 40.78 KiB HasDecl: 234 items, 168 bytes each, total 38.39 KiB OverflowOp: 191 items, 200 bytes each, total 37.30 KiB Export: 209 items, 176 bytes each, total 35.92 KiB SetCold: 219 items, 160 bytes each, total 34.22 KiB ReturnAddress: 216 items, 152 bytes each, total 32.06 KiB FromBytes: 178 items, 176 bytes each, total 30.59 KiB SetRuntimeSafety: 188 items, 160 bytes each, total 29.38 KiB OptionalWrap: 151 items, 168 bytes each, total 24.77 KiB Clz: 143 items, 168 bytes each, total 23.46 KiB ResizeSlice: 135 items, 168 bytes each, total 22.15 KiB UnionInitNamedField: 106 items, 184 bytes each, total 19.05 KiB Panic: 102 items, 160 bytes each, total 15.94 KiB SwitchElseVar: 93 items, 168 bytes each, total 15.26 KiB ToBytes: 89 items, 168 bytes each, total 14.60 KiB IntToFloat: 78 items, 168 bytes each, total 12.80 KiB Unknown_4360: 3 items, 4360 bytes each, total 12.77 KiB ErrWrapPayload: 72 items, 168 bytes each, total 11.81 KiB FloatOp: 62 items, 176 bytes each, total 10.66 KiB FloatToInt: 47 items, 168 bytes each, total 7.71 KiB FloatCast: 46 items, 168 bytes each, total 7.55 KiB ErrToInt: 47 items, 160 bytes each, total 7.34 KiB Asm: 33 items, 216 bytes each, total 6.96 KiB ErrSetCast: 40 items, 168 bytes each, total 6.56 KiB Memcpy: 34 items, 176 bytes each, total 5.84 KiB AtomicLoad: 17 items, 184 bytes each, total 3.05 KiB AwaitSrc: 16 items, 168 bytes each, total 2.62 KiB Resume: 14 items, 160 bytes each, total 2.19 KiB AwaitGen: 12 items, 176 bytes each, total 2.06 KiB ArgType: 12 items, 168 bytes each, total 1.97 KiB AnyFrameType: 12 items, 160 bytes each, total 1.88 KiB SuspendFinish: 10 items, 160 bytes each, total 1.56 KiB SuspendBegin: 10 items, 160 bytes each, total 1.56 KiB Ctz: 9 items, 168 bytes each, total 1.48 KiB FrameHandle: 8 items, 152 bytes each, total 1.19 KiB SetEvalBranchQuota: 7 items, 160 bytes each, total 1.09 KiB AssertZero: 7 items, 160 bytes each, total 1.09 KiB UndeclaredIdent: 7 items, 160 bytes each, total 1.09 KiB CmpxchgSrc: 5 items, 216 bytes each, total 1.05 KiB CmpxchgGen: 5 items, 200 bytes each, total 1000.00 bytes IntToEnum: 4 items, 168 bytes each, total 672.00 bytes VectorType: 4 items, 168 bytes each, total 672.00 bytes ErrorReturnTrace: 2 items, 160 bytes each, total 320.00 bytes Breakpoint: 2 items, 152 bytes each, total 304.00 bytes FrameAddress: 2 items, 152 bytes each, total 304.00 bytes Unknown_4: 61 items, 4 bytes each, total 244.00 bytes VectorToArray: 1 items, 168 bytes each, total 168.00 bytes SetAlignStack: 1 items, 160 bytes each, total 160.00 bytes Unknown_12: 2 items, 12 bytes each, total 24.00 bytes ErrorTableEntry *: 0 items, 8 bytes each, total 0.00 bytes AstNode *: 0 items, 8 bytes each, total 0.00 bytes Total bytes used: 3.51 GiB ``` You can see that most of the memory is taken up by IR instructions, as well as comptime values. This points toward 2 changes which will greatly reduce memory usage: * Rework semantic analysis so that IR instructions can be freed. Currently the comptime value struct (ConstExprValue) is embedded directly into the IrInstruction struct. If this is made to be separate, at the very least pass 1 IR instructions can be freed. This includes `Const` which is the largest usage of memory currently. * Rework the ConstExprValue struct to no longer be a tagged union. For example, there's no need for an integer comptime value to be 80 bytes. From this you can also see that this eliminates some things from being the culprit. Before doing this analysis, I considered whether doing string interning would help. From the above output, you can see that all strings in the compiler account for only 18 MiB, so string interning would have been a dead end. --- CMakeLists.txt | 2 + src/analyze.cpp | 4 +- src/config.h.in | 5 +- src/dump_analysis.cpp | 19 +----- src/ir.cpp | 52 +++++++++------ src/ir_print.cpp | 6 +- src/ir_print.hpp | 2 + src/main.cpp | 75 ++++++++++++++------- src/memory_profiling.cpp | 139 +++++++++++++++++++++++++++++++++++++++ src/memory_profiling.hpp | 22 +++++++ src/util.cpp | 18 +++++ src/util.hpp | 35 ++++++++-- 12 files changed, 305 insertions(+), 74 deletions(-) create mode 100644 src/memory_profiling.cpp create mode 100644 src/memory_profiling.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbc34f8051..3a34d034dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix") +set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation") if(ZIG_STATIC) set(ZIG_STATIC_LLVM "on") @@ -455,6 +456,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp" "${CMAKE_SOURCE_DIR}/src/link.cpp" + "${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp" "${CMAKE_SOURCE_DIR}/src/os.cpp" "${CMAKE_SOURCE_DIR}/src/parser.cpp" "${CMAKE_SOURCE_DIR}/src/range_set.cpp" diff --git a/src/analyze.cpp b/src/analyze.cpp index 92b221a2ec..ed4547957b 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_ ConstExprValue *create_const_vals(size_t count) { - ConstGlobalRefs *global_refs = allocate(count); - ConstExprValue *vals = allocate(count); + ConstGlobalRefs *global_refs = allocate(count, "ConstGlobalRefs"); + ConstExprValue *vals = allocate(count, "ConstExprValue"); for (size_t i = 0; i < count; i += 1) { vals[i].global_refs = &global_refs[i]; } diff --git a/src/config.h.in b/src/config.h.in index a99aab0d72..7a0ea3536a 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,9 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -// Only used for running tests before installing. -#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" - // Used for communicating build information to self hosted build. #define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@" #define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@" @@ -24,4 +21,6 @@ #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@" #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@" +#cmakedefine ZIG_ENABLE_MEM_PROFILE + #endif diff --git a/src/dump_analysis.cpp b/src/dump_analysis.cpp index d5b109fbcb..9703d3e57d 100644 --- a/src/dump_analysis.cpp +++ b/src/dump_analysis.cpp @@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) { static void tree_print(FILE *f, ZigType *ty, size_t indent); -static void pretty_print_bytes(FILE *f, double n) { - if (n > 1024.0 * 1024.0 * 1024.0) { - fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); - return; - } - if (n > 1024.0 * 1024.0) { - fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); - return; - } - if (n > 1024.0) { - fprintf(f, "%.02f KiB", n / 1024.0); - return; - } - fprintf(f, "%.02f bytes", n ); - return; -} - static int compare_type_abi_sizes_desc(const void *a, const void *b) { uint64_t size_a = (*(ZigType * const*)(a))->abi_size; uint64_t size_b = (*(ZigType * const*)(b))->abi_size; @@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) { start_peer(f, indent); fprintf(f, "\"sizef\": \""); - pretty_print_bytes(f, ty->abi_size); + zig_pretty_print_bytes(f, ty->abi_size); fprintf(f, "\""); start_peer(f, indent); diff --git a/src/ir.cpp b/src/ir.cpp index 7dd141423c..b71a48dc2b 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) { } static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) { - IrBasicBlock *result = allocate(1); + IrBasicBlock *result = allocate(1, "IrBasicBlock"); result->scope = scope; result->name_hint = name_hint; result->debug_id = exec_next_debug_id(irb->exec); @@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) { template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { - T *special_instruction = allocate(1); + const char *name = nullptr; +#ifdef ZIG_ENABLE_MEM_PROFILE + T *dummy = nullptr; + name = ir_instruction_type_str(ir_instruction_id(dummy)); +#endif + T *special_instruction = allocate(1, name); special_instruction->base.id = ir_instruction_id(special_instruction); special_instruction->base.scope = scope; special_instruction->base.source_node = source_node; special_instruction->base.debug_id = exec_next_debug_id(irb->exec); special_instruction->base.owner_bb = irb->current_basic_block; - special_instruction->base.value.global_refs = allocate(1); + special_instruction->base.value.global_refs = allocate(1, "ConstGlobalRefs"); return special_instruction; } @@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, switch (node->data.return_expr.kind) { case ReturnKindUnconditional: { - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); @@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr)); IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val, SpillIdRetErrCode); - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base); @@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime, bool skip_name_check) { - ZigVar *variable_entry = allocate(1); + ZigVar *variable_entry = allocate(1, "ZigVar"); variable_entry->parent_scope = parent_scope; variable_entry->shadowable = is_shadowable; variable_entry->mem_slot_index = SIZE_MAX; @@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n } static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) { - ResultLocPeer *result = allocate(1); + ResultLocPeer *result = allocate(1, "ResultLocPeer"); result->base.id = ResultLocIdPeer; result->base.source_instruction = peer_parent->base.source_instruction; result->parent = peer_parent; @@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node, ir_should_inline(irb->exec, parent_scope)); - scope_block->peer_parent = allocate(1); + scope_block->peer_parent = allocate(1, "ResultLocPeerParent"); scope_block->peer_parent->base.id = ResultLocIdPeerParent; scope_block->peer_parent->base.source_instruction = scope_block->is_comptime; scope_block->peer_parent->end_bb = scope_block->end_block; @@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node) if (lvalue == irb->codegen->invalid_instruction) return irb->codegen->invalid_instruction; - ResultLocInstruction *result_loc_inst = allocate(1); + ResultLocInstruction *result_loc_inst = allocate(1, "ResultLocInstruction"); result_loc_inst->base.id = ResultLocIdInstruction; result_loc_inst->base.source_instruction = lvalue; ir_ref_instruction(lvalue, irb->current_basic_block); @@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node ir_set_cursor_at_end_and_append_block(irb, true_block); - IrInstruction **incoming_values = allocate(2); + IrInstruction **incoming_values = allocate(2, "IrInstruction *"); incoming_values[0] = val1; incoming_values[1] = val2; - IrBasicBlock **incoming_blocks = allocate(2); + IrBasicBlock **incoming_blocks = allocate(2, "IrBasicBlock *"); incoming_blocks[0] = post_val1_block; incoming_blocks[1] = post_val2_block; @@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size; err_set_type->data.error_set.errors = allocate(err_count); - ErrorTableEntry **errors = allocate(irb->codegen->errors_by_index.length + err_count); + size_t errors_count = irb->codegen->errors_by_index.length + err_count; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < err_count; i += 1) { AstNode *field_node = node->data.err_set_decl.decls.at(i); @@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A } errors[err->value] = err; } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_build_const_type(irb, parent_scope, node, err_set_type); } @@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp if (type_is_global_error_set(set2)) { return set1; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); populate_error_set_table(errors, set1); ZigList intersection_list = {}; @@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name)); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); err_set_type->data.error_set.err_count = intersection_list.length; err_set_type->data.error_set.errors = intersection_list.items; @@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted return result; } - ErrorTableEntry **errors = allocate(g->errors_by_index.length); + size_t errors_count = g->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = container_set->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); @@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted result.data.error_set_mismatch->missing_errors.append(contained_error_entry); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return result; } @@ -14686,14 +14694,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) { ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); errors[error_entry->value] = error_entry; } ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name); - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_const_type(ira, &instruction->base, result_type); } @@ -24034,7 +24043,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - AstNode **field_prev_uses = allocate(ira->codegen->errors_by_index.length); + size_t field_prev_uses_count = ira->codegen->errors_by_index.length; + AstNode **field_prev_uses = allocate(field_prev_uses_count, "AstNode *"); for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i]; @@ -24091,7 +24101,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, } } - free(field_prev_uses); + deallocate(field_prev_uses, field_prev_uses_count, "AstNode *"); } else if (switch_type->id == ZigTypeIdInt) { RangeSet rs = {0}; for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index ecd8248d69..441808e834 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -38,8 +38,8 @@ struct IrPrint { static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction); -static const char* ir_instruction_type_str(IrInstruction* instruction) { - switch (instruction->id) { +const char* ir_instruction_type_str(IrInstructionId id) { + switch (id) { case IrInstructionIdInvalid: return "Invalid"; case IrInstructionIdShuffleVector: @@ -387,7 +387,7 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char *ref_count = ir_has_side_effects(instruction) ? "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, - ir_instruction_type_str(instruction), type_name, ref_count); + ir_instruction_type_str(instruction->id), type_name, ref_count); } static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { diff --git a/src/ir_print.hpp b/src/ir_print.hpp index 0960af4e6f..e3947077c8 100644 --- a/src/ir_print.hpp +++ b/src/ir_print.hpp @@ -15,4 +15,6 @@ void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass); void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass); +const char* ir_instruction_type_str(IrInstructionId id); + #endif diff --git a/src/main.cpp b/src/main.cpp index b398eec991..4709035859 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -fno-PIC disable Position Independent Code\n" " -ftime-report print timing diagnostics\n" " -fstack-report print stack size diagnostics\n" +#ifdef ZIG_ENABLE_MEM_PROFILE + " -fmem-report print memory usage diagnostics\n" +#endif " -fdump-analysis write analysis.json file with type information\n" " -femit-docs create a docs/ dir with html documentation\n" " -fno-emit-bin skip emitting machine code\n" @@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) { extern "C" int ZigClang_main(int argc, char **argv); +#ifdef ZIG_ENABLE_MEM_PROFILE +bool mem_report = false; +#endif + +int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { + if (root_progress_node != nullptr) { + stage2_progress_end(root_progress_node); + } +#ifdef ZIG_ENABLE_MEM_PROFILE + if (mem_report) { + memprof_dump_stats(stderr); + } +#endif + return exit_code; +} + int main(int argc, char **argv) { stage2_attach_segfault_handler(); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_init(); +#endif + char *arg0 = argv[0]; Error err; @@ -670,6 +693,13 @@ int main(int argc, char **argv) { timing_info = true; } else if (strcmp(arg, "-fstack-report") == 0) { stack_report = true; + } else if (strcmp(arg, "-fmem-report") == 0) { +#ifdef ZIG_ENABLE_MEM_PROFILE + mem_report = true; +#else + fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n"); + return print_error_usage(arg0); +#endif } else if (strcmp(arg, "-fdump-analysis") == 0) { enable_dump_analysis = true; } else if (strcmp(arg, "-femit-docs") == 0) { @@ -1038,16 +1068,14 @@ int main(int argc, char **argv) { if (in_file) { ZigLibCInstallation libc; if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true))) - return EXIT_FAILURE; - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_FAILURE); + return main_exit(root_progress_node, EXIT_SUCCESS); } ZigLibCInstallation libc; if ((err = zig_libc_find_native(&libc, true))) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); zig_libc_render(&libc, stdout); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdBuiltin: { CodeGen *g = codegen_create(main_pkg_path, nullptr, &target, @@ -1065,10 +1093,9 @@ int main(int argc, char **argv) { Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdRun: case CmdBuild: @@ -1142,7 +1169,7 @@ int main(int argc, char **argv) { libc = allocate(1); if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) { fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err)); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } } Buf *cache_dir_buf; @@ -1219,7 +1246,7 @@ int main(int argc, char **argv) { codegen_set_rdynamic(g, rdynamic); if (mmacosx_version_min && mios_version_min) { fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n"); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } if (mmacosx_version_min) { @@ -1259,6 +1286,11 @@ int main(int argc, char **argv) { zig_print_stack_report(g, stdout); if (cmd == CmdRun) { + stage2_progress_end(root_progress_node); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dump_stats(stderr); +#endif + const char *exec_path = buf_ptr(&g->output_file_path); ZigList args = {0}; @@ -1282,10 +1314,9 @@ int main(int argc, char **argv) { buf_replace(&g->output_file_path, '/', '\\'); #endif if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else { zig_unreachable(); } @@ -1293,8 +1324,7 @@ int main(int argc, char **argv) { codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland); if (timing_info) codegen_print_timing_report(g, stderr); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else if (cmd == CmdTest) { codegen_set_emit_file_type(g, emit_file_type); @@ -1314,7 +1344,7 @@ int main(int argc, char **argv) { if (g->disable_bin_generation) { fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n"); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Buf *test_exe_path_unresolved = &g->output_file_path; @@ -1324,7 +1354,7 @@ int main(int argc, char **argv) { if (emit_file_type != EmitFileTypeBinary) { fprintf(stderr, "Created %s but skipping execution because it is non executable.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } for (size_t i = 0; i < test_exec_args.length; i += 1) { @@ -1336,7 +1366,7 @@ int main(int argc, char **argv) { if (!target_can_exec(&native, &target) && test_exec_args.length == 0) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Termination term; @@ -1348,21 +1378,20 @@ int main(int argc, char **argv) { fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n"); fprintf(stderr, "%s\n", buf_ptr(test_exe_path)); } - stage2_progress_end(root_progress_node); - return (term.how == TerminationIdClean) ? term.code : -1; + return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1); } else { zig_unreachable(); } } case CmdVersion: printf("%s\n", ZIG_VERSION_STRING); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); case CmdZen: { const char *ptr; size_t len; stage2_zen(&ptr, &len); fwrite(ptr, len, 1, stdout); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdTargets: return print_target_list(stdout); diff --git a/src/memory_profiling.cpp b/src/memory_profiling.cpp new file mode 100644 index 0000000000..494ffae117 --- /dev/null +++ b/src/memory_profiling.cpp @@ -0,0 +1,139 @@ +#include "memory_profiling.hpp" +#include "hash_map.hpp" +#include "list.hpp" +#include "util.hpp" +#include + +#ifdef ZIG_ENABLE_MEM_PROFILE + +static bool str_eql_str(const char *a, const char *b) { + return strcmp(a, b) == 0; +} + +static uint32_t str_hash(const char *s) { + // FNV 32-bit hash + uint32_t h = 2166136261; + for (; *s; s += 1) { + h = h ^ *s; + h = h * 16777619; + } + return h; +} + +struct CountAndSize { + size_t item_count; + size_t type_size; +}; + +ZigList unknown_names = {}; +HashMap usage_table = {}; +bool table_active = false; + + +static const char *get_default_name(const char *name_or_null, size_t type_size) { + if (name_or_null != nullptr) return name_or_null; + if (type_size >= unknown_names.length) { + table_active = false; + unknown_names.resize(type_size + 1); + table_active = true; + } + if (unknown_names.at(type_size) == nullptr) { + char buf[100]; + sprintf(buf, "Unknown_%zu%c", type_size, 0); + unknown_names.at(type_size) = strdup(buf); + } + return unknown_names.at(type_size); +} + +void memprof_alloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + // temporarily disable during table put + table_active = false; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.put_unique(name, {count, type_size}); + if (existing_entry != nullptr) { + assert(existing_entry->value.type_size == type_size); // allocated name does not match type + existing_entry->value.item_count += count; + } + table_active = true; +} + +void memprof_dealloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.maybe_get(name); + if (existing_entry == nullptr) { + zig_panic("deallocated more than allocated; compromised memory usage stats"); + } + if (existing_entry->value.type_size != type_size) { + zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size); + } + existing_entry->value.item_count -= count; +} + +void memprof_init(void) { + usage_table.init(1024); + table_active = true; +} + +struct MemItem { + const char *type_name; + CountAndSize count_and_size; +}; + +static size_t get_bytes(const MemItem *item) { + return item->count_and_size.item_count * item->count_and_size.type_size; +} + +static int compare_bytes_desc(const void *a, const void *b) { + size_t size_a = get_bytes((const MemItem *)(a)); + size_t size_b = get_bytes((const MemItem *)(b)); + if (size_a > size_b) + return -1; + if (size_a < size_b) + return 1; + return 0; +} + +void memprof_dump_stats(FILE *file) { + assert(table_active); + // disable modifications from this function + table_active = false; + + ZigList list = {}; + + auto it = usage_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + list.append({entry->key, entry->value}); + } + + qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc); + + size_t total_bytes_used = 0; + + for (size_t i = 0; i < list.length; i += 1) { + const MemItem *item = &list.at(i); + fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name, + item->count_and_size.item_count, item->count_and_size.type_size); + size_t bytes = get_bytes(item); + zig_pretty_print_bytes(file, bytes); + fprintf(file, "\n"); + + total_bytes_used += bytes; + } + + fprintf(stderr, "Total bytes used: "); + zig_pretty_print_bytes(file, total_bytes_used); + fprintf(file, "\n"); + + list.deinit(); + table_active = true; +} + +#endif diff --git a/src/memory_profiling.hpp b/src/memory_profiling.hpp new file mode 100644 index 0000000000..6d43d81e3c --- /dev/null +++ b/src/memory_profiling.hpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_MEMORY_PROFILING_HPP +#define ZIG_MEMORY_PROFILING_HPP + +#include "config.h" + +#include +#include + +void memprof_init(void); + +void memprof_alloc(const char *name, size_t item_count, size_t type_size); +void memprof_dealloc(const char *name, size_t item_count, size_t type_size); + +void memprof_dump_stats(FILE *file); +#endif diff --git a/src/util.cpp b/src/util.cpp index 13bfbbcd47..055d572010 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -119,3 +119,21 @@ Slice SplitIterator_rest(SplitIterator *self) { SplitIterator memSplit(Slice buffer, Slice split_bytes) { return SplitIterator{0, buffer, split_bytes}; } + +void zig_pretty_print_bytes(FILE *f, double n) { + if (n > 1024.0 * 1024.0 * 1024.0) { + fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); + return; + } + if (n > 1024.0 * 1024.0) { + fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); + return; + } + if (n > 1024.0) { + fprintf(f, "%.02f KiB", n / 1024.0); + return; + } + fprintf(f, "%.02f bytes", n ); + return; +} + diff --git a/src/util.hpp b/src/util.hpp index 8abcef32ce..79bebd3355 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -8,6 +8,8 @@ #ifndef ZIG_UTIL_HPP #define ZIG_UTIL_HPP +#include "memory_profiling.hpp" + #include #include #include @@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) { template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { } template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { } template -static inline T *reallocate(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { T *ptr = reallocate_nonzero(old, old_count, new_count); if (new_count > old_count) { memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T)); @@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) { } template -static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, old_count, sizeof(T)); + memprof_alloc(name, new_count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (new_count == 0 && old == nullptr) @@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) return ptr; } +template +static inline void deallocate(T *old, size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, count, sizeof(T)); +#endif + free(old); +} + +template +static inline void destroy(T *old, const char *name = nullptr) { + return deallocate(old, 1); +} + template constexpr size_t array_length(const T (&)[n]) { return n; @@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) { return z; } +void zig_pretty_print_bytes(FILE *f, double n); + template struct Optional { T value; From 4e985123447dba40e282e9663e3ca669d3f9243a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Oct 2019 03:51:04 -0400 Subject: [PATCH 12/31] reduce the size of IrInstruction by 8 bytes on 64 bit targets This brings the std lib tests down from 3.51 GiB memory usage to 3.41 GiB, by making two fields that were 64 bits 32 bits. This is a small thing; the bigger wins will come from the strategy outlined in the previous commit. --- src/all_types.hpp | 6 +++--- src/ir.cpp | 7 ++++--- src/ir_print.cpp | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index f2e614f192..e2e1db48d4 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2569,12 +2569,12 @@ enum IrInstructionId { struct IrInstruction { Scope *scope; AstNode *source_node; - ConstExprValue value; - size_t debug_id; LLVMValueRef llvm_value; + ConstExprValue value; + uint32_t debug_id; // if ref_count is zero and the instruction has no side effects, // the instruction can be omitted in codegen - size_t ref_count; + uint32_t ref_count; // When analyzing IR, instructions that point to this instruction in the "old ir" // can find the instruction that corresponds to this value in the "new ir" // with this child field. diff --git a/src/ir.cpp b/src/ir.cpp index b71a48dc2b..981aa55b2a 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10822,7 +10822,8 @@ static IrInstruction *ira_suspend(IrAnalyze *ira, IrInstruction *old_instruction IrSuspendPosition *suspend_pos) { if (ira->codegen->verbose_ir) { - fprintf(stderr, "suspend %s_%zu %s_%zu #%zu (%zu,%zu)\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "suspend %s_%zu %s_%zu #%" PRIu32 " (%zu,%zu)\n", + ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->name_hint, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->debug_id, @@ -10860,7 +10861,7 @@ static IrInstruction *ira_resume(IrAnalyze *ira) { ira->instruction_index = pos.instruction_index; assert(pos.instruction_index < ira->old_irb.current_basic_block->instruction_list.length); if (ira->codegen->verbose_ir) { - fprintf(stderr, "%s_%zu #%zu\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "%s_%zu #%" PRIu32 "\n", ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.current_basic_block->instruction_list.at(pos.instruction_index)->debug_id); } @@ -26328,7 +26329,7 @@ ZigType *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutable *new_ } if (ira->codegen->verbose_ir) { - fprintf(stderr, "analyze #%zu\n", old_instruction->debug_id); + fprintf(stderr, "analyze #%" PRIu32 "\n", old_instruction->debug_id); } IrInstruction *new_instruction = ir_analyze_instruction_base(ira, old_instruction); if (new_instruction != nullptr) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 441808e834..d3c77f3638 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -385,8 +385,8 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char mark = trailing ? ':' : '#'; const char *type_name = instruction->value.type ? buf_ptr(&instruction->value.type->name) : "(unknown)"; const char *ref_count = ir_has_side_effects(instruction) ? - "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); - fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, + "-" : buf_ptr(buf_sprintf("%" PRIu32 "", instruction->ref_count)); + fprintf(irp->f, "%c%-3" PRIu32 "| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, ir_instruction_type_str(instruction->id), type_name, ref_count); } @@ -398,7 +398,7 @@ static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { } static void ir_print_var_instruction(IrPrint *irp, IrInstruction *instruction) { - fprintf(irp->f, "#%" ZIG_PRI_usize "", instruction->debug_id); + fprintf(irp->f, "#%" PRIu32 "", instruction->debug_id); if (irp->pass != IrPassSrc && irp->printed.maybe_get(instruction) == nullptr) { irp->printed.put(instruction, 0); irp->pending.append(instruction); From bab93e75611de79a1a1e898b0c9e45905817189f Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 19 Oct 2019 16:54:49 +0200 Subject: [PATCH 13/31] Fix crash when generating constant unions with single field --- src/codegen.cpp | 9 +++++++-- test/stage1/behavior/union.zig | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 18af74afd7..82d2480eb9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6355,12 +6355,17 @@ static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ConstExprValue *un ConstParent *parent = &union_const_val->parent; LLVMValueRef base_ptr = gen_parent_ptr(g, union_const_val, parent); + // Slot in the structure where the payload is stored, if equal to SIZE_MAX + // the union has no tag and a single field and is collapsed into the field + // itself + size_t union_payload_index = union_const_val->type->data.unionation.gen_union_index; + ZigType *u32 = g->builtin_types.entry_u32; LLVMValueRef indices[] = { LLVMConstNull(get_llvm_type(g, u32)), - LLVMConstInt(get_llvm_type(g, u32), 0, false), // TODO test const union with more aligned tag type than payload + LLVMConstInt(get_llvm_type(g, u32), union_payload_index, false), }; - return LLVMConstInBoundsGEP(base_ptr, indices, 2); + return LLVMConstInBoundsGEP(base_ptr, indices, (union_payload_index != SIZE_MAX) ? 2 : 1); } static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, ConstExprValue *const_val) { diff --git a/test/stage1/behavior/union.zig b/test/stage1/behavior/union.zig index f5e9ec09a8..40adf601c4 100644 --- a/test/stage1/behavior/union.zig +++ b/test/stage1/behavior/union.zig @@ -521,3 +521,17 @@ test "extern union doesn't trigger field check at comptime" { const x = U{ .x = 0x55AAAA55 }; comptime expect(x.y == 0x55); } + +const Foo1 = union(enum) { + f: struct { + x: usize, + }, +}; +var glbl: Foo1 = undefined; + +test "global union with single field is correctly initialized" { + glbl = Foo1{ + .f = @memberType(Foo1, 0){ .x = 123 }, + }; + expect(glbl.f.x == 123); +} From 6f7939a452e69b77581f5390b8075e9dfa81b03e Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sat, 19 Oct 2019 16:20:30 +0200 Subject: [PATCH 14/31] Prevent too eager constant-folding of switch expression A pointer was wrongly assumed to be comptime-available causing the analysis pass to assume its initial value was constant. Fixes #3481 --- src/ir.cpp | 2 +- test/stage1/behavior/switch.zig | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index 981aa55b2a..2ceabb91b0 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -19251,7 +19251,7 @@ static IrInstruction *ir_analyze_instruction_switch_target(IrAnalyze *ira, ZigType *target_type = target_value_ptr->value.type->data.pointer.child_type; ConstExprValue *pointee_val = nullptr; - if (instr_is_comptime(target_value_ptr)) { + if (instr_is_comptime(target_value_ptr) && target_value_ptr->value.data.x_ptr.mut != ConstPtrMutRuntimeVar) { pointee_val = const_ptr_pointee(ira, ira->codegen, &target_value_ptr->value, target_value_ptr->source_node); if (pointee_val == nullptr) return ira->codegen->invalid_instruction; diff --git a/test/stage1/behavior/switch.zig b/test/stage1/behavior/switch.zig index bc1b4a7a09..936dbed786 100644 --- a/test/stage1/behavior/switch.zig +++ b/test/stage1/behavior/switch.zig @@ -434,3 +434,21 @@ test "switch with disjoint range" { 126...126 => {}, } } + +var state: u32 = 0; +fn poll() void { + switch (state) { + 0 => { + state = 1; + }, + else => { + state += 1; + }, + } +} + +test "switch on global mutable var isn't constant-folded" { + while (state < 2) { + poll(); + } +} From e78d3750c58d26bac0e24c40eb89c2f4796bc15c Mon Sep 17 00:00:00 2001 From: stratact Date: Wed, 18 Sep 2019 23:56:45 -0700 Subject: [PATCH 15/31] Use 8192 sized buffers and remove allocator parameters --- doc/docgen.zig | 2 +- lib/std/build.zig | 4 +-- lib/std/event/fs.zig | 2 +- lib/std/fs.zig | 61 +++++++++++--------------------------- lib/std/os/test.zig | 6 ++-- src-self-hosted/main.zig | 2 +- src-self-hosted/stage1.zig | 2 +- src-self-hosted/test.zig | 4 +-- test/cli.zig | 2 +- tools/process_headers.zig | 2 +- 10 files changed, 31 insertions(+), 56 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 73289e5edc..bdc2604213 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -51,7 +51,7 @@ pub fn main() !void { var toc = try genToc(allocator, &tokenizer); try fs.makePath(allocator, tmp_dir_name); - defer fs.deleteTree(allocator, tmp_dir_name) catch {}; + defer fs.deleteTree(tmp_dir_name) catch {}; try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); try buffered_out_stream.flush(); diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a315..32974da92a 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -331,7 +331,7 @@ pub const Builder = struct { if (self.verbose) { warn("rm {}\n", full_path); } - fs.deleteTree(self.allocator, full_path) catch {}; + fs.deleteTree(full_path) catch {}; } // TODO remove empty directories @@ -2671,7 +2671,7 @@ pub const RemoveDirStep = struct { const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); - fs.deleteTree(self.builder.allocator, full_path) catch |err| { + fs.deleteTree(full_path) catch |err| { warn("Unable to remove {}: {}\n", full_path, @errorName(err)); return err; }; diff --git a/lib/std/event/fs.zig b/lib/std/event/fs.zig index 4490e1deae..81fd950e44 100644 --- a/lib/std/event/fs.zig +++ b/lib/std/event/fs.zig @@ -1312,7 +1312,7 @@ const test_tmp_dir = "std_event_fs_test"; // // // TODO move this into event loop too // try os.makePath(allocator, test_tmp_dir); -// defer os.deleteTree(allocator, test_tmp_dir) catch {}; +// defer os.deleteTree(test_tmp_dir) catch {}; // // var loop: Loop = undefined; // try loop.initMultiThreaded(allocator); diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 287c0316b7..1552f65ecd 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,6 +37,8 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; +pub const MAX_BUF_BYTES: usize = 8192; + // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -371,7 +373,7 @@ const DeleteTreeError = error{ /// this function recursively removes its entries and then tries again. /// TODO determine if we can remove the allocator requirement /// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError!void { +pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { start_over: while (true) { var got_access_denied = false; // First, try deleting the item as a file. This way we don't follow sym links. @@ -395,7 +397,7 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! => return err, } { - var dir = Dir.open(allocator, full_path) catch |err| switch (err) { + var dir = Dir.open(full_path) catch |err| switch (err) { error.NotDir => { if (got_access_denied) { return error.AccessDenied; @@ -424,17 +426,14 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! }; defer dir.close(); - var full_entry_buf = std.ArrayList(u8).init(allocator); - defer full_entry_buf.deinit(); - while (try dir.next()) |entry| { - try full_entry_buf.resize(full_path.len + entry.name.len + 1); - const full_entry_path = full_entry_buf.toSlice(); + var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; + const full_entry_path = full_entry_buf[0..]; mem.copy(u8, full_entry_path, full_path); full_entry_path[full_path.len] = path.sep; mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - try deleteTree(allocator, full_entry_path); + try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); } } return deleteDir(full_path); @@ -445,19 +444,18 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! /// files, and into the one that reads files from an open directory handle. pub const Dir = struct { handle: Handle, - allocator: *Allocator, pub const Handle = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { fd: i32, seek: i64, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, .linux => struct { fd: i32, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, @@ -512,9 +510,8 @@ pub const Dir = struct { /// Call close when done. /// TODO remove the allocator requirement from this API /// https://github.com/ziglang/zig/issues/2885 - pub fn open(allocator: *Allocator, dir_path: []const u8) OpenError!Dir { + pub fn open(dir_path: []const u8) OpenError!Dir { return Dir{ - .allocator = allocator, .handle = switch (builtin.os) { .windows => blk: { var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; @@ -548,7 +545,6 @@ pub const Dir = struct { if (os.windows.is_the_target) { return os.windows.FindClose(self.handle.handle); } - self.allocator.free(self.handle.buf); os.close(self.handle.fd); } @@ -579,14 +575,10 @@ pub const Dir = struct { fn nextDarwin(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.__getdirentries64( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -596,10 +588,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } } @@ -666,21 +655,14 @@ pub const Dir = struct { fn nextLinux(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf.ptr, self.handle.buf.len); + const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); switch (os.linux.getErrno(rc)) { 0 => {}, os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -720,14 +702,10 @@ pub const Dir = struct { fn nextBsd(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.getdirentries( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -736,10 +714,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -807,7 +782,7 @@ pub const Walker = struct { try self.name_buffer.append(base.name); if (base.kind == .Directory) { // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.stack.allocator, self.name_buffer.toSliceConst()); + var new_dir = try Dir.open(self.name_buffer.toSliceConst()); { errdefer new_dir.close(); try self.stack.append(StackItem{ @@ -841,7 +816,7 @@ pub const Walker = struct { pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(allocator, dir_path); + var dir_it = try Dir.open(dir_path); errdefer dir_it.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 8d6b437ff3..b6e6728142 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -19,8 +19,8 @@ test "makePath, put some files in it, deleteTree" { try fs.makePath(a, "os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c" ++ fs.path.sep_str ++ "file.txt", "nonsense"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "file2.txt", "blah"); - try fs.deleteTree(a, "os_test_tmp"); - if (fs.Dir.open(a, "os_test_tmp")) |dir| { + try fs.deleteTree("os_test_tmp"); + if (fs.Dir.open("os_test_tmp")) |dir| { @panic("expected error"); } else |err| { expect(err == error.FileNotFound); @@ -37,7 +37,7 @@ test "access file" { try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", ""); try os.access("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", os.F_OK); - try fs.deleteTree(a, "os_test_tmp"); + try fs.deleteTree("os_test_tmp"); } fn testThreadIdFn(thread_id: *Thread.Id) void { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 52eda5824a..a2e77394ce 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -747,7 +747,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro )) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.loop.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); var group = event.Group(FmtError!void).init(fmt.loop); diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index 48b14ebcac..d36073abf3 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -283,7 +283,7 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void const source_code = io.readFileAlloc(fmt.allocator, file_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); while (try dir.next()) |entry| { diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 526518ca47..ce51de9a7b 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -56,11 +56,11 @@ pub const TestContext = struct { errdefer allocator.free(self.zig_lib_dir); try std.fs.makePath(allocator, tmp_dir_name); - errdefer std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + errdefer std.fs.deleteTree(tmp_dir_name) catch {}; } fn deinit(self: *TestContext) void { - std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + std.fs.deleteTree(tmp_dir_name) catch {}; allocator.free(self.zig_lib_dir); self.zig_compiler.deinit(); self.loop.deinit(); diff --git a/test/cli.zig b/test/cli.zig index 63a116a811..2999df14b0 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -37,7 +37,7 @@ pub fn main() !void { testMissingOutputPath, }; for (test_fns) |testFn| { - try fs.deleteTree(a, dir_path); + try fs.deleteTree(dir_path); try fs.makeDir(dir_path); try testFn(zig_exe, dir_path); } diff --git a/tools/process_headers.zig b/tools/process_headers.zig index 420c118cb9..667da5f7c9 100644 --- a/tools/process_headers.zig +++ b/tools/process_headers.zig @@ -340,7 +340,7 @@ pub fn main() !void { try dir_stack.append(target_include_dir); while (dir_stack.popOrNull()) |full_dir_name| { - var dir = std.fs.Dir.open(allocator, full_dir_name) catch |err| switch (err) { + var dir = std.fs.Dir.open(full_dir_name) catch |err| switch (err) { error.FileNotFound => continue :search, error.AccessDenied => continue :search, else => return err, From 771dafbabaaf4cda43d722dbc4bcb7d8464b576d Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Oct 2019 16:58:00 +0200 Subject: [PATCH 16/31] Resolve lazy slice start/end values Fixes #3489 --- src/ir.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index 2ceabb91b0..6bdfd86b29 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -23084,17 +23084,22 @@ static IrInstruction *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstruction zig_unreachable(); } - uint64_t start_scalar = bigint_as_u64(&casted_start->value.data.x_bigint); + ConstExprValue *start_val = ir_resolve_const(ira, casted_start, UndefBad); + if (!start_val) + return ira->codegen->invalid_instruction; + + uint64_t start_scalar = bigint_as_u64(&start_val->data.x_bigint); if (!ptr_is_undef && start_scalar > rel_end) { ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); return ira->codegen->invalid_instruction; } - uint64_t end_scalar; + uint64_t end_scalar = rel_end; if (end) { - end_scalar = bigint_as_u64(&end->value.data.x_bigint); - } else { - end_scalar = rel_end; + ConstExprValue *end_val = ir_resolve_const(ira, end, UndefBad); + if (!end_val) + return ira->codegen->invalid_instruction; + end_scalar = bigint_as_u64(&end_val->data.x_bigint); } if (!ptr_is_undef) { if (end_scalar > rel_end) { From 5b1a492012241276a4b7539ca6664234f0629c79 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 20 Oct 2019 21:48:23 -0400 Subject: [PATCH 17/31] breaking: improve std.fs directory handling API * Added `std.c.unlinkat` and `std.os.unlinkat`. * Removed `std.fs.MAX_BUF_BYTES` (this declaration never made it to master branch) * Added `std.fs.Dir.deleteTree` to be used on an open directory handle. * `std.fs.deleteTree` has better behavior for both relative and absolute paths. For absolute paths, it opens the base directory and uses that handle for subsequent operations. For relative paths, it does a similar strategy, using the cwd handle. * The error set of `std.fs.deleteTree` is improved to no longer have these possible errors: - OutOfMemory - FileTooBig - IsDir - DirNotEmpty - PathAlreadyExists - NoSpaceLeft * Added `std.fs.Dir.posix_cwd` which is a statically initialized directory representing the current working directory. * The error set of `std.Dir.open` is improved to no longer have these possible errors: - FileTooBig - IsDir - NoSpaceLeft - PathAlreadyExists - OutOfMemory * Added more alternative functions to `std.fs` for when the path parameter is a null terminated string. This can sometimes be more effecient on systems which have an ABI based on null terminated strings. * Added `std.fs.Dir.openDir`, `std.fs.Dir.deleteFile`, and `std.fs.Dir.deleteDir` which all operate on an open directory handle. * `std.fs.Walker.Entry` now has a `dir` field, which can be used to do operations directly on `std.fs.Walker.Entry.basename`, avoiding `error.NameTooLong` for deeply nested paths. * Added more docs to `std.os.OpenError` This commit does the POSIX components for these changes. I plan to follow up shortly with a commit for Windows. --- lib/std/c.zig | 1 + lib/std/fs.zig | 869 +++++++++++++++++++++---------------- lib/std/os.zig | 56 ++- src-self-hosted/stage1.zig | 6 +- 4 files changed, 563 insertions(+), 369 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 9c7ec2f0e5..f591481d04 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -80,6 +80,7 @@ pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint pub extern "c" fn munmap(addr: *align(page_size) c_void, len: usize) c_int; pub extern "c" fn mprotect(addr: *align(page_size) c_void, len: usize, prot: c_uint) c_int; pub extern "c" fn unlink(path: [*]const u8) c_int; +pub extern "c" fn unlinkat(dirfd: fd_t, path: [*]const u8, flags: c_uint) c_int; pub extern "c" fn getcwd(buf: [*]u8, size: usize) ?[*]u8; pub extern "c" fn waitpid(pid: c_int, stat_loc: *c_uint, options: c_uint) c_int; pub extern "c" fn fork() c_int; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 1552f65ecd..89071b407f 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,8 +37,6 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; -pub const MAX_BUF_BYTES: usize = 8192; - // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -337,136 +335,35 @@ pub fn deleteDirW(dir_path: [*]const u16) !void { return os.rmdirW(dir_path); } -const DeleteTreeError = error{ - OutOfMemory, - AccessDenied, - FileTooBig, - IsDir, - SymLinkLoop, - ProcessFdQuotaExceeded, - NameTooLong, - SystemFdQuotaExceeded, - NoDevice, - SystemResources, - NoSpaceLeft, - PathAlreadyExists, - ReadOnlyFileSystem, - NotDir, - FileNotFound, - FileSystem, - FileBusy, - DirNotEmpty, - DeviceBusy, +/// Removes a symlink, file, or directory. +/// If `full_path` is relative, this is equivalent to `Dir.deleteTree` with the +/// current working directory as the open directory handle. +/// If `full_path` is absolute, this is equivalent to `Dir.deleteTree` with the +/// base directory. +pub fn deleteTree(full_path: []const u8) !void { + if (path.isAbsolute(full_path)) { + const dirname = path.dirname(full_path) orelse return error{ + /// Attempt to remove the root file system path. + /// This error is unreachable if `full_path` is relative. + CannotDeleteRootDirectory, + }.CannotDeleteRootDirectory; - /// On Windows, file paths must be valid Unicode. - InvalidUtf8, + var dir = try Dir.open(dirname); + defer dir.close(); - /// On Windows, file paths cannot contain these characters: - /// '/', '*', '?', '"', '<', '>', '|' - BadPathName, - - Unexpected, -}; - -/// Whether `full_path` describes a symlink, file, or directory, this function -/// removes it. If it cannot be removed because it is a non-empty directory, -/// this function recursively removes its entries and then tries again. -/// TODO determine if we can remove the allocator requirement -/// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { - start_over: while (true) { - var got_access_denied = false; - // First, try deleting the item as a file. This way we don't follow sym links. - if (deleteFile(full_path)) { - return; - } else |err| switch (err) { - error.FileNotFound => return, - error.IsDir => {}, - error.AccessDenied => got_access_denied = true, - - error.InvalidUtf8, - error.SymLinkLoop, - error.NameTooLong, - error.SystemResources, - error.ReadOnlyFileSystem, - error.NotDir, - error.FileSystem, - error.FileBusy, - error.BadPathName, - error.Unexpected, - => return err, - } - { - var dir = Dir.open(full_path) catch |err| switch (err) { - error.NotDir => { - if (got_access_denied) { - return error.AccessDenied; - } - continue :start_over; - }, - - error.OutOfMemory, - error.AccessDenied, - error.FileTooBig, - error.IsDir, - error.SymLinkLoop, - error.ProcessFdQuotaExceeded, - error.NameTooLong, - error.SystemFdQuotaExceeded, - error.NoDevice, - error.FileNotFound, - error.SystemResources, - error.NoSpaceLeft, - error.PathAlreadyExists, - error.Unexpected, - error.InvalidUtf8, - error.BadPathName, - error.DeviceBusy, - => return err, - }; - defer dir.close(); - - while (try dir.next()) |entry| { - var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; - const full_entry_path = full_entry_buf[0..]; - mem.copy(u8, full_entry_path, full_path); - full_entry_path[full_path.len] = path.sep; - mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - - try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); - } - } - return deleteDir(full_path); + return dir.deleteTree(path.basename(full_path)); + } else { + return Dir.posix_cwd.deleteTree(full_path); } } -/// TODO: separate this API into the one that opens directory handles to then subsequently open -/// files, and into the one that reads files from an open directory handle. pub const Dir = struct { - handle: Handle, + fd: os.fd_t, - pub const Handle = switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => struct { - fd: i32, - seek: i64, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .linux => struct { - fd: i32, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .windows => struct { - handle: os.windows.HANDLE, - find_file_data: os.windows.WIN32_FIND_DATAW, - first: bool, - name_data: [256]u8, - }, - else => @compileError("unimplemented"), - }; + /// An open handle to the current working directory. + /// Closing this directory is safety-checked illegal behavior. + /// Not available on Windows. + pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; pub const Entry = struct { name: []const u8, @@ -485,269 +382,504 @@ pub const Dir = struct { }; }; + pub const Iterator = switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => struct { + dir: Dir, + seek: i64, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + switch (builtin.os) { + .macosx, .ios => return self.nextDarwin(), + .freebsd, .netbsd => return self.nextBsd(), + else => @compileError("unimplemented"), + } + } + + fn nextDarwin(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + } + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextBsd(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + freebsd_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (freebsd_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .linux => struct { + dir: Dir, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; + break; + } + } + const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); + const next_index = self.index + linux_entry.d_reclen; + self.index = next_index; + + const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (linux_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .windows => struct { + dir: Dir, + find_file_data: os.windows.WIN32_FIND_DATAW, + first: bool, + name_data: [256]u8, + }, + else => @compileError("unimplemented"), + }; + pub const OpenError = error{ FileNotFound, NotDir, AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, NameTooLong, SystemFdQuotaExceeded, NoDevice, SystemResources, - NoSpaceLeft, - PathAlreadyExists, - OutOfMemory, InvalidUtf8, BadPathName, DeviceBusy, + } || os.UnexpectedError; - Unexpected, - }; - - /// Call close when done. - /// TODO remove the allocator requirement from this API - /// https://github.com/ziglang/zig/issues/2885 + /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return Dir{ - .handle = switch (builtin.os) { - .windows => blk: { - var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; - const handle = try os.windows.FindFirstFile(dir_path, &find_file_data); - break :blk Handle{ - .handle = handle, - .find_file_data = find_file_data, // TODO guaranteed copy elision - .first = true, - .name_data = undefined, - }; - }, - .macosx, .ios, .freebsd, .netbsd => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_NONBLOCK | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .seek = 0, - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - .linux => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - else => @compileError("unimplemented"), - }, - }; + return posix_cwd.openDir(dir_path); + } + + /// Same as `open` except the parameter is null-terminated. + pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { + return posix_cwd.openDirC(dir_path_c); } pub fn close(self: *Dir) void { if (os.windows.is_the_target) { - return os.windows.FindClose(self.handle.handle); + @panic("TODO"); } - os.close(self.handle.fd); + os.close(self.fd); + self.* = undefined; } - /// Memory such as file names referenced in this returned entry becomes invalid - /// with subsequent calls to next, as well as when this `Dir` is deinitialized. - pub fn next(self: *Dir) !?Entry { + /// Call `File.close` on the result when done. + pub fn openRead(self: Dir, sub_path: []const u8) File.OpenError!File { + const path_c = try os.toPosixPath(sub_path); + return self.openReadC(&path_c); + } + + /// Call `File.close` on the result when done. + pub fn openReadC(self: Dir, sub_path: [*]const u8) File.OpenError!File { + const flags = os.O_LARGEFILE | os.O_RDONLY; + const fd = try os.openatC(self.fd, sub_path, flags, 0); + return File.openHandle(fd); + } + + /// Call `close` on the result when done. + pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + const sub_path_c = try os.toPosixPath(sub_path); + return self.openDirC(&sub_path_c); + } + + /// Call `close` on the result when done. + pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; + const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + error.FileTooBig => unreachable, // can't happen for directories + error.IsDir => unreachable, // we're providing O_DIRECTORY + error.NoSpaceLeft => unreachable, // not providing O_CREAT + error.PathAlreadyExists => unreachable, // not providing O_CREAT + else => |e| return e, + }; + return Dir{ .fd = fd }; + } + + pub const DeleteFileError = os.UnlinkError; + + /// Delete a file name and possibly the file it refers to, based on an open directory handle. + pub fn deleteFile(self: Dir, sub_path: []const u8) DeleteFileError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteFileC(&sub_path_c); + } + + /// Same as `deleteFile` except the parameter is null-terminated. + pub fn deleteFileC(self: Dir, sub_path_c: [*]const u8) DeleteFileError!void { + os.unlinkatC(self.fd, sub_path_c, 0) catch |err| switch (err) { + error.DirNotEmpty => unreachable, // not passing AT_REMOVEDIR + else => |e| return e, + }; + } + + pub const DeleteDirError = error{ + DirNotEmpty, + FileNotFound, + AccessDenied, + FileBusy, + FileSystem, + SymLinkLoop, + NameTooLong, + NotDir, + SystemResources, + ReadOnlyFileSystem, + InvalidUtf8, + BadPathName, + Unexpected, + }; + + /// Returns `error.DirNotEmpty` if the directory is not empty. + /// To delete a directory recursively, see `deleteTree`. + pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteDirC(&sub_path_c); + } + + /// Same as `deleteDir` except the parameter is null-terminated. + pub fn deleteDirC(self: Dir, sub_path_c: [*]const u8) DeleteDirError!void { + os.unlinkatC(self.fd, sub_path_c, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + pub fn iterate(self: Dir) Iterator { switch (builtin.os) { - .linux => return self.nextLinux(), - .macosx, .ios => return self.nextDarwin(), - .windows => return self.nextWindows(), - .freebsd => return self.nextBsd(), - .netbsd => return self.nextBsd(), + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => @panic("TODO"), else => @compileError("unimplemented"), } } - pub fn openRead(self: Dir, file_path: []const u8) os.OpenError!File { - const path_c = try os.toPosixPath(file_path); - return self.openReadC(&path_c); - } + pub const DeleteTreeError = error{ + AccessDenied, + FileTooBig, + SymLinkLoop, + ProcessFdQuotaExceeded, + NameTooLong, + SystemFdQuotaExceeded, + NoDevice, + SystemResources, + ReadOnlyFileSystem, + FileSystem, + FileBusy, + DeviceBusy, - pub fn openReadC(self: Dir, file_path: [*]const u8) OpenError!File { - const flags = os.O_LARGEFILE | os.O_RDONLY; - const fd = try os.openatC(self.handle.fd, file_path, flags, 0); - return File.openHandle(fd); - } + /// One of the path components was not a directory. + /// This error is unreachable if `sub_path` does not contain a path separator. + NotDir, - fn nextDarwin(self: *Dir) !?Entry { + /// On Windows, file paths must be valid Unicode. + InvalidUtf8, + + /// On Windows, file paths cannot contain these characters: + /// '/', '*', '?', '"', '<', '>', '|' + BadPathName, + } || os.UnexpectedError; + + /// Whether `full_path` describes a symlink, file, or directory, this function + /// removes it. If it cannot be removed because it is a non-empty directory, + /// this function recursively removes its entries and then tries again. + /// This operation is not atomic on most file systems. + pub fn deleteTree(self: Dir, sub_path: []const u8) DeleteTreeError!void { start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } + var got_access_denied = false; + // First, try deleting the item as a file. This way we don't follow sym links. + if (self.deleteFile(sub_path)) { + return; + } else |err| switch (err) { + error.FileNotFound => return, + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.NotDir, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + var dir = self.openDir(sub_path) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; } - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; + continue :start_over; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :start_over; + }, + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + var cleanup_dir_parent: ?Dir = null; + defer if (cleanup_dir_parent) |*d| d.close(); + + var cleanup_dir = true; + defer if (cleanup_dir) dir.close(); + + var dir_name_buf: [MAX_PATH_BYTES]u8 = undefined; + var dir_name: []const u8 = sub_path; + var parent_dir = self; + + // Here we must avoid recursion, in order to provide O(1) memory guarantee of this function. + // Go through each entry and if it is not a directory, delete it. If it is a directory, + // open it, and close the original directory. Repeat. Then start the entire operation over. + + scan_dir: while (true) { + var dir_it = dir.iterate(); + while (try dir_it.next()) |entry| { + if (dir.deleteFile(entry.name)) { + continue; + } else |err| switch (err) { + error.FileNotFound => continue, + + // Impossible because we do not pass any path separators. + error.NotDir => unreachable, + + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + + const new_dir = dir.openDir(entry.name) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :scan_dir; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :scan_dir; + }, + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + if (cleanup_dir_parent) |*d| d.close(); + cleanup_dir_parent = dir; + dir = new_dir; + mem.copy(u8, &dir_name_buf, entry.name); + dir_name = dir_name_buf[0..entry.name.len]; + continue :scan_dir; + } + // Reached the end of the directory entries, which means we successfully deleted all of them. + // Now to remove the directory itself. + dir.close(); + cleanup_dir = false; + + if (cleanup_dir_parent) |d| { + d.deleteDir(dir_name) catch |err| switch (err) { + // These two things can happen due to file system race conditions. + error.FileNotFound, error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + continue :start_over; + } else { + self.deleteDir(sub_path) catch |err| switch (err) { + error.FileNotFound => return, + error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + return; } } - const darwin_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + darwin_entry.d_reclen; - self.handle.index = next_index; - - const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (darwin_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } - - fn nextWindows(self: *Dir) !?Entry { - while (true) { - if (self.handle.first) { - self.handle.first = false; - } else { - if (!try os.windows.FindNextFile(self.handle.handle, &self.handle.find_file_data)) - return null; - } - const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) - continue; - // Trust that Windows gives us valid UTF-16LE - const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; - const name_utf8 = self.handle.name_data[0..name_utf8_len]; - const kind = blk: { - const attrs = self.handle.find_file_data.dwFileAttributes; - if (attrs & os.windows.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; - if (attrs & os.windows.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; - break :blk Entry.Kind.File; - }; - return Entry{ - .name = name_utf8, - .kind = kind, - }; - } - } - - fn nextLinux(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = rc; - break; - } - } - const linux_entry = @ptrCast(*align(1) os.dirent64, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + linux_entry.d_reclen; - self.handle.index = next_index; - - const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); - - // skip . and .. entries - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (linux_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } - - fn nextBsd(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; - } - } - const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + freebsd_entry.d_reclen; - self.handle.index = next_index; - - const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (freebsd_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; } } }; @@ -757,13 +889,18 @@ pub const Walker = struct { name_buffer: std.Buffer, pub const Entry = struct { - path: []const u8, + /// The containing directory. This can be used to operate directly on `basename` + /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths. + /// The directory remains open until `next` or `deinit` is called. + dir: Dir, basename: []const u8, + + path: []const u8, kind: Dir.Entry.Kind, }; const StackItem = struct { - dir_it: Dir, + dir_it: Dir.Iterator, dirname_len: usize, }; @@ -781,23 +918,26 @@ pub const Walker = struct { try self.name_buffer.appendByte(path.sep); try self.name_buffer.append(base.name); if (base.kind == .Directory) { - // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.name_buffer.toSliceConst()); + var new_dir = top.dir_it.dir.openDir(base.name) catch |err| switch (err) { + error.NameTooLong => unreachable, // no path sep in base.name + else => |e| return e, + }; { errdefer new_dir.close(); try self.stack.append(StackItem{ - .dir_it = new_dir, + .dir_it = new_dir.iterate(), .dirname_len = self.name_buffer.len(), }); } } return Entry{ + .dir = top.dir_it.dir, .basename = self.name_buffer.toSliceConst()[dirname_len + 1 ..], .path = self.name_buffer.toSliceConst(), .kind = base.kind, }; } else { - self.stack.pop().dir_it.close(); + self.stack.pop().dir_it.dir.close(); } } } @@ -812,12 +952,13 @@ pub const Walker = struct { /// Recursively iterates over a directory. /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. +/// The order of returned file system entries is undefined. /// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(dir_path); - errdefer dir_it.close(); + var dir = try Dir.open(dir_path); + errdefer dir.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); errdefer name_buffer.deinit(); @@ -828,7 +969,7 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { }; try walker.stack.append(Walker.StackItem{ - .dir_it = dir_it, + .dir_it = dir.iterate(), .dirname_len = dir_path.len, }); diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5..e7319095cb 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -529,22 +529,36 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) WriteError!void pub const OpenError = error{ AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, - NameTooLong, SystemFdQuotaExceeded, NoDevice, FileNotFound, + /// The path exceeded `MAX_PATH_BYTES` bytes. + NameTooLong, + /// Insufficient kernel memory was available, or /// the named file is a FIFO and per-user hard limit on /// memory allocation for pipes has been reached. SystemResources, + /// The file is too large to be opened. This error is unreachable + /// for 64-bit targets, as well as when opening directories. + FileTooBig, + + /// The path refers to directory but the `O_DIRECTORY` flag was not provided. + IsDir, + + /// A new path cannot be created because the device has no room for the new file. + /// This error is only reachable when the `O_CREAT` flag is provided. NoSpaceLeft, + + /// A component used as a directory in the path was not, in fact, a directory, or + /// `O_DIRECTORY` was specified and the path was not a directory. NotDir, + + /// The path already exists and the `O_CREAT` and `O_EXCL` flags were provided. PathAlreadyExists, DeviceBusy, } || UnexpectedError; @@ -978,6 +992,42 @@ pub fn unlinkC(file_path: [*]const u8) UnlinkError!void { } } +pub const UnlinkatError = UnlinkError || error{ + /// When passing `AT_REMOVEDIR`, this error occurs when the named directory is not empty. + DirNotEmpty, +}; + +/// Delete a file name and possibly the file it refers to, based on an open directory handle. +pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + const file_path_c = try toPosixPath(file_path); + return unlinkatC(dirfd, &file_path_c, flags); +} + +/// Same as `unlinkat` but `file_path` is a null-terminated string. +pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { + 0 => return, + EACCES => return error.AccessDenied, + EPERM => return error.AccessDenied, + EBUSY => return error.FileBusy, + EFAULT => unreachable, + EIO => return error.FileSystem, + EISDIR => return error.IsDir, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + ENOMEM => return error.SystemResources, + EROFS => return error.ReadOnlyFileSystem, + ENOTEMPTY => return error.DirNotEmpty, + + EINVAL => unreachable, // invalid flags, or pathname has . as last component + EBADF => unreachable, // always a race condition + + else => |err| return unexpectedErrno(err), + } +} + const RenameError = error{ AccessDenied, FileBusy, diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index d36073abf3..5a0294fa7d 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -286,8 +286,10 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void var dir = try fs.Dir.open(file_path); defer dir.close(); - while (try dir.next()) |entry| { - if (entry.kind == fs.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { + var dir_it = dir.iterate(); + + while (try dir_it.next()) |entry| { + if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) { const full_path = try fs.path.join(fmt.allocator, [_][]const u8{ file_path, entry.name }); try fmtPath(fmt, full_path, check_mode); } From ddb87237de60cb0feacc606d21c9dd48c5c3bd4b Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Sun, 20 Oct 2019 22:56:58 +0200 Subject: [PATCH 18/31] Fix crash with peer type resolution & error unions Fixes #3353 --- src/ir.cpp | 12 ++++++++++++ test/compile_errors.zig | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/ir.cpp b/src/ir.cpp index 6bdfd86b29..4cce464a4c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -10120,6 +10120,18 @@ static ZigType *ir_resolve_peer_types(IrAnalyze *ira, AstNode *source_node, ZigT } else { err_set_type = cur_type; } + + if (!resolve_inferred_error_set(ira->codegen, err_set_type, cur_inst->source_node)) { + return ira->codegen->builtin_types.entry_invalid; + } + + if (type_is_global_error_set(err_set_type)) { + err_set_type = ira->codegen->builtin_types.entry_global_error_set; + continue; + } + + update_errors_helper(ira->codegen, &errors, &errors_count); + for (uint32_t i = 0; i < err_set_type->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = err_set_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 288fd501cf..35c6ded4a4 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -2,6 +2,16 @@ const tests = @import("tests.zig"); const builtin = @import("builtin"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "comparison with error union and error value", + \\export fn entry() void { + \\ var number_or_error: anyerror!i32 = error.SomethingAwful; + \\ _ = number_or_error == error.SomethingAwful; + \\} + , + "tmp.zig:3:25: error: operator not allowed for type 'anyerror!i32'", + ); + cases.add( "switch with overlapping case ranges", \\export fn entry() void { From ef67c497856fb97f7d2854991a72237ad4332810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 14:18:01 -0400 Subject: [PATCH 19/31] [wip] use NtDll APIs on Windows to implement std.fs.Dir --- lib/std/fs.zig | 365 +++++++++++++++++++++++++--------- lib/std/fs/file.zig | 1 + lib/std/fs/path.zig | 19 ++ lib/std/io.zig | 5 +- lib/std/os.zig | 79 ++++++++ lib/std/os/bits/windows.zig | 3 + lib/std/os/windows.zig | 23 ++- lib/std/os/windows/bits.zig | 146 +++++++++++++- lib/std/os/windows/ntdll.zig | 25 ++- lib/std/os/windows/status.zig | 6 + 10 files changed, 566 insertions(+), 106 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 89071b407f..51623d3057 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -353,18 +353,13 @@ pub fn deleteTree(full_path: []const u8) !void { return dir.deleteTree(path.basename(full_path)); } else { - return Dir.posix_cwd.deleteTree(full_path); + return Dir.cwd().deleteTree(full_path); } } pub const Dir = struct { fd: os.fd_t, - /// An open handle to the current working directory. - /// Closing this directory is safety-checked illegal behavior. - /// Not available on Windows. - pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; - pub const Entry = struct { name: []const u8, kind: Kind, @@ -386,12 +381,10 @@ pub const Dir = struct { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, seek: i64, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -407,27 +400,24 @@ pub const Dir = struct { fn nextDarwin(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.dir.fd, - &self.buf, - self.buf.len, - &self.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } - self.index = 0; - self.end_index = @intCast(usize, rc); - break; } + self.index = 0; + self.end_index = @intCast(usize, rc); } const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + darwin_entry.d_reclen; @@ -460,26 +450,23 @@ pub const Dir = struct { fn nextBsd(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.dir.fd, - self.buf[0..].ptr, - self.buf.len, - &self.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = @intCast(usize, rc); - break; + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); } const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + freebsd_entry.d_reclen; @@ -511,12 +498,10 @@ pub const Dir = struct { }, .linux => struct { dir: Dir, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent64)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -524,21 +509,18 @@ pub const Dir = struct { pub fn next(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = rc; - break; + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; } const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); const next_index = self.index + linux_entry.d_reclen; @@ -570,13 +552,117 @@ pub const Dir = struct { }, .windows => struct { dir: Dir, - find_file_data: os.windows.WIN32_FIND_DATAW, + buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)), + index: usize, + end_index: usize, first: bool, name_data: [256]u8, + + const Self = @This(); + + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + const w = os.windows; + if (self.index >= self.end_index) { + var io: w.IO_STATUS_BLOCK = undefined; + //var mask_buf = [2]u16{ 'a', 0 }; + //var mask = w.UNICODE_STRING{ + // .Length = 2, + // .MaximumLength = 2, + // .Buffer = &mask_buf, + //}; + const rc = w.ntdll.NtQueryDirectoryFile( + self.dir.fd, + null, + null, + null, + &io, + &self.buf, + self.buf.len, + .FileBothDirectoryInformation, + w.FALSE, + null, + if (self.first) w.BOOLEAN(w.TRUE) else w.BOOLEAN(w.FALSE), + ); + self.first = false; + if (io.Information == 0) return null; + self.index = 0; + self.end_index = io.Information; + switch (rc) { + w.STATUS.SUCCESS => {}, + else => return w.unexpectedStatus(rc), + } + } + + const aligned_ptr = @alignCast(@alignOf(w.FILE_BOTH_DIR_INFORMATION), &self.buf[self.index]); + const dir_info = @ptrCast(*w.FILE_BOTH_DIR_INFORMATION, aligned_ptr); + if (dir_info.NextEntryOffset != 0) { + self.index += dir_info.NextEntryOffset; + } else { + self.index = self.buf.len; + } + + const name_utf16le = @ptrCast([*]u16, &dir_info.FileName)[0 .. dir_info.FileNameLength / 2]; + + if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) + continue; + // Trust that Windows gives us valid UTF-16LE + const name_utf8_len = std.unicode.utf16leToUtf8(self.name_data[0..], name_utf16le) catch unreachable; + const name_utf8 = self.name_data[0..name_utf8_len]; + const kind = blk: { + const attrs = dir_info.FileAttributes; + if (attrs & w.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; + if (attrs & w.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; + break :blk Entry.Kind.File; + }; + return Entry{ + .name = name_utf8, + .kind = kind, + }; + } + } }, else => @compileError("unimplemented"), }; + pub fn iterate(self: Dir) Iterator { + switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .first = true, + .buf = undefined, + .name_data = undefined, + }, + else => @compileError("unimplemented"), + } + } + + /// Returns an open handle to the current working directory. + /// Closing the returned `Dir` is checked illegal behavior. + /// On POSIX targets, this function is comptime-callable. + pub fn cwd() Dir { + if (os.windows.is_the_target) { + return Dir{ .fd = os.windows.peb().ProcessParameters.CurrentDirectory.Handle }; + } else { + return Dir{ .fd = os.AT_FDCWD }; + } + } + pub const OpenError = error{ FileNotFound, NotDir, @@ -594,18 +680,15 @@ pub const Dir = struct { /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return posix_cwd.openDir(dir_path); + return cwd().openDir(dir_path); } /// Same as `open` except the parameter is null-terminated. pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { - return posix_cwd.openDirC(dir_path_c); + return cwd().openDirC(dir_path_c); } pub fn close(self: *Dir) void { - if (os.windows.is_the_target) { - @panic("TODO"); - } os.close(self.fd); self.* = undefined; } @@ -625,14 +708,25 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + std.debug.warn("openDir {}\n", sub_path); + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.openDirW(&sub_path_w); + } + const sub_path_c = try os.toPosixPath(sub_path); return self.openDirC(&sub_path_c); } - /// Call `close` on the result when done. - pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + /// Same as `openDir` except the parameter is null-terminated. + pub fn openDirC(self: Dir, sub_path_c: [*]const u8) OpenError!Dir { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); + return self.openDirW(&sub_path_w); + } + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; - const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + const fd = os.openatC(self.fd, sub_path_c, flags, 0) catch |err| switch (err) { error.FileTooBig => unreachable, // can't happen for directories error.IsDir => unreachable, // we're providing O_DIRECTORY error.NoSpaceLeft => unreachable, // not providing O_CREAT @@ -642,6 +736,79 @@ pub const Dir = struct { return Dir{ .fd = fd }; } + /// Same as `openDir` except the path parameter is UTF16LE, NT-prefixed. + /// This function is Windows-only. + pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { + const w = os.windows; + var result = Dir{ + .fd = undefined, + }; + //var mask: ?[*]const u16 = undefined; + //var nt_name: w.UNICODE_STRING = undefined; + //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + // return error.FileNotFound; + //} + //defer w.ntdll.RtlFreeUnicodeString(&nt_name); + //if (mask) |m| { + // if (m[0] == 0) { + // return error.FileNotFound; + // } else { + // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); + // } + //} else { + // return error.FileNotFound; + //} + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (path.isAbsoluteW(sub_path_w)) null else self.fd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + var io: w.IO_STATUS_BLOCK = undefined; + const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; + //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); + var buf: [200]u8 = undefined; + //var buf2: [200]u8 = undefined; + const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; + //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; + std.debug.warn("path: {}\n", buf[0..len]); + //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); + const rc = w.ntdll.NtCreateFile( + &result.fd, + w.GENERIC_READ | w.SYNCHRONIZE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT, + null, + 0, + ); + std.debug.warn("result.fd = {}\n", result.fd); + switch (rc) { + w.STATUS.SUCCESS => return result, + w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => { + @panic("invalid parameter"); + }, + else => return w.unexpectedStatus(rc), + } + } + pub const DeleteFileError = os.UnlinkError; /// Delete a file name and possibly the file it refers to, based on an open directory handle. @@ -677,6 +844,10 @@ pub const Dir = struct { /// Returns `error.DirNotEmpty` if the directory is not empty. /// To delete a directory recursively, see `deleteTree`. pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.deleteDirW(&sub_path_w); + } const sub_path_c = try os.toPosixPath(sub_path); return self.deleteDirC(&sub_path_c); } @@ -689,24 +860,25 @@ pub const Dir = struct { }; } - pub fn iterate(self: Dir) Iterator { - switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => return Iterator{ - .dir = self, - .seek = 0, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .linux => return Iterator{ - .dir = self, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .windows => @panic("TODO"), - else => @compileError("unimplemented"), - } + /// Same as `deleteDir` except the parameter is UTF16LE, NT prefixed. + /// This function is Windows-only. + pub fn deleteDirW(self: Dir, sub_path_w: [*]const u16) DeleteDirError!void { + os.unlinkatW(self.fd, sub_path_w, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + /// Read value of a symbolic link. + /// The return value is a slice of `buffer`, from index `0`. + pub fn readLink(self: Dir, sub_path: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + const sub_path_c = try os.toPosixPath(sub_path); + return self.readLinkC(&sub_path_c, buffer); + } + + /// Same as `readLink`, except the `pathname` parameter is null-terminated. + pub fn readLinkC(self: Dir, sub_path_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkatC(self.fd, sub_path_c, buffer); } pub const DeleteTreeError = error{ @@ -953,7 +1125,6 @@ pub const Walker = struct { /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. /// The order of returned file system entries is undefined. -/// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); @@ -978,15 +1149,13 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { /// Read value of a symbolic link. /// The return value is a slice of buffer, from index `0`. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLink(pathname: []const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { +pub fn readLink(pathname: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { return os.readlink(pathname, buffer); } -/// Same as `readLink`, except the `pathname` parameter is null-terminated. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLinkC(pathname: [*]const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { - return os.readlinkC(pathname, buffer); +/// Same as `readLink`, except the parameter is null-terminated. +pub fn readLinkC(pathname_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkC(pathname_c, buffer); } pub const OpenSelfExeError = os.OpenError || os.windows.CreateFileError || SelfExePathError; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 32c221d063..a9212d69d4 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -243,6 +243,7 @@ pub const File = struct { switch (rc) { windows.STATUS.SUCCESS => {}, windows.STATUS.BUFFER_OVERFLOW => {}, + windows.STATUS.INVALID_PARAMETER => unreachable, else => return windows.unexpectedStatus(rc), } return Stat{ diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 2bb23f04ce..560d9b58f3 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -136,6 +136,25 @@ pub fn isAbsolute(path: []const u8) bool { } } +pub fn isAbsoluteW(path_w: [*]const u16) bool { + if (path_w[0] == '/') + return true; + + if (path_w[0] == '\\') { + return true; + } + if (path_w[0] == 0 or path_w[1] == 0 or path_w[2] == 0) { + return false; + } + if (path_w[1] == ':') { + if (path_w[2] == '/') + return true; + if (path_w[2] == '\\') + return true; + } + return false; +} + pub fn isAbsoluteWindows(path: []const u8) bool { if (path[0] == '/') return true; diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b3..19623d93ee 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -127,6 +127,7 @@ pub fn OutStream(comptime WriteError: type) type { }; } +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn writeFile(path: []const u8, data: []const u8) !void { var file = try File.openWrite(path); defer file.close(); @@ -134,11 +135,13 @@ pub fn writeFile(path: []const u8, data: []const u8) !void { } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return readFileAllocAligned(allocator, path, @alignOf(u8)); } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAllocAligned(allocator: *mem.Allocator, path: []const u8, comptime A: u29) ![]align(A) u8 { var file = try File.openRead(path); defer file.close(); @@ -1084,7 +1087,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/os.zig b/lib/std/os.zig index e7319095cb..2d8ddb0abc 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -999,12 +999,20 @@ pub const UnlinkatError = UnlinkError || error{ /// Delete a file name and possibly the file it refers to, based on an open directory handle. pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.sliceToPrefixedFileW(file_path); + return unlinkatW(dirfd, &file_path_w, flags); + } const file_path_c = try toPosixPath(file_path); return unlinkatC(dirfd, &file_path_c, flags); } /// Same as `unlinkat` but `file_path` is a null-terminated string. pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path_c); + return unlinkatW(dirfd, &file_path_w, flags); + } switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { 0 => return, EACCES => return error.AccessDenied, @@ -1028,6 +1036,56 @@ pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatErro } } +/// Same as `unlinkat` but `sub_path_w` is UTF16LE, NT prefixed. Windows only. +pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatError!void { + const w = windows; + + const want_rmdir_behavior = (flags & AT_REMOVEDIR) != 0; + const create_options_flags = if (want_rmdir_behavior) + w.ULONG(w.FILE_DELETE_ON_CLOSE) + else + w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); + var nt_name: w.UNICODE_STRING = undefined; + if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + return error.FileNotFound; + } + defer w.ntdll.RtlFreeUnicodeString(&nt_name); + + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = dirfd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var io: w.IO_STATUS_BLOCK = undefined; + var tmp_handle: w.HANDLE = undefined; + var rc = w.ntdll.NtCreateFile( + &tmp_handle, + w.SYNCHRONIZE | w.DELETE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + create_options_flags, + null, + 0, + ); + if (rc == w.STATUS.SUCCESS) { + rc = w.ntdll.NtClose(tmp_handle); + } + switch (rc) { + w.STATUS.SUCCESS => return, + w.STATUS.OBJECT_NAME_INVALID => unreachable, + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => unreachable, + else => return w.unexpectedStatus(rc), + } +} + const RenameError = error{ AccessDenied, FileBusy, @@ -1287,6 +1345,27 @@ pub fn readlinkC(file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { } } +pub fn readlinkatC(dirfd: fd_t, file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path); + @compileError("TODO implement readlink for Windows"); + } + const rc = system.readlinkat(dirfd, file_path, out_buffer.ptr, out_buffer.len); + switch (errno(rc)) { + 0 => return out_buffer[0..@bitCast(usize, rc)], + EACCES => return error.AccessDenied, + EFAULT => unreachable, + EINVAL => unreachable, + EIO => return error.FileSystem, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOMEM => return error.SystemResources, + ENOTDIR => return error.NotDir, + else => |err| return unexpectedErrno(err), + } +} + pub const SetIdError = error{ ResourceLimitReached, InvalidUserId, diff --git a/lib/std/os/bits/windows.zig b/lib/std/os/bits/windows.zig index fc148d812f..c261b52bed 100644 --- a/lib/std/os/bits/windows.zig +++ b/lib/std/os/bits/windows.zig @@ -158,3 +158,6 @@ pub const EWOULDBLOCK = 140; pub const EDQUOT = 10069; pub const F_OK = 0; + +/// Remove directory instead of unlinking file +pub const AT_REMOVEDIR = 0x200; diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8..6a5d03b529 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -792,6 +792,25 @@ pub fn SetFileTime( } } +pub fn peb() *PEB { + switch (builtin.arch) { + .i386 => { + return asm ( + \\ mov %%fs:0x18, %[ptr] + \\ mov %%ds:0x30(%[ptr]), %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + .x86_64 => { + return asm ( + \\ mov %%gs:0x60, %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + else => @compileError("unsupported architecture"), + } +} + /// A file time is a 64-bit value that represents the number of 100-nanosecond /// intervals that have elapsed since 12:00 A.M. January 1, 1601 Coordinated /// Universal Time (UTC). @@ -844,8 +863,8 @@ pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) else => {}, } } - const start_index = if (mem.startsWith(u8, s, "\\\\") or !std.fs.path.isAbsolute(s)) 0 else blk: { - const prefix = [_]u16{ '\\', '\\', '?', '\\' }; + const start_index = if (mem.startsWith(u8, s, "\\?") or !std.fs.path.isAbsolute(s)) 0 else blk: { + const prefix = [_]u16{ '\\', '?', '?', '\\' }; mem.copy(u16, result[0..], prefix); break :blk prefix.len; }; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index ddfdd27e1b..68d5ab6b87 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -300,6 +300,44 @@ pub const FILE_SHARE_DELETE = 0x00000004; pub const FILE_SHARE_READ = 0x00000001; pub const FILE_SHARE_WRITE = 0x00000002; +pub const DELETE = 0x00010000; +pub const READ_CONTROL = 0x00020000; +pub const WRITE_DAC = 0x00040000; +pub const WRITE_OWNER = 0x00080000; +pub const SYNCHRONIZE = 0x00100000; +pub const STANDARD_RIGHTS_REQUIRED = 0x000f0000; + +// disposition for NtCreateFile +pub const FILE_SUPERSEDE = 0; +pub const FILE_OPEN = 1; +pub const FILE_CREATE = 2; +pub const FILE_OPEN_IF = 3; +pub const FILE_OVERWRITE = 4; +pub const FILE_OVERWRITE_IF = 5; +pub const FILE_MAXIMUM_DISPOSITION = 5; + +// flags for NtCreateFile and NtOpenFile +pub const FILE_DIRECTORY_FILE = 0x00000001; +pub const FILE_WRITE_THROUGH = 0x00000002; +pub const FILE_SEQUENTIAL_ONLY = 0x00000004; +pub const FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008; +pub const FILE_SYNCHRONOUS_IO_ALERT = 0x00000010; +pub const FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020; +pub const FILE_NON_DIRECTORY_FILE = 0x00000040; +pub const FILE_CREATE_TREE_CONNECTION = 0x00000080; +pub const FILE_COMPLETE_IF_OPLOCKED = 0x00000100; +pub const FILE_NO_EA_KNOWLEDGE = 0x00000200; +pub const FILE_OPEN_FOR_RECOVERY = 0x00000400; +pub const FILE_RANDOM_ACCESS = 0x00000800; +pub const FILE_DELETE_ON_CLOSE = 0x00001000; +pub const FILE_OPEN_BY_FILE_ID = 0x00002000; +pub const FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000; +pub const FILE_NO_COMPRESSION = 0x00008000; +pub const FILE_RESERVE_OPFILTER = 0x00100000; +pub const FILE_TRANSACTED_MODE = 0x00200000; +pub const FILE_OPEN_OFFLINE_FILE = 0x00400000; +pub const FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000; + pub const CREATE_ALWAYS = 2; pub const CREATE_NEW = 1; pub const OPEN_ALWAYS = 4; @@ -720,15 +758,117 @@ pub const VECTORED_EXCEPTION_HANDLER = stdcallcc fn (ExceptionInfo: *EXCEPTION_P pub const OBJECT_ATTRIBUTES = extern struct { Length: ULONG, - RootDirectory: HANDLE, + RootDirectory: ?HANDLE, ObjectName: *UNICODE_STRING, Attributes: ULONG, SecurityDescriptor: ?*c_void, SecurityQualityOfService: ?*c_void, }; +pub const OBJ_INHERIT = 0x00000002; +pub const OBJ_PERMANENT = 0x00000010; +pub const OBJ_EXCLUSIVE = 0x00000020; +pub const OBJ_CASE_INSENSITIVE = 0x00000040; +pub const OBJ_OPENIF = 0x00000080; +pub const OBJ_OPENLINK = 0x00000100; +pub const OBJ_KERNEL_HANDLE = 0x00000200; +pub const OBJ_VALID_ATTRIBUTES = 0x000003F2; + pub const UNICODE_STRING = extern struct { - Length: USHORT, - MaximumLength: USHORT, + Length: c_ushort, + MaximumLength: c_ushort, Buffer: [*]WCHAR, }; + +pub const PEB = extern struct { + Reserved1: [2]BYTE, + BeingDebugged: BYTE, + Reserved2: [1]BYTE, + Reserved3: [2]PVOID, + Ldr: *PEB_LDR_DATA, + ProcessParameters: *RTL_USER_PROCESS_PARAMETERS, + Reserved4: [3]PVOID, + AtlThunkSListPtr: PVOID, + Reserved5: PVOID, + Reserved6: ULONG, + Reserved7: PVOID, + Reserved8: ULONG, + AtlThunkSListPtr32: ULONG, + Reserved9: [45]PVOID, + Reserved10: [96]BYTE, + PostProcessInitRoutine: PPS_POST_PROCESS_INIT_ROUTINE, + Reserved11: [128]BYTE, + Reserved12: [1]PVOID, + SessionId: ULONG, +}; + +pub const PEB_LDR_DATA = extern struct { + Reserved1: [8]BYTE, + Reserved2: [3]PVOID, + InMemoryOrderModuleList: LIST_ENTRY, +}; + +pub const RTL_USER_PROCESS_PARAMETERS = extern struct { + AllocationSize: ULONG, + Size: ULONG, + Flags: ULONG, + DebugFlags: ULONG, + ConsoleHandle: HANDLE, + ConsoleFlags: ULONG, + hStdInput: HANDLE, + hStdOutput: HANDLE, + hStdError: HANDLE, + CurrentDirectory: CURDIR, + DllPath: UNICODE_STRING, + ImagePathName: UNICODE_STRING, + CommandLine: UNICODE_STRING, + Environment: [*]WCHAR, + dwX: ULONG, + dwY: ULONG, + dwXSize: ULONG, + dwYSize: ULONG, + dwXCountChars: ULONG, + dwYCountChars: ULONG, + dwFillAttribute: ULONG, + dwFlags: ULONG, + dwShowWindow: ULONG, + WindowTitle: UNICODE_STRING, + Desktop: UNICODE_STRING, + ShellInfo: UNICODE_STRING, + RuntimeInfo: UNICODE_STRING, + DLCurrentDirectory: [0x20]RTL_DRIVE_LETTER_CURDIR, +}; + +pub const RTL_DRIVE_LETTER_CURDIR = extern struct { + Flags: c_ushort, + Length: c_ushort, + TimeStamp: ULONG, + DosPath: UNICODE_STRING, +}; + +pub const PPS_POST_PROCESS_INIT_ROUTINE = ?extern fn () void; + +pub const FILE_BOTH_DIR_INFORMATION = extern struct { + NextEntryOffset: ULONG, + FileIndex: ULONG, + CreationTime: LARGE_INTEGER, + LastAccessTime: LARGE_INTEGER, + LastWriteTime: LARGE_INTEGER, + ChangeTime: LARGE_INTEGER, + EndOfFile: LARGE_INTEGER, + AllocationSize: LARGE_INTEGER, + FileAttributes: ULONG, + FileNameLength: ULONG, + EaSize: ULONG, + ShortNameLength: CHAR, + ShortName: [12]WCHAR, + FileName: [1]WCHAR, +}; +pub const FILE_BOTH_DIRECTORY_INFORMATION = FILE_BOTH_DIR_INFORMATION; + +pub const IO_APC_ROUTINE = extern fn (PVOID, *IO_STATUS_BLOCK, ULONG) void; + +pub const CURDIR = extern struct { + DosPath: UNICODE_STRING, + Handle: HANDLE, +}; diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index bfc98aba8a..f914920093 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -13,12 +13,33 @@ pub extern "NtDll" stdcallcc fn NtCreateFile( DesiredAccess: ACCESS_MASK, ObjectAttributes: *OBJECT_ATTRIBUTES, IoStatusBlock: *IO_STATUS_BLOCK, - AllocationSize: *LARGE_INTEGER, + AllocationSize: ?*LARGE_INTEGER, FileAttributes: ULONG, ShareAccess: ULONG, CreateDisposition: ULONG, CreateOptions: ULONG, - EaBuffer: *c_void, + EaBuffer: ?*c_void, EaLength: ULONG, ) NTSTATUS; pub extern "NtDll" stdcallcc fn NtClose(Handle: HANDLE) NTSTATUS; +pub extern "NtDll" stdcallcc fn RtlDosPathNameToNtPathName_U( + DosPathName: [*]const u16, + NtPathName: *UNICODE_STRING, + NtFileNamePart: ?*?[*]const u16, + DirectoryInfo: ?*CURDIR, +) BOOL; +pub extern "NtDll" stdcallcc fn RtlFreeUnicodeString(UnicodeString: *UNICODE_STRING) void; + +pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + FileInformation: *c_void, + Length: ULONG, + FileInformationClass: FILE_INFORMATION_CLASS, + ReturnSingleEntry: BOOLEAN, + FileName: ?*UNICODE_STRING, + RestartScan: BOOLEAN, +) NTSTATUS; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index b9fd2b495f..d381e3b2e1 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -3,3 +3,9 @@ pub const SUCCESS = 0x00000000; /// The data was too large to fit into the specified buffer. pub const BUFFER_OVERFLOW = 0x80000005; + +pub const INVALID_PARAMETER = 0xC000000D; +pub const ACCESS_DENIED = 0xC0000022; +pub const OBJECT_NAME_INVALID = 0xC0000033; +pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; From 16a91e6a4591d3d1198c0f0eaee9266333afd35e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 18:35:14 -0400 Subject: [PATCH 20/31] adjust test runner to print the test name when error occurs --- lib/std/progress.zig | 4 ++-- lib/std/special/test_runner.zig | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/std/progress.zig b/lib/std/progress.zig index 1d29763c10..fba77092b3 100644 --- a/lib/std/progress.zig +++ b/lib/std/progress.zig @@ -155,11 +155,11 @@ pub const Progress = struct { } if (node.estimated_total_items) |total| { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}/{}] ", node.completed_items, total); + self.bufWrite(&end, "[{}/{}] ", node.completed_items + 1, total); need_ellipse = false; } else if (node.completed_items != 0) { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}] ", node.completed_items); + self.bufWrite(&end, "[{}] ", node.completed_items + 1); need_ellipse = false; } } diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d..c16484d162 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -24,11 +24,14 @@ pub fn main() anyerror!void { test_node.end(); progress.log("{}...SKIP\n", test_fn.name); }, - else => return err, + else => { + progress.log(""); + return err; + }, } } root_node.end(); if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } From 859cbef8a41dfbfde9c8caab00272b77912d3f0b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:01:08 -0400 Subject: [PATCH 21/31] test runner: refresh display before running test This makes it so that when a test fails quickly, and writes output, it will do so after the test name is printed. --- lib/std/special/test_runner.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index c16484d162..bc57b3b925 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,6 +15,7 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + progress.refresh(); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); From 87f632b08a04f4a03facfa220df1cbc466a5de39 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:17:33 -0400 Subject: [PATCH 22/31] fs.Dir.openDir: use empty object name for "." on Windows --- CMakeLists.txt | 3 +++ lib/std/fs.zig | 35 +++++++++++++++++++++++++++------ lib/std/os/windows.zig | 4 +++- lib/std/os/windows/bits.zig | 2 ++ lib/std/os/windows/kernel32.zig | 2 ++ lib/std/os/windows/status.zig | 1 + 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a34d034dd..cd1ba9d3f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,5 +630,8 @@ set_target_properties(zig PROPERTIES LINK_FLAGS ${EXE_LDFLAGS} ) target_link_libraries(zig compiler "${LIBUSERLAND}") +if(MSVC) + target_link_libraries(zig ntdll.lib) +endif() add_dependencies(zig zig_build_libuserland) install(TARGETS zig DESTINATION bin) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 51623d3057..8f9f8c17d9 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -590,6 +590,7 @@ pub const Dir = struct { self.end_index = io.Information; switch (rc) { w.STATUS.SUCCESS => {}, + w.STATUS.ACCESS_DENIED => return error.AccessDenied, else => return w.unexpectedStatus(rc), } } @@ -708,7 +709,7 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - std.debug.warn("openDir {}\n", sub_path); + // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -740,9 +741,26 @@ pub const Dir = struct { /// This function is Windows-only. pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { const w = os.windows; + var result = Dir{ .fd = undefined, }; + + const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; + + // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. + + // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { + // switch (w.kernel32.GetLastError()) { + // else => |err| return w.unexpectedError(err), + // } + + // @panic("handle DuplicateHandle error"); + // } + // return result; + // } + //var mask: ?[*]const u16 = undefined; //var nt_name: w.UNICODE_STRING = undefined; //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { @@ -760,7 +778,7 @@ pub const Dir = struct { //} const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -774,7 +792,11 @@ pub const Dir = struct { .SecurityDescriptor = null, .SecurityQualityOfService = null, }; - std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; + } + // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); var io: w.IO_STATUS_BLOCK = undefined; const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); @@ -782,11 +804,11 @@ pub const Dir = struct { //var buf2: [200]u8 = undefined; const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - std.debug.warn("path: {}\n", buf[0..len]); + // std.debug.warn("path: {}\n", buf[0..len]); //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - w.GENERIC_READ | w.SYNCHRONIZE, + desired_access, &attr, &io, null, @@ -797,11 +819,12 @@ pub const Dir = struct { null, 0, ); - std.debug.warn("result.fd = {}\n", result.fd); + // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => { @panic("invalid parameter"); }, diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 6a5d03b529..93a5a65a92 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -20,6 +20,8 @@ pub const shell32 = @import("windows/shell32.zig"); pub usingnamespace @import("windows/bits.zig"); +pub const self_process_handle = @intToPtr(HANDLE, maxInt(usize)); + /// `builtin` is missing `subsystem` when the subsystem is automatically detected, /// so Zig standard library has the subsystem detection logic here. This should generally be /// used rather than `builtin.subsystem`. @@ -898,7 +900,7 @@ pub fn unexpectedError(err: DWORD) std.os.UnexpectedError { /// and you get an unexpected status. pub fn unexpectedStatus(status: NTSTATUS) std.os.UnexpectedError { if (std.os.unexpected_error_tracing) { - std.debug.warn("error.Unexpected NTSTATUS={}\n", status); + std.debug.warn("error.Unexpected NTSTATUS=0x{x}\n", status); std.debug.dumpCurrentStackTrace(null); } return error.Unexpected; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index 68d5ab6b87..214f75186f 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -872,3 +872,5 @@ pub const CURDIR = extern struct { DosPath: UNICODE_STRING, Handle: HANDLE, }; + +pub const DUPLICATE_SAME_ACCESS = 2; \ No newline at end of file diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 2ae73ad45a..736d81ae58 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -47,6 +47,8 @@ pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ pub extern "kernel32" stdcallcc fn DeleteFileW(lpFileName: [*]const u16) BOOL; +pub extern "kernel32" stdcallcc fn DuplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, hTargetProcessHandle: HANDLE, lpTargetHandle: *HANDLE, dwDesiredAccess: DWORD, bInheritHandle: BOOL, dwOptions: DWORD) BOOL; + pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; pub extern "kernel32" stdcallcc fn FindFirstFileW(lpFileName: [*]const u16, lpFindFileData: *WIN32_FIND_DATAW) HANDLE; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index d381e3b2e1..424cae5e29 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -8,4 +8,5 @@ pub const INVALID_PARAMETER = 0xC000000D; pub const ACCESS_DENIED = 0xC0000022; pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; From 98c4365b66e6184aa867ec7d5d1d01435ec99798 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:39:30 -0400 Subject: [PATCH 23/31] darwin: add AT_* bits --- lib/std/os/bits/darwin.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index b076f95884..fe3156bb90 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -1178,3 +1178,17 @@ pub fn S_IWHT(m: u32) bool { return m & S_IFMT == S_IFWHT; } pub const HOST_NAME_MAX = 72; + +pub const AT_FDCWD = -2; + +/// Use effective ids in access check +pub const AT_EACCESS = 0x0010; + +/// Act on the symlink itself not the target +pub const AT_SYMLINK_NOFOLLOW = 0x0020; + +/// Act on target of symlink +pub const AT_SYMLINK_FOLLOW = 0x0040; + +/// Path refers to directory +pub const AT_REMOVEDIR = 0x0080; From 5917572e59ff68e33ae7fb0ec7b8d592e2915463 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:43 -0400 Subject: [PATCH 24/31] cleanup --- lib/std/fs.zig | 76 +++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 8f9f8c17d9..3fdd175e77 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -377,6 +377,8 @@ pub const Dir = struct { }; }; + const IteratorError = error{AccessDenied} || os.UnexpectedError; + pub const Iterator = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, @@ -387,9 +389,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { switch (builtin.os) { .macosx, .ios => return self.nextDarwin(), .freebsd, .netbsd => return self.nextBsd(), @@ -504,9 +508,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { if (self.index >= self.end_index) { const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); @@ -560,17 +566,13 @@ pub const Dir = struct { const Self = @This(); - pub fn next(self: *Self) !?Entry { + pub const Error = IteratorError; + + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { const w = os.windows; if (self.index >= self.end_index) { var io: w.IO_STATUS_BLOCK = undefined; - //var mask_buf = [2]u16{ 'a', 0 }; - //var mask = w.UNICODE_STRING{ - // .Length = 2, - // .MaximumLength = 2, - // .Buffer = &mask_buf, - //}; const rc = w.ntdll.NtQueryDirectoryFile( self.dir.fd, null, @@ -709,7 +711,6 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -746,39 +747,7 @@ pub const Dir = struct { .fd = undefined, }; - const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; - - // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { - // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. - - // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { - // switch (w.kernel32.GetLastError()) { - // else => |err| return w.unexpectedError(err), - // } - - // @panic("handle DuplicateHandle error"); - // } - // return result; - // } - - //var mask: ?[*]const u16 = undefined; - //var nt_name: w.UNICODE_STRING = undefined; - //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - // return error.FileNotFound; - //} - //defer w.ntdll.RtlFreeUnicodeString(&nt_name); - //if (mask) |m| { - // if (m[0] == 0) { - // return error.FileNotFound; - // } else { - // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); - // } - //} else { - // return error.FileNotFound; - //} - const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -796,19 +765,15 @@ pub const Dir = struct { // Windows does not recognize this, but it does work with empty string. nt_name.Length = 0; } - // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // If you're looking to contribute to zig and fix this, see here for an example of how to + // implement this: https://git.midipix.org/ntapi/tree/src/fs/ntapi_tt_open_physical_parent_directory.c + @panic("TODO opening '..' with a relative directory handle is not yet implemented on Windows"); + } var io: w.IO_STATUS_BLOCK = undefined; - const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; - //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); - var buf: [200]u8 = undefined; - //var buf2: [200]u8 = undefined; - const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; - //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - // std.debug.warn("path: {}\n", buf[0..len]); - //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - desired_access, + w.GENERIC_READ | w.SYNCHRONIZE, &attr, &io, null, @@ -819,15 +784,12 @@ pub const Dir = struct { null, 0, ); - // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, - w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - w.STATUS.INVALID_PARAMETER => { - @panic("invalid parameter"); - }, + w.STATUS.INVALID_PARAMETER => unreachable, else => return w.unexpectedStatus(rc), } } From 9a82f00a4fb897acf50446a725768c3aaa562513 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:52 -0400 Subject: [PATCH 25/31] add freebsd AT_* bits --- lib/std/os/bits/freebsd.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index 3d07e92e01..4427acb334 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -939,3 +939,23 @@ pub fn S_IWHT(m: u32) bool { } pub const HOST_NAME_MAX = 255; + +/// Magic value that specify the use of the current working directory +/// to determine the target of relative file paths in the openat() and +/// similar syscalls. +pub const AT_FDCWD = -100; + +/// Check access using effective user and group ID +pub const AT_EACCESS = 0x0100; + +/// Do not follow symbolic links +pub const AT_SYMLINK_NOFOLLOW = 0x0200; + +/// Follow symbolic link +pub const AT_SYMLINK_FOLLOW = 0x0400; + +/// Remove directory instead of file +pub const AT_REMOVEDIR = 0x0800; + +/// Fail if not under dirfd +pub const AT_BENEATH = 0x1000; From 65165554d0c6cf39ff091ace5ab5d48106a935fd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:49:35 -0400 Subject: [PATCH 26/31] test runner: restore previous behavior when... ...stderr does not support ansi escape codes --- lib/std/special/test_runner.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d..78db63e4d6 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,14 +15,17 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + if (progress.terminal == null) std.debug.warn("{}/{} {}...", i + 1, test_fn_list.len, test_fn.name); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); + if (progress.terminal == null) std.debug.warn("OK\n"); } else |err| switch (err) { error.SkipZigTest => { skip_count += 1; test_node.end(); progress.log("{}...SKIP\n", test_fn.name); + if (progress.terminal == null) std.debug.warn("SKIP\n"); }, else => return err, } @@ -30,5 +33,6 @@ pub fn main() anyerror!void { root_node.end(); if (ok_count != test_fn_list.len) { progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } From fc6f84f3f08aca0d4e2352091cc2db2ca5779c7d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:58:53 -0400 Subject: [PATCH 27/31] fix os.unlinkatW implementation --- lib/std/os.zig | 22 ++++++++++++++++++---- lib/std/os/windows/status.zig | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2d8ddb0abc..7a5eeda82c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1045,11 +1045,24 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.ULONG(w.FILE_DELETE_ON_CLOSE) else w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); - var nt_name: w.UNICODE_STRING = undefined; - if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - return error.FileNotFound; + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + // The Windows API makes this mutable, but it will not mutate here. + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; } - defer w.ntdll.RtlFreeUnicodeString(&nt_name); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // Can't remove the parent directory with an open handle. + return error.FileBusy; + } + var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), @@ -1082,6 +1095,7 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => unreachable, + w.STATUS.FILE_IS_A_DIRECTORY => return error.IsDir, else => return w.unexpectedStatus(rc), } } diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index 424cae5e29..5f38948620 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -10,3 +10,4 @@ pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; +pub const FILE_IS_A_DIRECTORY = 0xC00000BA; From 064377be9aaf409bf483f512354ef22f656bf213 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 22:20:45 -0400 Subject: [PATCH 28/31] test runner: restore previous behavior of printing all tests passed --- lib/std/special/test_runner.zig | 7 ++++--- test/cli.zig | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 78db63e4d6..f835401e1c 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -31,8 +31,9 @@ pub fn main() anyerror!void { } } root_node.end(); - if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); - if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); + if (ok_count == test_fn_list.len) { + std.debug.warn("All tests passed.\n"); + } else { + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } diff --git a/test/cli.zig b/test/cli.zig index 2999df14b0..6095536422 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -87,7 +87,7 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, [_][]const u8{ zig_exe, "init-lib" }); const test_result = try exec(dir_path, [_][]const u8{ zig_exe, "build", "test" }); - testing.expect(std.mem.eql(u8, test_result.stderr, "")); + testing.expect(std.mem.endsWith(u8, test_result.stderr, "All tests passed.\n")); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { From 2550cb4638e12e94073cbca5c0ee3aa263c25dbe Mon Sep 17 00:00:00 2001 From: Vexu <15308111+Vexu@users.noreply.github.com> Date: Mon, 21 Oct 2019 22:22:08 +0300 Subject: [PATCH 29/31] remove pub syntax for container fields --- doc/langref.html.in | 4 ++-- lib/std/child_process.zig | 32 ++++++++++++++++---------------- lib/std/heap.zig | 2 +- lib/std/http/headers.zig | 6 +++--- lib/std/io.zig | 20 ++++++++++---------- lib/std/io/seekable_stream.zig | 4 ++-- lib/std/zig/ast.zig | 4 +--- lib/std/zig/parse.zig | 16 +++++++--------- lib/std/zig/parser_test.zig | 6 +++--- lib/std/zig/render.zig | 8 ++------ src/all_types.hpp | 2 -- src/parser.cpp | 16 +++++++--------- 12 files changed, 54 insertions(+), 66 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index a30670b0c7..17792d6470 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -10086,8 +10086,8 @@ ContainerMembers <- TestDecl ContainerMembers / TopLevelComptime ContainerMembers / KEYWORD_pub? TopLevelDecl ContainerMembers - / KEYWORD_pub? ContainerField COMMA ContainerMembers - / KEYWORD_pub? ContainerField + / ContainerField COMMA ContainerMembers + / ContainerField / TestDecl <- KEYWORD_test STRINGLITERAL Block diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 7fb663ac28..d786c3f4de 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -17,35 +17,35 @@ const TailQueue = std.TailQueue; const maxInt = std.math.maxInt; pub const ChildProcess = struct { - pub pid: if (os.windows.is_the_target) void else i32, - pub handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, + pid: if (os.windows.is_the_target) void else i32, + handle: if (os.windows.is_the_target) windows.HANDLE else void, + thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub allocator: *mem.Allocator, + allocator: *mem.Allocator, - pub stdin: ?File, - pub stdout: ?File, - pub stderr: ?File, + stdin: ?File, + stdout: ?File, + stderr: ?File, - pub term: ?(SpawnError!Term), + term: ?(SpawnError!Term), - pub argv: []const []const u8, + argv: []const []const u8, /// Leave as null to use the current env map using the supplied allocator. - pub env_map: ?*const BufMap, + env_map: ?*const BufMap, - pub stdin_behavior: StdIo, - pub stdout_behavior: StdIo, - pub stderr_behavior: StdIo, + stdin_behavior: StdIo, + stdout_behavior: StdIo, + stderr_behavior: StdIo, /// Set to change the user id when spawning the child process. - pub uid: if (os.windows.is_the_target) void else ?u32, + uid: if (os.windows.is_the_target) void else ?u32, /// Set to change the group id when spawning the child process. - pub gid: if (os.windows.is_the_target) void else ?u32, + gid: if (os.windows.is_the_target) void else ?u32, /// Set to change the current working directory when spawning the child process. - pub cwd: ?[]const u8, + cwd: ?[]const u8, err_pipe: if (os.windows.is_the_target) void else [2]os.fd_t, llnode: if (os.windows.is_the_target) void else TailQueue(*ChildProcess).Node, diff --git a/lib/std/heap.zig b/lib/std/heap.zig index b968b6242f..f3a0d457d2 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -338,7 +338,7 @@ pub const HeapAllocator = switch (builtin.os) { /// This allocator takes an existing allocator, wraps it, and provides an interface /// where you can allocate without freeing, and then free it all together. pub const ArenaAllocator = struct { - pub allocator: Allocator, + allocator: Allocator, child_allocator: *Allocator, buffer_list: std.SinglyLinkedList([]u8), diff --git a/lib/std/http/headers.zig b/lib/std/http/headers.zig index a8dfa68629..7ee035ce80 100644 --- a/lib/std/http/headers.zig +++ b/lib/std/http/headers.zig @@ -28,9 +28,9 @@ fn never_index_default(name: []const u8) bool { const HeaderEntry = struct { allocator: *Allocator, - pub name: []const u8, - pub value: []u8, - pub never_index: bool, + name: []const u8, + value: []u8, + never_index: bool, const Self = @This(); diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b3..f9082ee952 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -161,7 +161,7 @@ pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) const Self = @This(); const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, unbuffered_in_stream: *Stream, @@ -273,7 +273,7 @@ pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) typ pub const Error = InStreamError; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, base: *Stream, // Right now the look-ahead space is statically allocated, but a version with dynamic allocation @@ -336,7 +336,7 @@ pub const SliceInStream = struct { pub const Error = error{}; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, pos: usize, slice: []const u8, @@ -514,9 +514,9 @@ pub const SliceOutStream = struct { pub const Error = error{OutOfSpace}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, - pub pos: usize, + pos: usize, slice: []u8, pub fn init(slice: []u8) SliceOutStream { @@ -571,7 +571,7 @@ pub const NullOutStream = struct { pub const Error = error{}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, pub fn init() NullOutStream { return NullOutStream{ @@ -595,8 +595,8 @@ pub fn CountingOutStream(comptime OutStreamError: type) type { pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, - pub bytes_written: u64, + stream: Stream, + bytes_written: u64, child_stream: *Stream, pub fn init(child_stream: *Stream) Self { @@ -635,7 +635,7 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamEr pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, + stream: Stream, unbuffered_out_stream: *Stream, @@ -1084,7 +1084,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/io/seekable_stream.zig b/lib/std/io/seekable_stream.zig index 86f76d8c14..48dc31b785 100644 --- a/lib/std/io/seekable_stream.zig +++ b/lib/std/io/seekable_stream.zig @@ -39,8 +39,8 @@ pub const SliceSeekableInStream = struct { pub const Stream = InStream(Error); pub const SeekableInStream = SeekableStream(SeekError, GetSeekPosError); - pub stream: Stream, - pub seekable_stream: SeekableInStream, + stream: Stream, + seekable_stream: SeekableInStream, pos: usize, slice: []const u8, diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 79e6647144..9f32c6e54b 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -290,7 +290,7 @@ pub const Error = union(enum) { pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found '{}'"); pub const ExpectedParamType = SimpleError("Expected parameter type"); - pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member"); + pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); pub const ExtraAlignQualifier = SimpleError("Extra align qualifier"); pub const ExtraConstQualifier = SimpleError("Extra const qualifier"); @@ -757,7 +757,6 @@ pub const Node = struct { pub const ContainerField = struct { base: Node, doc_comments: ?*DocComment, - visib_token: ?TokenIndex, name_token: TokenIndex, type_expr: ?*Node, value_expr: ?*Node, @@ -780,7 +779,6 @@ pub const Node = struct { } pub fn firstToken(self: *const ContainerField) TokenIndex { - if (self.visib_token) |visib_token| return visib_token; return self.name_token; } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 151784284d..71fa376361 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -138,9 +138,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } + if (visib_token != null) { + try tree.errors.push(AstError{ + .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, + }); + return error.ParseError; + } + if (try parseContainerField(arena, it, tree)) |node| { const field = node.cast(Node.ContainerField).?; - field.visib_token = visib_token; field.doc_comments = doc_comments; try list.push(node); const comma = eatToken(it, .Comma) orelse break; @@ -149,13 +155,6 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } - // Dangling pub - if (visib_token != null) { - try tree.errors.push(AstError{ - .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, - }); - } - break; } @@ -407,7 +406,6 @@ fn parseContainerField(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No node.* = Node.ContainerField{ .base = Node{ .id = .ContainerField }, .doc_comments = null, - .visib_token = null, .name_token = name_token, .type_expr = type_expr, .value_expr = value_expr, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 78f80786c5..28bd287c8a 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1766,7 +1766,7 @@ test "zig fmt: struct declaration" { \\const S = struct { \\ const Self = @This(); \\ f1: u8, - \\ pub f3: u8, + \\ f3: u8, \\ \\ fn method(self: *Self) Self { \\ return self.*; @@ -1777,14 +1777,14 @@ test "zig fmt: struct declaration" { \\ \\const Ps = packed struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; \\ \\const Es = extern struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 1be4410f38..e5875fd04f 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -254,10 +254,6 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i try renderDocComments(tree, stream, field, indent, start_col); - if (field.visib_token) |visib_token| { - try renderToken(tree, stream, visib_token, indent, start_col, Space.Space); // pub - } - if (field.type_expr == null and field.value_expr == null) { return renderToken(tree, stream, field.name_token, indent, start_col, Space.Comma); // name, } else if (field.type_expr != null and field.value_expr == null) { @@ -2206,8 +2202,8 @@ const FindByteOutStream = struct { pub const Error = error{}; pub const Stream = std.io.OutStream(Error); - pub stream: Stream, - pub byte_found: bool, + stream: Stream, + byte_found: bool, byte: u8, pub fn init(byte: u8) Self { diff --git a/src/all_types.hpp b/src/all_types.hpp index e2e1db48d4..267c5479be 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -990,8 +990,6 @@ struct AstNodeStructField { // populated if the "align(A)" is present AstNode *align_expr; Buf doc_comments; - - VisibMod visib_mod; }; struct AstNodeStringLiteral { diff --git a/src/parser.cpp b/src/parser.cpp index 9fcf233e2d..b6742ef1f1 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -518,8 +518,8 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { // <- TestDecl ContainerMembers // / TopLevelComptime ContainerMembers // / KEYWORD_pub? TopLevelDecl ContainerMembers -// / KEYWORD_pub? ContainerField COMMA ContainerMembers -// / KEYWORD_pub? ContainerField +// / ContainerField COMMA ContainerMembers +// / ContainerField // / static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { AstNodeContainerDecl res = {}; @@ -548,10 +548,13 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { continue; } + if (visib_token != nullptr) { + ast_error(pc, peek_token(pc), "expected function or variable declaration after pub"); + } + AstNode *container_field = ast_parse_container_field(pc); if (container_field != nullptr) { assert(container_field->type == NodeTypeStructField); - container_field->data.struct_field.visib_mod = visib_mod; container_field->data.struct_field.doc_comments = doc_comment_buf; res.fields.append(container_field); if (eat_token_if(pc, TokenIdComma) != nullptr) { @@ -561,12 +564,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { } } - // We visib_token wasn't eaten, then we haven't consumed the first token in this rule yet. - // It is therefore safe to return and let the caller continue parsing. - if (visib_token == nullptr) - break; - - ast_invalid_token_error(pc, peek_token(pc)); + break; } return res; From 87b11617b55254abab267e8739ee2f0d926fbcf5 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Mon, 21 Oct 2019 14:26:15 -0600 Subject: [PATCH 30/31] Fix accept function API The sockaddr pointer and size of the accept function points to a data structure that can only be determined at runtime. The only requirement is that it must be large enough to hold 2 bytes for the address family value. Typeical usage of the socket API is for UDP/TCP IPv4 and IPv6 sockets, which use sockaddr_in and sockaddr_in6. And some sockets can actually support both simultaneously in which case the app may want to have access to the size of the returned sockaddr. Operating systems can even support custom protocols where they use custom sockaddr data structures. In this case the standard library would have no knowledge of the actual size of the sockaddr being passed into the accept function. In this case the standard library should defer to the app to pass in the size of their structure. --- lib/std/os.zig | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5..4d0dac896c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1478,10 +1478,9 @@ pub const AcceptError = error{ /// Accept a connection on a socket. `fd` must be opened in blocking mode. /// See also `accept4_async`. -pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { +pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); + const rc = system.accept4(fd, addr, addrSize, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, @@ -1506,10 +1505,9 @@ pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { /// This is the same as `accept4` except `fd` is expected to be non-blocking. /// Returns -1 if would block. -pub fn accept4_async(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { +pub fn accept4_async(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); + const rc = system.accept4(fd, addr, addrSize, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, From a5cc758036720babeb13ec8cdec68b720c6af1eb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 23:23:11 -0400 Subject: [PATCH 31/31] std.os.accept4: improve docs and integrate with evented I/O --- lib/std/os.zig | 66 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 4d0dac896c..3395ef669d 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1476,17 +1476,46 @@ pub const AcceptError = error{ BlockedByFirewall, } || UnexpectedError; -/// Accept a connection on a socket. `fd` must be opened in blocking mode. -/// See also `accept4_async`. -pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { +/// Accept a connection on a socket. +/// If the application has a global event loop enabled, EAGAIN is handled +/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +pub fn accept4( + /// This argument is a socket that has been created with `socket`, bound to a local address + /// with `bind`, and is listening for connections after a `listen`. + sockfd: i32, + /// This argument is a pointer to a sockaddr structure. This structure is filled in with the + /// address of the peer socket, as known to the communications layer. The exact format of the + /// address returned addr is determined by the socket's address family (see `socket` and the + /// respective protocol man pages). + addr: *sockaddr, + /// This argument is a value-result argument: the caller must initialize it to contain the + /// size (in bytes) of the structure pointed to by addr; on return it will contain the actual size + /// of the peer address. + /// + /// The returned address is truncated if the buffer provided is too small; in this case, `addr_size` + /// will return a value greater than was supplied to the call. + addr_size: *usize, + /// If flags is 0, then `accept4` is the same as `accept`. The following values can be bitwise + /// ORed in flags to obtain different behavior: + /// * `SOCK_NONBLOCK` - Set the `O_NONBLOCK` file status flag on the open file description (see `open`) + /// referred to by the new file descriptor. Using this flag saves extra calls to `fcntl` to achieve + /// the same result. + /// * `SOCK_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. See the + /// description of the `O_CLOEXEC` flag in `open` for reasons why this may be useful. + flags: u32, +) AcceptError!i32 { while (true) { - const rc = system.accept4(fd, addr, addrSize, flags); + const rc = system.accept4(sockfd, addr, addr_size, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, - else => |err| return unexpectedErrno(err), - EAGAIN => unreachable, // This function is for blocking only. + EAGAIN => if (std.event.Loop.instance) |loop| { + loop.waitUntilFdReadable(sockfd) catch return error.WouldBlock; + continue; + } else { + return error.WouldBlock; + }, EBADF => unreachable, // always a race condition ECONNABORTED => return error.ConnectionAborted, EFAULT => unreachable, @@ -1499,33 +1528,8 @@ pub fn accept4(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptErr EOPNOTSUPP => return error.OperationNotSupported, EPROTO => return error.ProtocolFailure, EPERM => return error.BlockedByFirewall, - } - } -} -/// This is the same as `accept4` except `fd` is expected to be non-blocking. -/// Returns -1 if would block. -pub fn accept4_async(fd: i32, addr: *sockaddr, addrSize: *usize, flags: u32) AcceptError!i32 { - while (true) { - const rc = system.accept4(fd, addr, addrSize, flags); - switch (errno(rc)) { - 0 => return @intCast(i32, rc), - EINTR => continue, else => |err| return unexpectedErrno(err), - - EAGAIN => return -1, - EBADF => unreachable, // always a race condition - ECONNABORTED => return error.ConnectionAborted, - EFAULT => unreachable, - EINVAL => unreachable, - EMFILE => return error.ProcessFdQuotaExceeded, - ENFILE => return error.SystemFdQuotaExceeded, - ENOBUFS => return error.SystemResources, - ENOMEM => return error.SystemResources, - ENOTSOCK => return error.FileDescriptorNotASocket, - EOPNOTSUPP => return error.OperationNotSupported, - EPROTO => return error.ProtocolFailure, - EPERM => return error.BlockedByFirewall, } } }