AstGen: improved handling of declarations

* Every decl provides a 16 byte source hash which can be used to detect
   if the source code for any particular decl has changed.
 * Include comptime decls, test decls, and usingnamespace decls in the
   decls list of namespaces.
   - Tests are encoded as extended functions with is_test bit set.

src/AstGen.zig

@@ -35,10 +35,10 @@ string_bytes: ArrayListUnmanaged(u8) = .{},
 arena: *Allocator,
 string_table: std.StringHashMapUnmanaged(u32) = .{},
 compile_errors: ArrayListUnmanaged(Zir.Inst.CompileErrors.Item) = .{},
-/// String table indexes, keeps track of all `@import` operands.
-imports: std.AutoArrayHashMapUnmanaged(u32, void) = .{},
 /// The topmost block of the current function.
 fn_block: ?*GenZir = null,
+/// String table indexes, keeps track of all `@import` operands.
+imports: std.AutoArrayHashMapUnmanaged(u32, void) = .{},
 
 pub fn addExtra(astgen: *AstGen, extra: anytype) Allocator.Error!u32 {
     const fields = std.meta.fields(@TypeOf(extra));
@@ -1078,6 +1078,7 @@ pub fn fnProtoExpr(
         .lib_name = 0,
         .is_var_args = is_var_args,
         .is_inferred_error = false,
+        .is_test = false,
     });
     return rvalue(gz, scope, rl, result, fn_proto.ast.proto_node);
 }
@@ -2610,6 +2611,26 @@ const WipDecls = struct {
     const bits_per_field = 4;
     const fields_per_u32 = 32 / bits_per_field;
 
+    fn next(
+        wip_decls: *WipDecls,
+        gpa: *Allocator,
+        is_pub: bool,
+        is_export: bool,
+        has_align: bool,
+        has_section: bool,
+    ) Allocator.Error!void {
+        if (wip_decls.decl_index % fields_per_u32 == 0 and wip_decls.decl_index != 0) {
+            try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
+            wip_decls.cur_bit_bag = 0;
+        }
+        wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> bits_per_field) |
+            (@as(u32, @boolToInt(is_pub)) << 28) |
+            (@as(u32, @boolToInt(is_export)) << 29) |
+            (@as(u32, @boolToInt(has_align)) << 30) |
+            (@as(u32, @boolToInt(has_section)) << 31);
+        wip_decls.decl_index += 1;
+    }
+
     fn deinit(wip_decls: *WipDecls, gpa: *Allocator) void {
         wip_decls.bit_bag.deinit(gpa);
         wip_decls.payload.deinit(gpa);
@@ -2652,16 +2673,7 @@ fn fnDecl(
         break :inst try comptimeExpr(&decl_gz, &decl_gz.base, .{ .ty = .const_slice_u8_type }, fn_proto.ast.section_expr);
     };
 
-    if (wip_decls.decl_index % WipDecls.fields_per_u32 == 0 and wip_decls.decl_index != 0) {
-        try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
-        wip_decls.cur_bit_bag = 0;
-    }
-    wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> WipDecls.bits_per_field) |
-        (@as(u32, @boolToInt(is_pub)) << 28) |
-        (@as(u32, @boolToInt(is_export)) << 29) |
-        (@as(u32, @boolToInt(align_inst != .none)) << 30) |
-        (@as(u32, @boolToInt(section_inst != .none)) << 31);
-    wip_decls.decl_index += 1;
+    try wip_decls.next(gpa, is_pub, is_export, align_inst != .none, section_inst != .none);
 
     // The AST params array does not contain anytype and ... parameters.
     // We must iterate to count how many param types to allocate.
@@ -2750,6 +2762,7 @@ fn fnDecl(
             .lib_name = lib_name,
             .is_var_args = is_var_args,
             .is_inferred_error = false,
+            .is_test = false,
         });
     } else func: {
         if (is_var_args) {
@@ -2821,6 +2834,7 @@ fn fnDecl(
             .lib_name = lib_name,
             .is_var_args = is_var_args,
             .is_inferred_error = is_inferred_error,
+            .is_test = false,
         });
     };
 
@@ -2833,7 +2847,12 @@ fn fnDecl(
     _ = try decl_gz.addBreak(.break_inline, block_inst, func_inst);
     try decl_gz.setBlockBody(block_inst);
 
-    try wip_decls.payload.ensureUnusedCapacity(gpa, 4);
+    try wip_decls.payload.ensureUnusedCapacity(gpa, 8);
+    {
+        const contents_hash = std.zig.hashSrc(tree.getNodeSource(fn_proto.ast.proto_node));
+        const casted = @bitCast([4]u32, contents_hash);
+        wip_decls.payload.appendSliceAssumeCapacity(&casted);
+    }
     wip_decls.payload.appendAssumeCapacity(fn_name_str_index);
     wip_decls.payload.appendAssumeCapacity(block_inst);
     if (align_inst != .none) {
@@ -2879,16 +2898,7 @@ fn globalVarDecl(
     const section_inst: Zir.Inst.Ref = if (var_decl.ast.section_node == 0) .none else inst: {
         break :inst try comptimeExpr(&block_scope, &block_scope.base, .{ .ty = .const_slice_u8_type }, var_decl.ast.section_node);
     };
-    if (wip_decls.decl_index % WipDecls.fields_per_u32 == 0 and wip_decls.decl_index != 0) {
-        try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
-        wip_decls.cur_bit_bag = 0;
-    }
-    wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> WipDecls.bits_per_field) |
-        (@as(u32, @boolToInt(is_pub)) << 28) |
-        (@as(u32, @boolToInt(is_export)) << 29) |
-        (@as(u32, @boolToInt(align_inst != .none)) << 30) |
-        (@as(u32, @boolToInt(section_inst != .none)) << 31);
-    wip_decls.decl_index += 1;
+    try wip_decls.next(gpa, is_pub, is_export, align_inst != .none, section_inst != .none);
 
     const is_mutable = token_tags[var_decl.ast.mut_token] == .keyword_var;
     const is_threadlocal = if (var_decl.threadlocal_token) |tok| blk: {
@@ -2950,7 +2960,12 @@ fn globalVarDecl(
     const name_token = var_decl.ast.mut_token + 1;
     const name_str_index = try gz.identAsString(name_token);
 
-    try wip_decls.payload.ensureUnusedCapacity(gpa, 4);
+    try wip_decls.payload.ensureUnusedCapacity(gpa, 8);
+    {
+        const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
+        const casted = @bitCast([4]u32, contents_hash);
+        wip_decls.payload.appendSliceAssumeCapacity(&casted);
+    }
     wip_decls.payload.appendAssumeCapacity(name_str_index);
     wip_decls.payload.appendAssumeCapacity(var_inst);
     if (align_inst != .none) {
@@ -2965,21 +2980,48 @@ fn comptimeDecl(
     astgen: *AstGen,
     gz: *GenZir,
     scope: *Scope,
+    wip_decls: *WipDecls,
     node: ast.Node.Index,
 ) InnerError!void {
+    const gpa = astgen.gpa;
     const tree = &astgen.file.tree;
     const node_datas = tree.nodes.items(.data);
-    const block_expr = node_datas[node].lhs;
-    // TODO probably we want to put these into a block and store a list of them
-    _ = try expr(gz, scope, .none, block_expr);
+    const body_node = node_datas[node].lhs;
+
+    // Up top so the ZIR instruction index marks the start range of this
+    // top-level declaration.
+    const block_inst = try gz.addBlock(.block_inline, node);
+    try wip_decls.next(gpa, false, false, false, false);
+
+    var decl_block: GenZir = .{
+        .force_comptime = true,
+        .decl_node_index = node,
+        .parent = scope,
+        .astgen = astgen,
+    };
+    defer decl_block.instructions.deinit(gpa);
+
+    _ = try expr(&decl_block, &decl_block.base, .none, body_node);
+    try decl_block.setBlockBody(block_inst);
+
+    try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
+    {
+        const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
+        const casted = @bitCast([4]u32, contents_hash);
+        wip_decls.payload.appendSliceAssumeCapacity(&casted);
+    }
+    wip_decls.payload.appendAssumeCapacity(0);
+    wip_decls.payload.appendAssumeCapacity(block_inst);
 }
 
 fn usingnamespaceDecl(
     astgen: *AstGen,
     gz: *GenZir,
     scope: *Scope,
+    wip_decls: *WipDecls,
     node: ast.Node.Index,
 ) InnerError!void {
+    const gpa = astgen.gpa;
     const tree = &astgen.file.tree;
     const node_datas = tree.nodes.items(.data);
 
@@ -2990,14 +3032,38 @@ fn usingnamespaceDecl(
         const main_token = main_tokens[node];
         break :blk (main_token > 0 and token_tags[main_token - 1] == .keyword_pub);
     };
-    // TODO probably we want to put these into a block and store a list of them
-    const namespace_inst = try expr(gz, scope, .{ .ty = .type_type }, type_expr);
+    // Up top so the ZIR instruction index marks the start range of this
+    // top-level declaration.
+    const block_inst = try gz.addBlock(.block_inline, node);
+    try wip_decls.next(gpa, is_pub, true, false, false);
+
+    var decl_block: GenZir = .{
+        .force_comptime = true,
+        .decl_node_index = node,
+        .parent = scope,
+        .astgen = astgen,
+    };
+    defer decl_block.instructions.deinit(gpa);
+
+    const namespace_inst = try typeExpr(&decl_block, &decl_block.base, type_expr);
+    _ = try decl_block.addBreak(.break_inline, block_inst, namespace_inst);
+    try decl_block.setBlockBody(block_inst);
+
+    try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
+    {
+        const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
+        const casted = @bitCast([4]u32, contents_hash);
+        wip_decls.payload.appendSliceAssumeCapacity(&casted);
+    }
+    wip_decls.payload.appendAssumeCapacity(0);
+    wip_decls.payload.appendAssumeCapacity(block_inst);
 }
 
 fn testDecl(
     astgen: *AstGen,
     gz: *GenZir,
     scope: *Scope,
+    wip_decls: *WipDecls,
     node: ast.Node.Index,
 ) InnerError!void {
     const gpa = astgen.gpa;
@@ -3005,10 +3071,16 @@ fn testDecl(
     const node_datas = tree.nodes.items(.data);
     const body_node = node_datas[node].rhs;
 
+    // Up top so the ZIR instruction index marks the start range of this
+    // top-level declaration.
+    const block_inst = try gz.addBlock(.block_inline, node);
+
+    try wip_decls.next(gpa, false, false, false, false);
+
     var decl_block: GenZir = .{
         .force_comptime = true,
         .decl_node_index = node,
-        .parent = &gz.base,
+        .parent = scope,
         .astgen = astgen,
     };
     defer decl_block.instructions.deinit(gpa);
@@ -3053,15 +3125,20 @@ fn testDecl(
         .lib_name = 0,
         .is_var_args = false,
         .is_inferred_error = true,
+        .is_test = true,
     });
 
-    const block_inst = try gz.addBlock(.block_inline, node);
     _ = try decl_block.addBreak(.break_inline, block_inst, func_inst);
     try decl_block.setBlockBody(block_inst);
 
-    // TODO collect these into a test decl list
-    _ = test_name;
-    _ = block_inst;
+    try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
+    {
+        const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
+        const casted = @bitCast([4]u32, contents_hash);
+        wip_decls.payload.appendSliceAssumeCapacity(&casted);
+    }
+    wip_decls.payload.appendAssumeCapacity(test_name);
+    wip_decls.payload.appendAssumeCapacity(block_inst);
 }
 
 fn structDeclInner(
@@ -3179,15 +3256,15 @@ fn structDeclInner(
             },
 
             .@"comptime" => {
-                try astgen.comptimeDecl(gz, scope, member_node);
+                try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             .@"usingnamespace" => {
-                try astgen.usingnamespaceDecl(gz, scope, member_node);
+                try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             .test_decl => {
-                try astgen.testDecl(gz, scope, member_node);
+                try astgen.testDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             else => unreachable,
@@ -3382,15 +3459,15 @@ fn unionDeclInner(
             },
 
             .@"comptime" => {
-                try astgen.comptimeDecl(gz, scope, member_node);
+                try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             .@"usingnamespace" => {
-                try astgen.usingnamespaceDecl(gz, scope, member_node);
+                try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             .test_decl => {
-                try astgen.testDecl(gz, scope, member_node);
+                try astgen.testDecl(gz, scope, &wip_decls, member_node);
                 continue;
             },
             else => unreachable,
@@ -3731,15 +3808,15 @@ fn containerDecl(
                     },
 
                     .@"comptime" => {
-                        try astgen.comptimeDecl(gz, scope, member_node);
+                        try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     .@"usingnamespace" => {
-                        try astgen.usingnamespaceDecl(gz, scope, member_node);
+                        try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     .test_decl => {
-                        try astgen.testDecl(gz, scope, member_node);
+                        try astgen.testDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     else => unreachable,
@@ -3896,15 +3973,15 @@ fn containerDecl(
                     },
 
                     .@"comptime" => {
-                        try astgen.comptimeDecl(gz, scope, member_node);
+                        try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     .@"usingnamespace" => {
-                        try astgen.usingnamespaceDecl(gz, scope, member_node);
+                        try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     .test_decl => {
-                        try astgen.testDecl(gz, scope, member_node);
+                        try astgen.testDecl(gz, scope, &wip_decls, member_node);
                         continue;
                     },
                     else => unreachable,

src/Module.zig

@@ -1311,6 +1311,7 @@ pub const Scope = struct {
             lib_name: u32,
             is_var_args: bool,
             is_inferred_error: bool,
+            is_test: bool,
         }) !Zir.Inst.Ref {
             assert(args.src_node != 0);
             assert(args.ret_ty != .none);
@@ -1320,7 +1321,7 @@ pub const Scope = struct {
             try gz.instructions.ensureUnusedCapacity(gpa, 1);
             try astgen.instructions.ensureUnusedCapacity(gpa, 1);
 
-            if (args.cc != .none or args.lib_name != 0 or args.is_var_args) {
+            if (args.cc != .none or args.lib_name != 0 or args.is_var_args or args.is_test) {
                 try astgen.extra.ensureUnusedCapacity(
                     gpa,
                     @typeInfo(Zir.Inst.ExtendedFunc).Struct.fields.len +
@@ -1353,6 +1354,7 @@ pub const Scope = struct {
                             .is_inferred_error = args.is_inferred_error,
                             .has_lib_name = args.lib_name != 0,
                             .has_cc = args.cc != .none,
+                            .is_test = args.is_test,
                         }),
                         .operand = payload_index,
                     } },

src/Zir.zig

@@ -2201,7 +2201,8 @@ pub const Inst = struct {
             is_inferred_error: bool,
             has_lib_name: bool,
             has_cc: bool,
-            _: u12 = undefined,
+            is_test: bool,
+            _: u11 = undefined,
         };
     };
 
@@ -2375,7 +2376,10 @@ pub const Inst = struct {
     ///      0b0X00: whether corresponding decl has an align expression
     ///      0bX000: whether corresponding decl has a linksection expression
     /// 1. decl: { // for every decls_len
+    ///        src_hash: [4]u32, // hash of source bytes
     ///        name: u32, // null terminated string index
+    ///        - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
+    ///        - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
     ///        value: Index,
     ///        align: Ref, // if corresponding bit is set
     ///        link_section: Ref, // if corresponding bit is set
@@ -2405,7 +2409,10 @@ pub const Inst = struct {
     ///      0b0X00: whether corresponding decl has an align expression
     ///      0bX000: whether corresponding decl has a linksection expression
     /// 1. decl: { // for every decls_len
+    ///        src_hash: [4]u32, // hash of source bytes
     ///        name: u32, // null terminated string index
+    ///        - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
+    ///        - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
     ///        value: Index,
     ///        align: Ref, // if corresponding bit is set
     ///        link_section: Ref, // if corresponding bit is set
@@ -2433,7 +2440,10 @@ pub const Inst = struct {
     ///      0b0X00: whether corresponding decl has an align expression
     ///      0bX000: whether corresponding decl has a linksection expression
     /// 1. decl: { // for every decls_len
+    ///        src_hash: [4]u32, // hash of source bytes
     ///        name: u32, // null terminated string index
+    ///        - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
+    ///        - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
     ///        value: Index,
     ///        align: Ref, // if corresponding bit is set
     ///        link_section: Ref, // if corresponding bit is set
@@ -2471,8 +2481,12 @@ pub const Inst = struct {
     ///      0b0X00: whether corresponding decl has an align expression
     ///      0bX000: whether corresponding decl has a linksection expression
     /// 1. decl: { // for every decls_len
+    ///        src_hash: [4]u32, // hash of source bytes
     ///        name: u32, // null terminated string index
+    ///        - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
+    ///        - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
     ///        value: Index,
+    ///        - one of: block_inline, block_inline_var
     ///        align: Ref, // if corresponding bit is set
     ///        link_section: Ref, // if corresponding bit is set
     ///    }
@@ -3553,7 +3567,10 @@ const Writer = struct {
             const has_section = @truncate(u1, cur_bit_bag) != 0;
             cur_bit_bag >>= 1;
 
-            const decl_name = self.code.nullTerminatedString(self.code.extra[extra_index]);
+            const hash_u32s = self.code.extra[extra_index..][0..4];
+            extra_index += 4;
+            const decl_name_index = self.code.extra[extra_index];
+            const decl_name = self.code.nullTerminatedString(decl_name_index);
             extra_index += 1;
             const decl_index = self.code.extra[extra_index];
             extra_index += 1;
@@ -3568,24 +3585,33 @@ const Writer = struct {
                 break :inst inst;
             };
 
-            const tag = self.code.instructions.items(.tag)[decl_index];
             const pub_str = if (is_pub) "pub " else "";
-            const export_str = if (is_exported) "export " else "";
+            const hash_bytes = @bitCast([16]u8, hash_u32s.*);
             try stream.writeByteNTimes(' ', self.indent);
-            try stream.print("{s}{s}{}", .{
-                pub_str, export_str, std.zig.fmtId(decl_name),
+            if (decl_name_index == 0) {
+                const name = if (is_exported) "usingnamespace" else "comptime";
+                try stream.writeAll(pub_str);
+                try stream.writeAll(name);
+            } else {
+                const export_str = if (is_exported) "export " else "";
+                try stream.print("{s}{s}{}", .{
+                    pub_str, export_str, std.zig.fmtId(decl_name),
+                });
+                if (align_inst != .none) {
+                    try stream.writeAll(" align(");
+                    try self.writeInstRef(stream, align_inst);
+                    try stream.writeAll(")");
+                }
+                if (section_inst != .none) {
+                    try stream.writeAll(" linksection(");
+                    try self.writeInstRef(stream, section_inst);
+                    try stream.writeAll(")");
+                }
+            }
+            const tag = self.code.instructions.items(.tag)[decl_index];
+            try stream.print(" hash({}): %{d} = {s}(", .{
+                std.fmt.fmtSliceHexLower(&hash_bytes), decl_index, @tagName(tag),
             });
-            if (align_inst != .none) {
-                try stream.writeAll(" align(");
-                try self.writeInstRef(stream, align_inst);
-                try stream.writeAll(")");
-            }
-            if (section_inst != .none) {
-                try stream.writeAll(" linksection(");
-                try self.writeInstRef(stream, section_inst);
-                try stream.writeAll(")");
-            }
-            try stream.print(": %{d} = {s}(", .{ decl_index, @tagName(tag) });
 
             const decl_block_inst_data = self.code.instructions.items(.data)[decl_index].pl_node;
             const sub_decl_node_off = decl_block_inst_data.src_node;
@@ -3939,6 +3965,7 @@ const Writer = struct {
             extra_index += 1;
             try stream.print("lib_name=\"{}\", ", .{std.zig.fmtEscapes(lib_name)});
         }
+        try self.writeFlag(stream, "test, ", small.is_test);
         const cc: Inst.Ref = if (!small.has_cc) .none else blk: {
             const cc = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]);
             extra_index += 1;