From fe0e8c87b762aba6eb5ff1fad8ad7d37404476f1 Mon Sep 17 00:00:00 2001 From: Lachlan Easton Date: Tue, 17 Dec 2019 03:01:02 +1100 Subject: [PATCH] Tokenizer: Copy optional tokens prior to being set to null #3737 (#3910) * Tokenizer: Copy optional tokens prior to being set to null #3737 * Add TODO comments, reminder to audit copying optional pattern. --- lib/std/json.zig | 1 + lib/std/zig/tokenizer.zig | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/std/json.zig b/lib/std/json.zig index c1a0b42009..030b940276 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -886,6 +886,7 @@ pub const TokenStream = struct { pub fn next(self: *TokenStream) Error!?Token { if (self.token) |token| { + // TODO: Audit this pattern once #2915 is closed const copy = token; self.token = null; return copy; diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index eda1527100..d95fb47d41 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -411,8 +411,10 @@ pub const Tokenizer = struct { pub fn next(self: *Tokenizer) Token { if (self.pending_invalid_token) |token| { + // TODO: Audit this pattern once #2915 is closed + const copy = token; self.pending_invalid_token = null; - return token; + return copy; } const start_index = self.index; var state = State.Start; @@ -1265,8 +1267,10 @@ pub const Tokenizer = struct { if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { + // TODO: Audit this pattern once #2915 is closed + const copy = token; self.pending_invalid_token = null; - return token; + return copy; } }