diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index b050944884..dc9d61aa22 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -293,85 +293,11 @@ static void cancel_token(Tokenize *t) {
 }
 
 static void end_float_token(Tokenize *t) {
-    if (t->radix == 10 || t->radix == 16) {
-        uint8_t *ptr_buf = (uint8_t*)buf_ptr(t->buf) + t->cur_tok->start_pos;
-        size_t buf_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
-        if (bigfloat_init_buf(&t->cur_tok->data.float_lit.bigfloat, ptr_buf, buf_len)) {
-            t->cur_tok->data.float_lit.overflow = true;
-        }
-        return;
-    }
-
-    BigInt int_max;
-    bigint_init_unsigned(&int_max, INT_MAX);
-
-    if (bigint_cmp(&t->specified_exponent, &int_max) != CmpLT) {
+    uint8_t *ptr_buf = (uint8_t*)buf_ptr(t->buf) + t->cur_tok->start_pos;
+    size_t buf_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
+    if (bigfloat_init_buf(&t->cur_tok->data.float_lit.bigfloat, ptr_buf, buf_len)) {
         t->cur_tok->data.float_lit.overflow = true;
-        return;
     }
-
-    if (!bigint_fits_in_bits(&t->specified_exponent, 128, true)) {
-        t->cur_tok->data.float_lit.overflow = true;
-        return;
-    }
-
-    int64_t specified_exponent = bigint_as_signed(&t->specified_exponent);
-    if (t->is_exp_negative) {
-        specified_exponent = -specified_exponent;
-    }
-    t->exponent_in_bin_or_dec = (int)(t->exponent_in_bin_or_dec + specified_exponent);
-
-    if (!bigint_fits_in_bits(&t->significand, 128, false)) {
-        t->cur_tok->data.float_lit.overflow = true;
-        return;
-    }
-
-    // A SoftFloat-3e float128 is represented internally as a standard
-    // quad-precision float with 15bit exponent and 112bit fractional.
-    union { uint64_t repr[2]; float128_t actual; } f_bits;
-
-    if (bigint_cmp_zero(&t->significand) == CmpEQ) {
-        f_bits.repr[0] = 0;
-        f_bits.repr[1] = 0;
-    } else {
-        int significand_magnitude_in_bin = 127 - bigint_clz(&t->significand, 128);
-        t->exponent_in_bin_or_dec += significand_magnitude_in_bin;
-        if (!(-16382 <= t->exponent_in_bin_or_dec && t->exponent_in_bin_or_dec <= 16383)) {
-            t->cur_tok->data.float_lit.overflow = true;
-            return;
-        }
-
-        // Shift bits of significand so they are left-justified at the 112-bit
-        // mark. We truncate excess bits and lose precision. No rounding.
-        //
-        // -16 <= shift <= 112
-        //
-        // NOTE: The loss of precision could be considered a limitation of using
-        // 128-bit floats. In stage2 we should use an arbitrary precision
-        // float/rational type to represent these and avoid this.
-        const int shift = 112 - significand_magnitude_in_bin;
-        bigint_write_twos_complement(&t->significand, (uint8_t*) f_bits.repr, 128, false);
-
-        if (shift >= 64) {
-            f_bits.repr[1] = f_bits.repr[0] << (shift - 64);
-            f_bits.repr[0] = 0;
-        } else if (shift > 0) {
-            f_bits.repr[1] = (f_bits.repr[1] << shift) | (f_bits.repr[0] >> (64 - shift));
-            f_bits.repr[0] = f_bits.repr[0] << shift;
-        } else if (shift < 0) {
-            int positive_shift = -shift;
-            assert(positive_shift <= 16);
-            f_bits.repr[0] = (f_bits.repr[0] >> positive_shift) | (f_bits.repr[1] << (64 - positive_shift));
-            f_bits.repr[1] = f_bits.repr[1] >> positive_shift;
-        }
-
-        // Lexer separates negative sign from value so this is always non-negative.
-        const uint64_t exp_mask = 0xffffull << 48;
-        f_bits.repr[1] &= ~exp_mask;
-        f_bits.repr[1] |= (uint64_t)(t->exponent_in_bin_or_dec + 16383) << 48;
-    }
-
-    bigfloat_init_128(&t->cur_tok->data.float_lit.bigfloat, f_bits.actual);
 }
 
 static void end_token(Tokenize *t) {
@@ -1265,10 +1191,16 @@ void tokenize(Buf *buf, Tokenization *out) {
             case TokenizeStateNumber:
                 {
                     if (c == '.') {
+                        if (t.radix != 16 && t.radix != 10) {
+                            invalid_char_error(&t, c);
+                        }
                         t.state = TokenizeStateNumberDot;
                         break;
                     }
                     if (is_exponent_signifier(c, t.radix)) {
+                        if (t.radix != 16 && t.radix != 10) {
+                            invalid_char_error(&t, c);
+                        }
                         t.state = TokenizeStateFloatExponentUnsigned;
                         assert(t.cur_tok->id == TokenIdIntLiteral);
                         bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint);
diff --git a/test/compare_output.zig b/test/compare_output.zig
index c2b4640239..72f4e223aa 100644
--- a/test/compare_output.zig
+++ b/test/compare_output.zig
@@ -215,13 +215,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\    _ = c.printf(c"0x103.70p-5: %.013a\n",
         \\         f64(0x103.70p-5));
         \\
-        \\    _ = c.printf(c"\n");
-        \\
-        \\    _ = c.printf(c"0b10100.00010e0: %.013a\n",
-        \\         f64(0b10100.00010e0));
-        \\    _ = c.printf(c"0o10700.00010e0: %.013a\n",
-        \\         f64(0o10700.00010e0));
-        \\
         \\    return 0;
         \\}
     ,
@@ -260,9 +253,6 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\0x103.70p+5: 0x1.0370000000000p+13
         \\0x103.70p-5: 0x1.0370000000000p+3
         \\
-        \\0b10100.00010e0: 0x1.4100000000000p+4
-        \\0o10700.00010e0: 0x1.1c00010000000p+12
-        \\
     );
 
     cases.add("order-independent declarations",
diff --git a/test/stage1/behavior/math.zig b/test/stage1/behavior/math.zig
index 5d10887d32..a2db2dcca8 100644
--- a/test/stage1/behavior/math.zig
+++ b/test/stage1/behavior/math.zig
@@ -597,8 +597,3 @@ test "vector integer addition" {
     S.doTheTest();
     comptime S.doTheTest();
 }
-
-test "binary and octal float literals" {
-    expect(0b10100.00010e0 == 0x1.4100000000000p+4);
-    expect(0o10700.00010e0 == 0x1.1c00010000000p+12);
-}