From 2d4b95900e7d7273dbd3ce6b7b9a33d2a19779da Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 4 Sep 2018 18:15:27 -0400 Subject: [PATCH 01/20] stage1: import blake2b implementation from master branch of blake2 reference implementation 320c325437539ae91091ce62efec1913cd8093c2 --- CMakeLists.txt | 1 + src/blake2.h | 196 ++++++++++++++++++ src/blake2b.cpp | 539 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 736 insertions(+) create mode 100644 src/blake2.h create mode 100644 src/blake2b.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c3ee37ea89..fbf5b03eac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" diff --git a/src/blake2.h b/src/blake2.h new file mode 100644 index 0000000000..6420c5367a --- /dev/null +++ b/src/blake2.h @@ -0,0 +1,196 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum { + BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/src/blake2b.cpp b/src/blake2b.cpp new file mode 100644 index 0000000000..600f951b9b --- /dev/null +++ b/src/blake2b.cpp @@ -0,0 +1,539 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return ( uint16_t )((( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8)); +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif + From 97c9f61db47ddba43a0db562e13773514875fa6a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 00:34:46 -0400 Subject: [PATCH 02/20] start creating a hash of input parameters See #1416 --- src/all_types.hpp | 30 ++++---- src/codegen.cpp | 175 ++++++++++++++++++++++++++++++++++++++++++---- src/codegen.hpp | 1 - src/link.cpp | 2 - src/main.cpp | 1 - 5 files changed, 179 insertions(+), 30 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index dbaa3b5467..0b0de58791 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1543,23 +1543,17 @@ struct LinkLib { bool provided_explicitly; }; +// When adding fields, check if they should be added to the hash computation in build_with_cache struct CodeGen { + //////////////////////////// Runtime State LLVMModuleRef module; ZigList errors; LLVMBuilderRef builder; ZigLLVMDIBuilder *dbuilder; ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; - - ZigList link_libs_list; LinkLib *libc_link_lib; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - // reminder: hash tables must be initialized before use HashMap import_table; HashMap builtin_fn_table; @@ -1575,7 +1569,6 @@ struct CodeGen { HashMap string_literals_table; HashMap type_info_cache; - ZigList import_queue; size_t import_queue_index; ZigList resolve_queue; @@ -1620,7 +1613,20 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + + + //////////////////////////// Unsorted + ZigTarget zig_target; LLVMTargetDataRef target_data_ref; unsigned pointer_size_bytes; @@ -1647,7 +1653,6 @@ struct CodeGen { Buf *ar_path; ZigWindowsSDK *win_sdk; Buf triple_str; - BuildMode build_mode; bool is_test_build; bool have_err_ret_tracing; uint32_t target_os_index; @@ -1657,13 +1662,13 @@ struct CodeGen { LLVMTargetMachineRef target_machine; ZigLLVMDIFile *dummy_di_file; bool is_native_target; - PackageTableEntry *root_package; + PackageTableEntry *root_package; // participates in cache hash PackageTableEntry *std_package; PackageTableEntry *panic_package; PackageTableEntry *test_runner_package; PackageTableEntry *compile_var_package; ImportTableEntry *compile_var_import; - Buf *root_out_name; + Buf *root_out_name; // participates in cache hash bool windows_subsystem_windows; bool windows_subsystem_console; Buf *mmacosx_version_min; @@ -1676,7 +1681,6 @@ struct CodeGen { size_t fn_defs_index; ZigList global_vars; - OutType out_type; ZigFn *cur_fn; ZigFn *main_fn; ZigFn *panic_fn; diff --git a/src/codegen.cpp b/src/codegen.cpp index 0300ccca99..e258f8d609 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,6 +19,7 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" +#include "blake2.h" #include #include @@ -183,10 +184,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out return g; } -void codegen_destroy(CodeGen *codegen) { - LLVMDisposeTargetMachine(codegen->target_machine); -} - void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } @@ -7112,23 +7109,30 @@ static void create_test_compile_var_and_add_test_runner(CodeGen *g) { g->test_runner_import = add_special_code(g, g->test_runner_package, "test_runner.zig"); } -static void gen_root_source(CodeGen *g) { +static Buf *get_resolved_root_src_path(CodeGen *g) { + // TODO memoize if (buf_len(&g->root_package->root_src_path) == 0) - return; + return nullptr; - codegen_add_time_event(g, "Semantic Analysis"); - - Buf *rel_full_path = buf_alloc(); - os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, rel_full_path); + Buf rel_full_path = BUF_INIT; + os_path_join(&g->root_package->root_src_dir, &g->root_package->root_src_path, &rel_full_path); Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {rel_full_path}; + Buf *resolve_paths[] = {&rel_full_path}; *resolved_path = os_path_resolve(resolve_paths, 1); + return resolved_path; +} + +static void gen_root_source(CodeGen *g) { + Buf *resolved_path = get_resolved_root_src_path(g); + if (resolved_path == nullptr) + return; + Buf *source_code = buf_alloc(); int err; - if ((err = os_fetch_file_path(rel_full_path, source_code, true))) { - fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(rel_full_path), err_str(err)); + if ((err = os_fetch_file_path(resolved_path, source_code, true))) { + fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); } @@ -7671,10 +7675,155 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_str(blake2b_state *blake, const char *ptr) { + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(blake, ptr, strlen(ptr) + 1); +} + +static void add_cache_int(blake2b_state *blake, int x) { + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(blake, buf, sizeof(int) + 1); +} + +static void add_cache_buf(blake2b_state *blake, Buf *buf) { + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); +} + +static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { + if (buf == nullptr) { + add_cache_str(blake, ""); + add_cache_str(blake, ""); + } else { + add_cache_buf(blake, buf); + } +} + +static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + add_cache_buf(blake, lib->name); + } + } + add_cache_str(blake, ""); +} + +static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + add_cache_buf(blake, buf); + } + add_cache_str(blake, ""); +} + +//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// assert(file_name != nullptr); +// g->cache_files.append(resolved_path); +//} +// +//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { +// if (resolved_path == nullptr) { +// add_cache_str(blake, ""); +// add_cache_str(blake, ""); +// } else { +// add_cache_file(g, blake, resolved_path); +// } +//} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); + //if (i < source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + // out_index += 1; + + // if (i + 1 == source.len) { + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; + // out_index += 1; + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + // } else { + // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + // out_index += 1; + + // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; + // out_index += 1; + // } + + // dest.ptr[out_index] = encoder.pad_char; + // out_index += 1; + //} +} + +// Called before init() +static bool build_with_cache(CodeGen *g) { + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + // TODO zig exe & dynamic libraries + + add_cache_buf(&blake, g->root_out_name); + add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file + add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); + add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); + add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); + add_cache_int(&blake, g->emit_file_type); + add_cache_int(&blake, g->build_mode); + add_cache_int(&blake, g->out_type); + // TODO the rest of the struct CodeGen fields + + uint8_t bin_digest[48]; + rc = blake2b_final(&blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); + // TODO next look for a manifest file that has all the files from the input parameters + // use that to construct the real hash, which looks up the output directory and another manifest file + + return false; +} + void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); + if (build_with_cache(g)) + return; init(g); + codegen_add_time_event(g, "Semantic Analysis"); + gen_global_asm(g); gen_root_source(g); do_code_gen(g); diff --git a/src/codegen.hpp b/src/codegen.hpp index 6297c4611b..55b38a0050 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -16,7 +16,6 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, Buf *zig_lib_dir); -void codegen_destroy(CodeGen *codegen); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index f44e8b105e..78ad204fe9 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -69,8 +69,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) os_path_join(&parent_gen->cache_dir, o_out_name, output_path); codegen_link(child_gen, buf_ptr(output_path)); - codegen_destroy(child_gen); - return output_path; } diff --git a/src/main.cpp b/src/main.cpp index 4394a1d9ae..eed31438d9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -461,7 +461,6 @@ int main(int argc, char **argv) { g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); codegen_build(g); codegen_link(g, buf_ptr(path_to_build_exe)); - codegen_destroy(g); Termination term; os_spawn_process(buf_ptr(path_to_build_exe), args, &term); From b4d5d4d1748d25efa6197914a36e276e93509f57 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 5 Sep 2018 23:39:14 -0400 Subject: [PATCH 03/20] assume evenly divided base64 --- src/codegen.cpp | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index e258f8d609..f4823f1048 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7760,27 +7760,6 @@ static void base64_encode(Slice dest, Slice source) { // Assert that we never need pad characters. assert(i == source.len); - //if (i < source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - // out_index += 1; - - // if (i + 1 == source.len) { - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] & 0x3) << 4]; - // out_index += 1; - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - // } else { - // dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - // out_index += 1; - - // dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i + 1] & 0xf) << 2]; - // out_index += 1; - // } - - // dest.ptr[out_index] = encoder.pad_char; - // out_index += 1; - //} } // Called before init() From 173fc842c4eb1da6ca07a4ab6026c4c62bc8c09b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 18:07:11 -0400 Subject: [PATCH 04/20] basic compiler id hash working --- CMakeLists.txt | 1 + src/buffer.hpp | 4 + src/cache_hash.cpp | 407 +++++++++++++++++++++++++++++++++++++++++++++ src/cache_hash.hpp | 56 +++++++ src/codegen.cpp | 152 ++++------------- src/error.cpp | 2 + src/error.hpp | 2 + src/main.cpp | 61 +++++++ src/os.cpp | 314 ++++++++++++++++++++++++++-------- src/os.hpp | 158 ++++++++++-------- src/util.cpp | 39 +++++ src/util.hpp | 12 ++ 12 files changed, 951 insertions(+), 257 deletions(-) create mode 100644 src/cache_hash.cpp create mode 100644 src/cache_hash.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fbf5b03eac..615128159f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -409,6 +409,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/bigint.cpp" "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" diff --git a/src/buffer.hpp b/src/buffer.hpp index 501e44b5ac..8155df87a1 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -78,6 +78,10 @@ static inline Buf *buf_create_from_mem(const char *ptr, size_t len) { return buf; } +static inline Buf *buf_create_from_slice(Slice slice) { + return buf_create_from_mem((const char *)slice.ptr, slice.len); +} + static inline Buf *buf_create_from_str(const char *str) { return buf_create_from_mem(str, strlen(str)); } diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp new file mode 100644 index 0000000000..5447a5f3f2 --- /dev/null +++ b/src/cache_hash.cpp @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#include "cache_hash.hpp" +#include "buffer.hpp" +#include "os.hpp" + +#include + +void cache_init(CacheHash *ch, Buf *manifest_dir) { + int rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + ch->files = {}; + ch->manifest_dir = manifest_dir; + ch->manifest_file_path = nullptr; + ch->manifest_dirty = false; +} + +void cache_str(CacheHash *ch, const char *ptr) { + assert(ch->manifest_file_path == nullptr); + assert(ptr != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, ptr, strlen(ptr) + 1); +} + +void cache_int(CacheHash *ch, int x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(int) + 1]; + memcpy(buf, &x, sizeof(int)); + buf[sizeof(int)] = 0; + blake2b_update(&ch->blake, buf, sizeof(int) + 1); +} + +void cache_buf(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + assert(buf != nullptr); + // + 1 to include the null byte + blake2b_update(&ch->blake, buf_ptr(buf), buf_len(buf) + 1); +} + +void cache_buf_opt(CacheHash *ch, Buf *buf) { + assert(ch->manifest_file_path == nullptr); + if (buf == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_buf(ch, buf); + } +} + +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + LinkLib *lib = ptr[i]; + if (lib->provided_explicitly) { + cache_buf(ch, lib->name); + } + } + cache_str(ch, ""); +} + +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_buf(ch, buf); + } + cache_str(ch, ""); +} + +void cache_file(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + assert(file_path != nullptr); + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&file_path, 1); + CacheHashFile *chf = ch->files.add_one(); + chf->path = resolved_path; + cache_buf(ch, resolved_path); +} + +void cache_file_opt(CacheHash *ch, Buf *file_path) { + assert(ch->manifest_file_path == nullptr); + if (file_path == nullptr) { + cache_str(ch, ""); + cache_str(ch, ""); + } else { + cache_file(ch, file_path); + } +} + +// Ported from std/base64.zig +static uint8_t base64_fs_alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; +static void base64_encode(Slice dest, Slice source) { + size_t dest_len = ((source.len + 2) / 3) * 4; + assert(dest.len == dest_len); + + size_t i = 0; + size_t out_index = 0; + for (; i + 2 < source.len; i += 3) { + dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; + out_index += 1; + + dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; + out_index += 1; + } + + // Assert that we never need pad characters. + assert(i == source.len); +} + +// Ported from std/base64.zig +static Error base64_decode(Slice dest, Slice source) { + assert(source.len % 4 == 0); + assert(dest.len == (source.len / 4) * 3); + + // In Zig this is comptime computed. In C++ it's not worth it to do that. + uint8_t char_to_index[256]; + bool char_in_alphabet[256] = {0}; + for (size_t i = 0; i < 64; i += 1) { + uint8_t c = base64_fs_alphabet[i]; + assert(!char_in_alphabet[c]); + char_in_alphabet[c] = true; + char_to_index[c] = i; + } + + size_t src_cursor = 0; + size_t dest_cursor = 0; + + for (;src_cursor < source.len; src_cursor += 4) { + if (!char_in_alphabet[source.ptr[src_cursor + 0]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 1]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 2]]) return ErrorInvalidFormat; + if (!char_in_alphabet[source.ptr[src_cursor + 3]]) return ErrorInvalidFormat; + dest.ptr[dest_cursor + 0] = (char_to_index[source.ptr[src_cursor + 0]] << 2) | (char_to_index[source.ptr[src_cursor + 1]] >> 4); + dest.ptr[dest_cursor + 1] = (char_to_index[source.ptr[src_cursor + 1]] << 4) | (char_to_index[source.ptr[src_cursor + 2]] >> 2); + dest.ptr[dest_cursor + 2] = (char_to_index[source.ptr[src_cursor + 2]] << 6) | (char_to_index[source.ptr[src_cursor + 3]]); + dest_cursor += 3; + } + + assert(src_cursor == source.len); + assert(dest_cursor == dest.len); + return ErrorNone; +} + +static Error hash_file(uint8_t *digest, OsFile handle) { + Error err; + + blake2b_state blake; + int rc = blake2b_init(&blake, 48); + assert(rc == 0); + + for (;;) { + uint8_t buf[4096]; + size_t amt = 4096; + if ((err = os_file_read(handle, buf, &amt))) + return err; + if (amt == 0) { + rc = blake2b_final(&blake, digest, 48); + assert(rc == 0); + return ErrorNone; + } + blake2b_update(&blake, buf, amt); + } +} + +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { + Error err; + + assert(chf->path != nullptr); + + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) + return err; + + if ((err = os_file_mtime(this_file, &chf->mtime))) { + os_file_close(this_file); + return err; + } + + if ((err = hash_file(chf->bin_digest, this_file))) { + os_file_close(this_file); + return err; + } + os_file_close(this_file); + + blake2b_update(&ch->blake, chf->bin_digest, 48); + + return ErrorNone; +} + +Error cache_hit(CacheHash *ch, Buf *out_digest) { + Error err; + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + + Buf b64_digest = BUF_INIT; + buf_resize(&b64_digest, 64); + base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); + + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + + ch->manifest_file_path = buf_alloc(); + os_path_join(ch->manifest_dir, &b64_digest, ch->manifest_file_path); + + buf_append_str(ch->manifest_file_path, ".txt"); + + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) + return err; + + Buf line_buf = BUF_INIT; + buf_resize(&line_buf, 512); + if ((err = os_file_read_all(ch->manifest_file, &line_buf))) { + os_file_close(ch->manifest_file); + return err; + } + + size_t input_file_count = ch->files.length; + bool any_file_changed = false; + size_t file_i = 0; + SplitIterator line_it = memSplit(buf_to_slice(&line_buf), str("\n")); + for (;; file_i += 1) { + Optional> opt_line = SplitIterator_next(&line_it); + if (!opt_line.is_some) + break; + + CacheHashFile *chf; + if (file_i < input_file_count) { + chf = &ch->files.at(file_i); + } else if (any_file_changed) { + // cache miss. + // keep the the manifest file open with the rw lock + // reset the hash + rc = blake2b_init(&ch->blake, 48); + assert(rc == 0); + blake2b_update(&ch->blake, bin_digest, 48); + ch->files.resize(input_file_count); + // bring the hash up to the input file hashes + for (file_i = 0; file_i < input_file_count; file_i += 1) { + blake2b_update(&ch->blake, ch->files.at(file_i).bin_digest, 48); + } + // caller can notice that out_digest is unmodified. + return ErrorNone; + } else { + chf = ch->files.add_one(); + chf->path = nullptr; + } + + SplitIterator it = memSplit(opt_line.value, str(" ")); + + Optional> opt_mtime_sec = SplitIterator_next(&it); + if (!opt_mtime_sec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.sec = strtoull((const char *)opt_mtime_sec.value.ptr, nullptr, 10); + + Optional> opt_mtime_nsec = SplitIterator_next(&it); + if (!opt_mtime_nsec.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->mtime.nsec = strtoull((const char *)opt_mtime_nsec.value.ptr, nullptr, 10); + + Optional> opt_digest = SplitIterator_next(&it); + if (!opt_digest.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + if ((err = base64_decode({chf->bin_digest, 48}, opt_digest.value))) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + + Optional> opt_file_path = SplitIterator_next(&it); + if (!opt_file_path.is_some) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + Buf *this_path = buf_create_from_slice(opt_file_path.value); + if (chf->path != nullptr && !buf_eql_buf(this_path, chf->path)) { + os_file_close(ch->manifest_file); + return ErrorInvalidFormat; + } + chf->path = this_path; + + // if the mtime matches we can trust the digest + OsFile this_file; + if ((err = os_file_open_r(chf->path, &this_file))) { + os_file_close(ch->manifest_file); + return err; + } + OsTimeStamp actual_mtime; + if ((err = os_file_mtime(this_file, &actual_mtime))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + if (chf->mtime.sec == actual_mtime.sec && chf->mtime.nsec == actual_mtime.nsec) { + os_file_close(this_file); + } else { + // we have to recompute the digest. + // later we'll rewrite the manifest with the new mtime/digest values + ch->manifest_dirty = true; + chf->mtime = actual_mtime; + + uint8_t actual_digest[48]; + if ((err = hash_file(actual_digest, this_file))) { + os_file_close(this_file); + os_file_close(ch->manifest_file); + return err; + } + os_file_close(this_file); + if (memcmp(chf->bin_digest, actual_digest, 48) != 0) { + memcpy(chf->bin_digest, actual_digest, 48); + // keep going until we have the input file digests + any_file_changed = true; + } + } + if (!any_file_changed) { + blake2b_update(&ch->blake, chf->bin_digest, 48); + } + } + if (file_i < input_file_count) { + // manifest file is empty or missing entries, so this is a cache miss + ch->manifest_dirty = true; + for (; file_i < input_file_count; file_i += 1) { + CacheHashFile *chf = &ch->files.at(file_i); + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + } + return ErrorNone; + } + // Cache Hit + return cache_final(ch, out_digest); +} + +Error cache_add_file(CacheHash *ch, Buf *path) { + Error err; + + assert(ch->manifest_file_path != nullptr); + CacheHashFile *chf = ch->files.add_one(); + chf->path = path; + if ((err = populate_file_hash(ch, chf))) { + os_file_close(ch->manifest_file); + return err; + } + + return ErrorNone; +} + +static Error write_manifest_file(CacheHash *ch) { + Error err; + Buf contents = BUF_INIT; + buf_resize(&contents, 0); + uint8_t encoded_digest[65]; + encoded_digest[64] = 0; + for (size_t i = 0; i < ch->files.length; i += 1) { + CacheHashFile *chf = &ch->files.at(i); + base64_encode({encoded_digest, 64}, {chf->bin_digest, 48}); + buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", + chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); + } + fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); + if ((err = os_file_overwrite(ch->manifest_file, &contents))) + return err; + + return ErrorNone; +} + +Error cache_final(CacheHash *ch, Buf *out_digest) { + Error err; + + assert(ch->manifest_file_path != nullptr); + + if (ch->manifest_dirty) { + if ((err = write_manifest_file(ch))) { + fprintf(stderr, "Warning: Unable to write cache file '%s': %s\n", + buf_ptr(ch->manifest_file_path), err_str(err)); + } + } + os_file_close(ch->manifest_file); + + uint8_t bin_digest[48]; + int rc = blake2b_final(&ch->blake, bin_digest, 48); + assert(rc == 0); + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + + return ErrorNone; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp new file mode 100644 index 0000000000..77b22a1e41 --- /dev/null +++ b/src/cache_hash.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_CACHE_HASH_HPP +#define ZIG_CACHE_HASH_HPP + +#include "all_types.hpp" +#include "blake2.h" +#include "os.hpp" + +struct CacheHashFile { + Buf *path; + OsTimeStamp mtime; + uint8_t bin_digest[48]; +}; + +struct CacheHash { + blake2b_state blake; + ZigList files; + Buf *manifest_dir; + Buf *manifest_file_path; + OsFile manifest_file; + bool manifest_dirty; +}; + +// Always call this first to set up. +void cache_init(CacheHash *ch, Buf *manifest_dir); + +// Next, use the hash population functions to add the initial parameters. +void cache_str(CacheHash *ch, const char *ptr); +void cache_int(CacheHash *ch, int x); +void cache_buf(CacheHash *ch, Buf *buf); +void cache_buf_opt(CacheHash *ch, Buf *buf); +void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); +void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_file(CacheHash *ch, Buf *path); +void cache_file_opt(CacheHash *ch, Buf *path); + +// Then call cache_hit when you're ready to see if you can skip the next step. +// out_b64_digest will be left unchanged if it was a cache miss +Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); + +// If you got a cache hit, the flow is done. No more functions to call. +// Next call this function for every file that is depended on. +Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); + +// If you did not get a cache hit, use the hash population functions again +// and do all the actual work. When done use cache_final to save the cache +// for next time. +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); + +#endif diff --git a/src/codegen.cpp b/src/codegen.cpp index f4823f1048..6c5648f626 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -19,7 +19,6 @@ #include "target.hpp" #include "util.hpp" #include "zig_llvm.h" -#include "blake2.h" #include #include @@ -7675,130 +7674,45 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } -static void add_cache_str(blake2b_state *blake, const char *ptr) { - assert(ptr != nullptr); - // + 1 to include the null byte - blake2b_update(blake, ptr, strlen(ptr) + 1); -} -static void add_cache_int(blake2b_state *blake, int x) { - // + 1 to include the null byte - uint8_t buf[sizeof(int) + 1]; - memcpy(buf, &x, sizeof(int)); - buf[sizeof(int)] = 0; - blake2b_update(blake, buf, sizeof(int) + 1); -} - -static void add_cache_buf(blake2b_state *blake, Buf *buf) { - assert(buf != nullptr); - // + 1 to include the null byte - blake2b_update(blake, buf_ptr(buf), buf_len(buf) + 1); -} - -static void add_cache_buf_opt(blake2b_state *blake, Buf *buf) { - if (buf == nullptr) { - add_cache_str(blake, ""); - add_cache_str(blake, ""); - } else { - add_cache_buf(blake, buf); - } -} - -static void add_cache_list_of_link_lib(blake2b_state *blake, LinkLib **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - LinkLib *lib = ptr[i]; - if (lib->provided_explicitly) { - add_cache_buf(blake, lib->name); - } - } - add_cache_str(blake, ""); -} - -static void add_cache_list_of_buf(blake2b_state *blake, Buf **ptr, size_t len) { - for (size_t i = 0; i < len; i += 1) { - Buf *buf = ptr[i]; - add_cache_buf(blake, buf); - } - add_cache_str(blake, ""); -} - -//static void add_cache_file(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// assert(file_name != nullptr); -// g->cache_files.append(resolved_path); -//} +//// Called before init() +//static bool build_with_cache(CodeGen *g) { +// // TODO zig exe & dynamic libraries +// // should be in main.cpp I think. only needs to happen +// // once on startup. // -//static void add_cache_file_opt(CodeGen *g, blake2b_state *blake, Buf *resolved_path) { -// if (resolved_path == nullptr) { -// add_cache_str(blake, ""); -// add_cache_str(blake, ""); -// } else { -// add_cache_file(g, blake, resolved_path); -// } +// CacheHash comp; +// cache_init(&comp); +// +// add_cache_buf(&blake, g->root_out_name); +// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file +// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); +// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); +// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); +// add_cache_int(&blake, g->emit_file_type); +// add_cache_int(&blake, g->build_mode); +// add_cache_int(&blake, g->out_type); +// // TODO the rest of the struct CodeGen fields +// +// uint8_t bin_digest[48]; +// rc = blake2b_final(&blake, bin_digest, 48); +// assert(rc == 0); +// +// Buf b64_digest = BUF_INIT; +// buf_resize(&b64_digest, 64); +// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); +// +// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); +// // TODO next look for a manifest file that has all the files from the input parameters +// // use that to construct the real hash, which looks up the output directory and another manifest file +// +// return false; //} -// Ported from std/base64.zig -static uint8_t base64_fs_alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"; -static void base64_encode(Slice dest, Slice source) { - size_t dest_len = ((source.len + 2) / 3) * 4; - assert(dest.len == dest_len); - - size_t i = 0; - size_t out_index = 0; - for (; i + 2 < source.len; i += 3) { - dest.ptr[out_index] = base64_fs_alphabet[(source.ptr[i] >> 2) & 0x3f]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i] & 0x3) << 4) | ((source.ptr[i + 1] & 0xf0) >> 4)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[((source.ptr[i + 1] & 0xf) << 2) | ((source.ptr[i + 2] & 0xc0) >> 6)]; - out_index += 1; - - dest.ptr[out_index] = base64_fs_alphabet[source.ptr[i + 2] & 0x3f]; - out_index += 1; - } - - // Assert that we never need pad characters. - assert(i == source.len); -} - -// Called before init() -static bool build_with_cache(CodeGen *g) { - blake2b_state blake; - int rc = blake2b_init(&blake, 48); - assert(rc == 0); - - // TODO zig exe & dynamic libraries - - add_cache_buf(&blake, g->root_out_name); - add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file - add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); - add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); - add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); - add_cache_int(&blake, g->emit_file_type); - add_cache_int(&blake, g->build_mode); - add_cache_int(&blake, g->out_type); - // TODO the rest of the struct CodeGen fields - - uint8_t bin_digest[48]; - rc = blake2b_final(&blake, bin_digest, 48); - assert(rc == 0); - - Buf b64_digest = BUF_INIT; - buf_resize(&b64_digest, 64); - base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); - - fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); - // TODO next look for a manifest file that has all the files from the input parameters - // use that to construct the real hash, which looks up the output directory and another manifest file - - return false; -} - void codegen_build(CodeGen *g) { assert(g->out_type != OutTypeUnknown); - if (build_with_cache(g)) - return; + //if (build_with_cache(g)) + // return; init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/error.cpp b/src/error.cpp index 8303a80a1d..d9da6e6c52 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -27,6 +27,8 @@ const char *err_str(int err) { case ErrorNegativeDenominator: return "negative denominator"; case ErrorShiftedOutOneBits: return "exact shift shifted out one bits"; case ErrorCCompileErrors: return "C compile errors"; + case ErrorEndOfFile: return "end of file"; + case ErrorIsDir: return "is directory"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index e3b87fc6b8..c99c6e6b12 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -27,6 +27,8 @@ enum Error { ErrorNegativeDenominator, ErrorShiftedOutOneBits, ErrorCCompileErrors, + ErrorEndOfFile, + ErrorIsDir, }; const char *err_str(int err); diff --git a/src/main.cpp b/src/main.cpp index eed31438d9..172fe04da9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,6 +13,7 @@ #include "link.hpp" #include "os.hpp" #include "target.hpp" +#include "cache_hash.hpp" #include @@ -270,6 +271,66 @@ int main(int argc, char **argv) { return 0; } + if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "get app dir: %s\n", err_str(err)); + return 1; + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) { + fprintf(stderr, "make path: %s\n", err_str(err)); + return 1; + } + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) { + fprintf(stderr, "self exe path: %s\n", err_str(err)); + return 1; + } + + cache_file(ch, &self_exe_path); + + Buf exe_digest = BUF_INIT; + buf_resize(&exe_digest, 0); + if ((err = cache_hit(ch, &exe_digest))) { + fprintf(stderr, "cache hit error: %s\n", err_str(err)); + return 1; + } + if (buf_len(&exe_digest) != 0) { + fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); + return 0; + } + fprintf(stderr, "cache miss\n"); + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) { + fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); + return 1; + } + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) { + fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); + return 1; + } + } + if ((err = cache_final(ch, &exe_digest))) { + fprintf(stderr, "final: %s\n", err_str(err)); + return 1; + } + + + fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); + return 0; + } + os_init(); char *arg0 = argv[0]; diff --git a/src/os.cpp b/src/os.cpp index 0e62d84a48..d6e17ca5a1 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -40,6 +40,10 @@ typedef SSIZE_T ssize_t; #endif +#if defined(ZIG_OS_LINUX) +#include +#endif + #if defined(__MACH__) #include @@ -57,54 +61,6 @@ static clock_serv_t cclock; #include #include -// Ported from std/mem.zig. -// Coordinate struct fields with memSplit function -struct SplitIterator { - size_t index; - Slice buffer; - Slice split_bytes; -}; - -// Ported from std/mem.zig. -static bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { - for (size_t i = 0; i < self->split_bytes.len; i += 1) { - if (byte == self->split_bytes.ptr[i]) { - return true; - } - } - return false; -} - -// Ported from std/mem.zig. -static Optional> SplitIterator_next(SplitIterator *self) { - // move to beginning of token - while (self->index < self->buffer.len && - SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t start = self->index; - if (start == self->buffer.len) { - return {}; - } - - // move to end of token - while (self->index < self->buffer.len && - !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) - { - self->index += 1; - } - size_t end = self->index; - - return Optional>::some(self->buffer.slice(start, end)); -} - -// Ported from std/mem.zig -static SplitIterator memSplit(Slice buffer, Slice split_bytes) { - return SplitIterator{0, buffer, split_bytes}; -} - - #if defined(ZIG_OS_POSIX) static void populate_termination(Termination *term, int status) { if (WIFEXITED(status)) { @@ -1368,16 +1324,16 @@ double os_get_time(void) { #endif } -int os_make_path(Buf *path) { +Error os_make_path(Buf *path) { Buf resolved_path = os_path_resolve(&path, 1); size_t end_index = buf_len(&resolved_path); - int err; + Error err; while (true) { if ((err = os_make_dir(buf_slice(&resolved_path, 0, end_index)))) { if (err == ErrorPathAlreadyExists) { if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; } else if (err == ErrorFileNotFound) { // march end_index backward until next path component while (true) { @@ -1391,7 +1347,7 @@ int os_make_path(Buf *path) { } } if (end_index == buf_len(&resolved_path)) - return 0; + return ErrorNone; // march end_index forward until next path component while (true) { end_index += 1; @@ -1399,10 +1355,10 @@ int os_make_path(Buf *path) { break; } } - return 0; + return ErrorNone; } -int os_make_dir(Buf *path) { +Error os_make_dir(Buf *path) { #if defined(ZIG_OS_WINDOWS) if (!CreateDirectory(buf_ptr(path), NULL)) { if (GetLastError() == ERROR_ALREADY_EXISTS) @@ -1413,7 +1369,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #else if (mkdir(buf_ptr(path), 0755) == -1) { if (errno == EEXIST) @@ -1424,7 +1380,7 @@ int os_make_dir(Buf *path) { return ErrorAccess; return ErrorUnexpected; } - return 0; + return ErrorNone; #endif } @@ -1447,7 +1403,7 @@ int os_init(void) { return 0; } -int os_self_exe_path(Buf *out_path) { +Error os_self_exe_path(Buf *out_path) { #if defined(ZIG_OS_WINDOWS) buf_resize(out_path, 256); for (;;) { @@ -1480,27 +1436,21 @@ int os_self_exe_path(Buf *out_path) { char *real_path = realpath(buf_ptr(tmp), buf_ptr(out_path)); if (!real_path) { buf_init_from_buf(out_path, tmp); - return 0; + return ErrorNone; } // Resize out_path for the correct length. buf_resize(out_path, strlen(buf_ptr(out_path))); - return 0; + return ErrorNone; #elif defined(ZIG_OS_LINUX) - buf_resize(out_path, 256); - for (;;) { - ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); - if (amt == -1) { - return ErrorUnexpected; - } - if (amt == (ssize_t)buf_len(out_path)) { - buf_resize(out_path, buf_len(out_path) * 2); - continue; - } - buf_resize(out_path, amt); - return 0; + buf_resize(out_path, PATH_MAX); + ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); + if (amt == -1) { + return ErrorUnexpected; } + buf_resize(out_path, amt); + return ErrorNone; #endif return ErrorFileNotFound; } @@ -1685,3 +1635,225 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy return ErrorFileNotFound; #endif } + +// Ported from std/os/get_app_data_dir.zig +Error os_get_app_data_dir(Buf *out_path, const char *appname) { +#if defined(ZIG_OS_WINDOWS) +#error "Unimplemented" +#elif defined(ZIG_OS_DARWIN) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/Library/Application Support/%s", home_dir, appname); + return ErrorNone; +#elif defined(ZIG_OS_LINUX) + const char *home_dir = getenv("HOME"); + if (home_dir == nullptr) { + // TODO use /etc/passwd + return ErrorFileNotFound; + } + buf_resize(out_path, 0); + buf_appendf(out_path, "%s/.local/share/%s", home_dir, appname); + return ErrorNone; +#endif +} + + +#if defined(ZIG_OS_LINUX) +static int self_exe_shared_libs_callback(struct dl_phdr_info *info, size_t size, void *data) { + ZigList *libs = reinterpret_cast< ZigList *>(data); + if (info->dlpi_name[0] == '/') { + libs->append(buf_create_from_str(info->dlpi_name)); + } + return 0; +} +#endif + +Error os_self_exe_shared_libs(ZigList &paths) { +#if defined(ZIG_OS_LINUX) + paths.resize(0); + dl_iterate_phdr(self_exe_shared_libs_callback, &paths); + return ErrorNone; +#else +#error "unimplemented" +#endif +} + +Error os_file_open_r(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + for (;;) { + int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + *out_file = fd; + return ErrorNone; + } +#endif +} + +Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) { +#if defined(ZIG_OS_WINDOWS) +#error "unimplemented" +#else + int fd; + for (;;) { + fd = open(buf_ptr(full_path), O_RDWR|O_CLOEXEC|O_CREAT, 0666); + if (fd == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EACCES: + return ErrorAccess; + case EISDIR: + return ErrorIsDir; + case ENOENT: + return ErrorFileNotFound; + default: + return ErrorFileSystem; + } + } + break; + } + for (;;) { + struct flock lock; + lock.l_type = F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 0; + if (fcntl(fd, F_SETLKW, &lock) == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EINVAL: + zig_unreachable(); + default: + close(fd); + return ErrorFileSystem; + } + } + break; + } + *out_file = fd; + return ErrorNone; +#endif +} + +Error os_file_mtime(OsFile file, OsTimeStamp *mtime) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + struct stat statbuf; + if (fstat(file, &statbuf) == -1) + return ErrorFileSystem; + + mtime->sec = statbuf.st_mtim.tv_sec; + mtime->nsec = statbuf.st_mtim.tv_nsec; + return ErrorNone; +#endif +} + +Error os_file_read(OsFile file, void *ptr, size_t *len) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + for (;;) { + ssize_t rc = read(file, ptr, *len); + if (rc == -1) { + switch (errno) { + case EINTR: + continue; + case EBADF: + zig_unreachable(); + case EFAULT: + zig_unreachable(); + case EISDIR: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + *len = rc; + return ErrorNone; + } +#endif +} + +Error os_file_read_all(OsFile file, Buf *contents) { + Error err; + size_t index = 0; + for (;;) { + size_t amt = buf_len(contents) - index; + + if (amt < 512) { + buf_resize(contents, buf_len(contents) + 512); + amt += 512; + } + + if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) + return err; + + if (amt == 0) { + buf_resize(contents, index); + return ErrorNone; + } + + index += amt; + } +} + +Error os_file_overwrite(OsFile file, Buf *contents) { +#if defined(ZIG_OS_WINDOWS) +#error unimplemented +#else + if (lseek(file, 0, SEEK_SET) == -1) + return ErrorFileSystem; + for (;;) { + if (write(file, buf_ptr(contents), buf_len(contents)) == -1) { + switch (errno) { + case EINTR: + continue; + case EINVAL: + zig_unreachable(); + case EBADF: + zig_unreachable(); + default: + return ErrorFileSystem; + } + } + return ErrorNone; + } +#endif +} + +void os_file_close(OsFile file) { + close(file); +} diff --git a/src/os.hpp b/src/os.hpp index a44fa8160e..c64eccf8d4 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -13,77 +13,11 @@ #include "error.hpp" #include "zig_llvm.h" #include "windows_sdk.h" +#include "result.hpp" #include #include -enum TermColor { - TermColorRed, - TermColorGreen, - TermColorCyan, - TermColorWhite, - TermColorBold, - TermColorReset, -}; - -enum TerminationId { - TerminationIdClean, - TerminationIdSignaled, - TerminationIdStopped, - TerminationIdUnknown, -}; - -struct Termination { - TerminationId how; - int code; -}; - -int os_init(void); - -void os_spawn_process(const char *exe, ZigList &args, Termination *term); -int os_exec_process(const char *exe, ZigList &args, - Termination *term, Buf *out_stderr, Buf *out_stdout); - -void os_path_dirname(Buf *full_path, Buf *out_dirname); -void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); -void os_path_extname(Buf *full_path, Buf *out_basename, Buf *out_extname); -void os_path_join(Buf *dirname, Buf *basename, Buf *out_full_path); -int os_path_real(Buf *rel_path, Buf *out_abs_path); -Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); -bool os_path_is_absolute(Buf *path); - -int os_get_global_cache_directory(Buf *out_tmp_path); - -int os_make_path(Buf *path); -int os_make_dir(Buf *path); - -void os_write_file(Buf *full_path, Buf *contents); -int os_copy_file(Buf *src_path, Buf *dest_path); - -int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); - -int os_get_cwd(Buf *out_cwd); - -bool os_stderr_tty(void); -void os_stderr_set_color(TermColor color); - -int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); -int os_delete_file(Buf *path); - -int os_file_exists(Buf *full_path, bool *result); - -int os_rename(Buf *src_path, Buf *dest_path); -double os_get_time(void); - -bool os_is_sep(uint8_t c); - -int os_self_exe_path(Buf *out_path); - -int os_get_win32_ucrt_include_path(ZigWindowsSDK *sdk, Buf *output_buf); -int os_get_win32_ucrt_lib_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); -int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); - #if defined(__APPLE__) #define ZIG_OS_DARWIN #elif defined(_WIN32) @@ -116,4 +50,94 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchTy #define ZIG_OS_SEP_CHAR '/' #endif +enum TermColor { + TermColorRed, + TermColorGreen, + TermColorCyan, + TermColorWhite, + TermColorBold, + TermColorReset, +}; + +enum TerminationId { + TerminationIdClean, + TerminationIdSignaled, + TerminationIdStopped, + TerminationIdUnknown, +}; + +struct Termination { + TerminationId how; + int code; +}; + +#if defined(ZIG_OS_WINDOWS) +#define OsFile (void *) +#else +#define OsFile int +#endif + +struct OsTimeStamp { + uint64_t sec; + uint64_t nsec; +}; + +int os_init(void); + +void os_spawn_process(const char *exe, ZigList &args, Termination *term); +int os_exec_process(const char *exe, ZigList &args, + Termination *term, Buf *out_stderr, Buf *out_stdout); + +void os_path_dirname(Buf *full_path, Buf *out_dirname); +void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); +void os_path_extname(Buf *full_path, Buf *out_basename, Buf *out_extname); +void os_path_join(Buf *dirname, Buf *basename, Buf *out_full_path); +int os_path_real(Buf *rel_path, Buf *out_abs_path); +Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); +bool os_path_is_absolute(Buf *path); + +int os_get_global_cache_directory(Buf *out_tmp_path); + +Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); +Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); + +Error ATTRIBUTE_MUST_USE os_file_open_r(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_open_lock_rw(Buf *full_path, OsFile *out_file); +Error ATTRIBUTE_MUST_USE os_file_mtime(OsFile file, OsTimeStamp *mtime); +Error ATTRIBUTE_MUST_USE os_file_read(OsFile file, void *ptr, size_t *len); +Error ATTRIBUTE_MUST_USE os_file_read_all(OsFile file, Buf *contents); +Error ATTRIBUTE_MUST_USE os_file_overwrite(OsFile file, Buf *contents); +void os_file_close(OsFile file); + +void os_write_file(Buf *full_path, Buf *contents); +int os_copy_file(Buf *src_path, Buf *dest_path); + +int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); + +int os_get_cwd(Buf *out_cwd); + +bool os_stderr_tty(void); +void os_stderr_set_color(TermColor color); + +int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); +int os_delete_file(Buf *path); + +int os_file_exists(Buf *full_path, bool *result); + +int os_rename(Buf *src_path, Buf *dest_path); +double os_get_time(void); + +bool os_is_sep(uint8_t c); + +Error ATTRIBUTE_MUST_USE os_self_exe_path(Buf *out_path); + +Error ATTRIBUTE_MUST_USE os_get_app_data_dir(Buf *out_path, const char *appname); + +int os_get_win32_ucrt_include_path(ZigWindowsSDK *sdk, Buf *output_buf); +int os_get_win32_ucrt_lib_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); +int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf *output_buf, ZigLLVM_ArchType platform_type); + +Error ATTRIBUTE_MUST_USE os_self_exe_shared_libs(ZigList &paths); + #endif diff --git a/src/util.cpp b/src/util.cpp index cafd2c3d85..060d7f8fb5 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -43,3 +43,42 @@ uint32_t ptr_hash(const void *ptr) { bool ptr_eq(const void *a, const void *b) { return a == b; } + +// Ported from std/mem.zig. +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte) { + for (size_t i = 0; i < self->split_bytes.len; i += 1) { + if (byte == self->split_bytes.ptr[i]) { + return true; + } + } + return false; +} + +// Ported from std/mem.zig. +Optional> SplitIterator_next(SplitIterator *self) { + // move to beginning of token + while (self->index < self->buffer.len && + SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t start = self->index; + if (start == self->buffer.len) { + return {}; + } + + // move to end of token + while (self->index < self->buffer.len && + !SplitIterator_isSplitByte(self, self->buffer.ptr[self->index])) + { + self->index += 1; + } + size_t end = self->index; + + return Optional>::some(self->buffer.slice(start, end)); +} + +// Ported from std/mem.zig +SplitIterator memSplit(Slice buffer, Slice split_bytes) { + return SplitIterator{0, buffer, split_bytes}; +} diff --git a/src/util.hpp b/src/util.hpp index 1e02e3043c..f18c369fb5 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -254,4 +254,16 @@ static inline void memCopy(Slice dest, Slice src) { memcpy(dest.ptr, src.ptr, src.len * sizeof(T)); } +// Ported from std/mem.zig. +// Coordinate struct fields with memSplit function +struct SplitIterator { + size_t index; + Slice buffer; + Slice split_bytes; +}; + +bool SplitIterator_isSplitByte(SplitIterator *self, uint8_t byte); +Optional> SplitIterator_next(SplitIterator *self); +SplitIterator memSplit(Slice buffer, Slice split_bytes); + #endif From c0bdcc7417a23d338dc1df6bd74b30fe6e3d1a1a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 9 Sep 2018 23:58:52 -0400 Subject: [PATCH 05/20] `zig id` command --- src/cache_hash.cpp | 1 - src/main.cpp | 113 +++++++++++++++++++++++---------------------- src/os.cpp | 6 +-- 3 files changed, 60 insertions(+), 60 deletions(-) diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index 5447a5f3f2..ff530ca65c 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -377,7 +377,6 @@ static Error write_manifest_file(CacheHash *ch) { buf_appendf(&contents, "%" ZIG_PRI_u64 " %" ZIG_PRI_u64 " %s %s\n", chf->mtime.sec, chf->mtime.nsec, encoded_digest, buf_ptr(chf->path)); } - fprintf(stderr, "overwrite with\n%s\n", buf_ptr(&contents)); if ((err = os_file_overwrite(ch->manifest_file, &contents))) return err; diff --git a/src/main.cpp b/src/main.cpp index 172fe04da9..c385a1c88e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -25,6 +25,7 @@ static int usage(const char *arg0) { " build-lib [source] create library from source or object files\n" " build-obj [source] create object from source or assembly\n" " builtin show the source code of that @import(\"builtin\")\n" + " id print the base64-encoded compiler id\n" " run [source] create executable and run immediately\n" " translate-c [source] convert c code to zig code\n" " targets list available compilation targets\n" @@ -257,6 +258,55 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +static Buf saved_compiler_id = BUF_INIT; +static Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) + return err; + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + if ((err = os_make_path(manifest_dir))) + return err; + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + *result = &saved_compiler_id; + return ErrorNone; +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -271,64 +321,15 @@ int main(int argc, char **argv) { return 0; } - if (argc == 2 && strcmp(argv[1], "TEST") == 0) { + if (argc == 2 && strcmp(argv[1], "id") == 0) { Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "get app dir: %s\n", err_str(err)); - return 1; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - if ((err = os_make_path(manifest_dir))) { - fprintf(stderr, "make path: %s\n", err_str(err)); - return 1; - } - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) { - fprintf(stderr, "self exe path: %s\n", err_str(err)); - return 1; - } - - cache_file(ch, &self_exe_path); - - Buf exe_digest = BUF_INIT; - buf_resize(&exe_digest, 0); - if ((err = cache_hit(ch, &exe_digest))) { - fprintf(stderr, "cache hit error: %s\n", err_str(err)); - return 1; - } - if (buf_len(&exe_digest) != 0) { - fprintf(stderr, "cache hit: %s\n", buf_ptr(&exe_digest)); - return 0; - } - fprintf(stderr, "cache miss\n"); - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) { - fprintf(stderr, "finding out shared libs: %s\n", err_str(err)); - return 1; - } - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) { - fprintf(stderr, "cache add file %s: %s", buf_ptr(lib_path), err_str(err)); - return 1; - } - } - if ((err = cache_final(ch, &exe_digest))) { - fprintf(stderr, "final: %s\n", err_str(err)); - return 1; - } - - - fprintf(stderr, "computed2: %s\n", buf_ptr(&exe_digest)); - return 0; + printf("%s\n", buf_ptr(compiler_id)); + return EXIT_SUCCESS; } os_init(); diff --git a/src/os.cpp b/src/os.cpp index d6e17ca5a1..4ee4618184 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -1813,9 +1813,9 @@ Error os_file_read_all(OsFile file, Buf *contents) { for (;;) { size_t amt = buf_len(contents) - index; - if (amt < 512) { - buf_resize(contents, buf_len(contents) + 512); - amt += 512; + if (amt < 4096) { + buf_resize(contents, buf_len(contents) + (4096 - amt)); + amt = buf_len(contents) - index; } if ((err = os_file_read(file, buf_ptr(contents) + index, &amt))) From fbe5737c84c783cd31e6e2d595fc47eb782c5e3c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 09:46:15 -0400 Subject: [PATCH 06/20] stage1: always optimize blake and softfloat even in debug mode --- CMakeLists.txt | 14 +++++-- src/all_types.hpp | 4 ++ src/cache_hash.cpp | 4 ++ src/cache_hash.hpp | 3 +- src/codegen.cpp | 91 ++++++++++++++++++++++++++++------------------ src/codegen.hpp | 2 +- src/link.cpp | 2 +- src/main.cpp | 21 ++++++++--- 8 files changed, 94 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 615128159f..8339be71b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,7 +390,7 @@ if(MSVC) ) else() set_target_properties(embedded_softfloat PROPERTIES - COMPILE_FLAGS "-std=c99" + COMPILE_FLAGS "-std=c99 -O3" ) endif() target_include_directories(embedded_softfloat PUBLIC @@ -407,10 +407,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ast_render.cpp" "${CMAKE_SOURCE_DIR}/src/bigfloat.cpp" "${CMAKE_SOURCE_DIR}/src/bigint.cpp" - "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" "${CMAKE_SOURCE_DIR}/src/buffer.cpp" - "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" + "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" @@ -426,6 +425,9 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/util.cpp" "${CMAKE_SOURCE_DIR}/src/translate_c.cpp" ) +set(ZIG_SOURCES_O3 + "${CMAKE_SOURCE_DIR}/src/blake2b.cpp" +) set(ZIG_CPP_SOURCES "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp" "${CMAKE_SOURCE_DIR}/src/windows_sdk.cpp" @@ -813,6 +815,11 @@ set_target_properties(zig_cpp PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} ) +add_library(zig_O3 STATIC ${ZIG_SOURCES_O3}) +set_target_properties(zig_O3 PROPERTIES + COMPILE_FLAGS "${EXE_CFLAGS} -O3" +) + add_executable(zig ${ZIG_SOURCES}) set_target_properties(zig PROPERTIES COMPILE_FLAGS ${EXE_CFLAGS} @@ -821,6 +828,7 @@ set_target_properties(zig PROPERTIES target_link_libraries(zig LINK_PUBLIC zig_cpp + zig_O3 ${SOFTFLOAT_LIBRARIES} ${CLANG_LIBRARIES} ${LLD_LIBRARIES} diff --git a/src/all_types.hpp b/src/all_types.hpp index 0b0de58791..2dfbcaaa62 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -10,6 +10,7 @@ #include "list.hpp" #include "buffer.hpp" +#include "cache_hash.hpp" #include "zig_llvm.h" #include "hash_map.hpp" #include "errmsg.hpp" @@ -1613,7 +1614,10 @@ struct CodeGen { ZigType *entry_promise; } builtin_types; + CacheHash cache_hash; + //////////////////////////// Participates in Input Parameter Cache Hash + Buf *compiler_id; ZigList link_libs_list; // add -framework [name] args to linker ZigList darwin_frameworks; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index ff530ca65c..a0c43bc810 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -6,6 +6,7 @@ */ #include "cache_hash.hpp" +#include "all_types.hpp" #include "buffer.hpp" #include "os.hpp" @@ -219,6 +220,9 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { buf_append_str(ch->manifest_file_path, ".txt"); + if ((err = os_make_path(ch->manifest_dir))) + return err; + if ((err = os_file_open_lock_rw(ch->manifest_file_path, &ch->manifest_file))) return err; diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 77b22a1e41..3c2d77abc0 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -8,10 +8,11 @@ #ifndef ZIG_CACHE_HASH_HPP #define ZIG_CACHE_HASH_HPP -#include "all_types.hpp" #include "blake2.h" #include "os.hpp" +struct LinkLib; + struct CacheHashFile { Buf *path; OsTimeStamp mtime; diff --git a/src/codegen.cpp b/src/codegen.cpp index 6c5648f626..a18b26bb18 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -88,12 +88,13 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir) + Buf *zig_lib_dir, Buf *compiler_id) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); + g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -7675,44 +7676,62 @@ void codegen_add_time_event(CodeGen *g, const char *name) { } -//// Called before init() -//static bool build_with_cache(CodeGen *g) { -// // TODO zig exe & dynamic libraries -// // should be in main.cpp I think. only needs to happen -// // once on startup. -// -// CacheHash comp; -// cache_init(&comp); -// -// add_cache_buf(&blake, g->root_out_name); -// add_cache_buf_opt(&blake, get_resolved_root_src_path(g)); // Root source file -// add_cache_list_of_link_lib(&blake, g->link_libs_list.items, g->link_libs_list.length); -// add_cache_list_of_buf(&blake, g->darwin_frameworks.items, g->darwin_frameworks.length); -// add_cache_list_of_buf(&blake, g->rpath_list.items, g->rpath_list.length); -// add_cache_int(&blake, g->emit_file_type); -// add_cache_int(&blake, g->build_mode); -// add_cache_int(&blake, g->out_type); -// // TODO the rest of the struct CodeGen fields -// -// uint8_t bin_digest[48]; -// rc = blake2b_final(&blake, bin_digest, 48); -// assert(rc == 0); -// -// Buf b64_digest = BUF_INIT; -// buf_resize(&b64_digest, 64); -// base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); -// -// fprintf(stderr, "input params hash: %s\n", buf_ptr(&b64_digest)); -// // TODO next look for a manifest file that has all the files from the input parameters -// // use that to construct the real hash, which looks up the output directory and another manifest file -// -// return false; -//} +// Called before init() +static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { + Error err; + + CacheHash *ch = &g->cache_hash; + cache_init(ch, manifest_dir); + + cache_buf(ch, g->compiler_id); + cache_buf(ch, g->root_out_name); + cache_file(ch, get_resolved_root_src_path(g)); // Root source file + cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); + cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); + cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_int(ch, g->emit_file_type); + cache_int(ch, g->build_mode); + cache_int(ch, g->out_type); + // TODO the rest of the struct CodeGen fields + + buf_resize(digest, 0); + if ((err = cache_hit(ch, digest))) + return err; + + return ErrorNone; +} void codegen_build(CodeGen *g) { + Error err; assert(g->out_type != OutTypeUnknown); - //if (build_with_cache(g)) - // return; + + Buf app_data_dir = BUF_INIT; + if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + Buf *stage1_dir = buf_alloc(); + os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + Buf digest = BUF_INIT; + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + if (buf_len(&digest) != 0) { + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + + Buf *this_artifact_dir = buf_alloc(); + os_path_join(artifact_dir, &digest, this_artifact_dir); + + fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); + return; + } + init(g); codegen_add_time_event(g, "Semantic Analysis"); diff --git a/src/codegen.hpp b/src/codegen.hpp index 55b38a0050..649cf06856 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir); + Buf *zig_lib_dir, Buf *compiler_id); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); diff --git a/src/link.cpp b/src/link.cpp index 78ad204fe9..c0eb25bd55 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -34,7 +34,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir); + parent_gen->zig_lib_dir, parent_gen->compiler_id); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; diff --git a/src/main.cpp b/src/main.cpp index c385a1c88e..1520ae7d67 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -274,8 +274,6 @@ static Error get_compiler_id(Buf **result) { Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - if ((err = os_make_path(manifest_dir))) - return err; CacheHash cache_hash; CacheHash *ch = &cache_hash; cache_init(ch, manifest_dir); @@ -432,8 +430,14 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -796,7 +800,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -871,7 +875,14 @@ int main(int argc, char **argv) { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) { + fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); + return EXIT_FAILURE; + } + + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, + compiler_id); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); From 32be6e9b2a9e6de501aadbe271c554a4682a10f8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 13:46:23 -0400 Subject: [PATCH 07/20] caching is working * add almost all the input parameter state to the hash - missing items are the detected MSVC installation on Windows and detected libc installation on POSIX - also missing are C files and .h files that libclang finds * artifacts are created in global cache directory instead of zig-cache. - exception: builtin.zig is still in zig-cache * zig run uses the new cache correctly * zig run uses execv on posix systems --- src/all_types.hpp | 295 ++++++++++++++++++++++----------------------- src/cache_hash.cpp | 70 +++++++++-- src/cache_hash.hpp | 27 +++-- src/codegen.cpp | 199 +++++++++++++++++++++--------- src/codegen.hpp | 4 +- src/error.cpp | 1 + src/error.hpp | 1 + src/ir.cpp | 11 +- src/link.cpp | 74 +++--------- src/link.hpp | 17 --- src/main.cpp | 64 ++++------ src/os.cpp | 54 +++------ src/os.hpp | 3 +- 13 files changed, 434 insertions(+), 386 deletions(-) delete mode 100644 src/link.hpp diff --git a/src/all_types.hpp b/src/all_types.hpp index 2dfbcaaa62..6d11244a25 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1554,6 +1554,40 @@ struct CodeGen { ZigLLVMDICompileUnit *compile_unit; ZigLLVMDIFile *compile_unit_file; LinkLib *libc_link_lib; + LLVMTargetDataRef target_data_ref; + LLVMTargetMachineRef target_machine; + ZigLLVMDIFile *dummy_di_file; + LLVMValueRef cur_ret_ptr; + LLVMValueRef cur_fn_val; + LLVMValueRef cur_err_ret_trace_val_arg; + LLVMValueRef cur_err_ret_trace_val_stack; + LLVMValueRef memcpy_fn_val; + LLVMValueRef memset_fn_val; + LLVMValueRef trap_fn_val; + LLVMValueRef return_address_fn_val; + LLVMValueRef frame_address_fn_val; + LLVMValueRef coro_destroy_fn_val; + LLVMValueRef coro_id_fn_val; + LLVMValueRef coro_alloc_fn_val; + LLVMValueRef coro_size_fn_val; + LLVMValueRef coro_begin_fn_val; + LLVMValueRef coro_suspend_fn_val; + LLVMValueRef coro_end_fn_val; + LLVMValueRef coro_free_fn_val; + LLVMValueRef coro_resume_fn_val; + LLVMValueRef coro_save_fn_val; + LLVMValueRef coro_promise_fn_val; + LLVMValueRef coro_alloc_helper_fn_val; + LLVMValueRef coro_frame_fn_val; + LLVMValueRef merge_err_ret_traces_fn_val; + LLVMValueRef add_error_return_trace_addr_fn_val; + LLVMValueRef stacksave_fn_val; + LLVMValueRef stackrestore_fn_val; + LLVMValueRef write_register_fn_val; + LLVMValueRef sp_md_node; + LLVMValueRef err_name_table; + LLVMValueRef safety_crash_err_fn; + LLVMValueRef return_err_fn; // reminder: hash tables must be initialized before use HashMap import_table; @@ -1576,8 +1610,23 @@ struct CodeGen { size_t resolve_queue_index; ZigList use_queue; size_t use_queue_index; + ZigList timing_events; + ZigList error_di_types; + ZigList tld_ref_source_node_stack; + ZigList inline_fns; + ZigList test_fns; + ZigList err_enumerators; + ZigList errors_by_index; + size_t largest_err_name_len; - uint32_t next_unresolved_index; + PackageTableEntry *std_package; + PackageTableEntry *panic_package; + PackageTableEntry *test_runner_package; + PackageTableEntry *compile_var_package; + ImportTableEntry *compile_var_import; + ImportTableEntry *root_import; + ImportTableEntry *bootstrap_import; + ImportTableEntry *test_runner_import; struct { ZigType *entry_bool; @@ -1613,30 +1662,45 @@ struct CodeGen { ZigType *entry_arg_tuple; ZigType *entry_promise; } builtin_types; + ZigType *align_amt_type; + ZigType *stack_trace_type; + ZigType *ptr_to_stack_trace_type; + ZigType *err_tag_type; + ZigType *test_fn_type; + + Buf triple_str; + Buf global_asm; + Buf *out_h_path; + Buf cache_dir; + Buf artifact_dir; + Buf output_file_path; + Buf o_file_output_path; + Buf *wanted_output_file_path; + + IrInstruction *invalid_instruction; + + ConstExprValue const_void_val; + ConstExprValue panic_msg_vals[PanicMsgIdCount]; + + // The function definitions this module includes. + ZigList fn_defs; + size_t fn_defs_index; + ZigList global_vars; + + ZigFn *cur_fn; + ZigFn *main_fn; + ZigFn *panic_fn; + AstNode *root_export_decl; CacheHash cache_hash; - - //////////////////////////// Participates in Input Parameter Cache Hash - Buf *compiler_id; - ZigList link_libs_list; - // add -framework [name] args to linker - ZigList darwin_frameworks; - // add -rpath [name] args to linker - ZigList rpath_list; - - EmitFileType emit_file_type; - BuildMode build_mode; - OutType out_type; - - - //////////////////////////// Unsorted - - ZigTarget zig_target; - LLVMTargetDataRef target_data_ref; + ErrColor err_color; + uint32_t next_unresolved_index; unsigned pointer_size_bytes; + uint32_t target_os_index; + uint32_t target_arch_index; + uint32_t target_environ_index; + uint32_t target_oformat_index; bool is_big_endian; - bool is_static; - bool strip_debug_symbols; bool want_h_file; bool have_pub_main; bool have_c_main; @@ -1644,6 +1708,64 @@ struct CodeGen { bool have_winmain_crt_startup; bool have_dllmain_crt_startup; bool have_pub_panic; + bool have_err_ret_tracing; + bool c_want_stdint; + bool c_want_stdbool; + bool verbose_tokenize; + bool verbose_ast; + bool verbose_link; + bool verbose_ir; + bool verbose_llvm_ir; + bool verbose_cimport; + bool error_during_imports; + bool generate_error_name_table; + + //////////////////////////// Participates in Input Parameter Cache Hash + ZigList link_libs_list; + // add -framework [name] args to linker + ZigList darwin_frameworks; + // add -rpath [name] args to linker + ZigList rpath_list; + ZigList forbidden_libs; + ZigList link_objects; + ZigList assembly_files; + ZigList lib_dirs; + + Buf *compiler_id; + size_t version_major; + size_t version_minor; + size_t version_patch; + const char *linker_script; + + EmitFileType emit_file_type; + BuildMode build_mode; + OutType out_type; + ZigTarget zig_target; + bool is_static; + bool strip_debug_symbols; + bool is_test_build; + bool is_native_target; + bool windows_subsystem_windows; + bool windows_subsystem_console; + bool linker_rdynamic; + bool no_rosegment_workaround; + bool each_lib_rpath; + + Buf *mmacosx_version_min; + Buf *mios_version_min; + Buf *root_out_name; + Buf *test_filter; + Buf *test_name_prefix; + PackageTableEntry *root_package; + + const char **llvm_argv; + size_t llvm_argv_len; + + const char **clang_argv; + size_t clang_argv_len; + + //////////////////////////// Unsorted + Buf *libc_lib_dir; Buf *libc_static_lib_dir; Buf *libc_include_dir; @@ -1654,138 +1776,7 @@ struct CodeGen { Buf *zig_c_headers_dir; Buf *zig_std_special_dir; Buf *dynamic_linker; - Buf *ar_path; ZigWindowsSDK *win_sdk; - Buf triple_str; - bool is_test_build; - bool have_err_ret_tracing; - uint32_t target_os_index; - uint32_t target_arch_index; - uint32_t target_environ_index; - uint32_t target_oformat_index; - LLVMTargetMachineRef target_machine; - ZigLLVMDIFile *dummy_di_file; - bool is_native_target; - PackageTableEntry *root_package; // participates in cache hash - PackageTableEntry *std_package; - PackageTableEntry *panic_package; - PackageTableEntry *test_runner_package; - PackageTableEntry *compile_var_package; - ImportTableEntry *compile_var_import; - Buf *root_out_name; // participates in cache hash - bool windows_subsystem_windows; - bool windows_subsystem_console; - Buf *mmacosx_version_min; - Buf *mios_version_min; - bool linker_rdynamic; - const char *linker_script; - - // The function definitions this module includes. - ZigList fn_defs; - size_t fn_defs_index; - ZigList global_vars; - - ZigFn *cur_fn; - ZigFn *main_fn; - ZigFn *panic_fn; - LLVMValueRef cur_ret_ptr; - LLVMValueRef cur_fn_val; - LLVMValueRef cur_err_ret_trace_val_arg; - LLVMValueRef cur_err_ret_trace_val_stack; - bool c_want_stdint; - bool c_want_stdbool; - AstNode *root_export_decl; - size_t version_major; - size_t version_minor; - size_t version_patch; - bool verbose_tokenize; - bool verbose_ast; - bool verbose_link; - bool verbose_ir; - bool verbose_llvm_ir; - bool verbose_cimport; - ErrColor err_color; - ImportTableEntry *root_import; - ImportTableEntry *bootstrap_import; - ImportTableEntry *test_runner_import; - LLVMValueRef memcpy_fn_val; - LLVMValueRef memset_fn_val; - LLVMValueRef trap_fn_val; - LLVMValueRef return_address_fn_val; - LLVMValueRef frame_address_fn_val; - LLVMValueRef coro_destroy_fn_val; - LLVMValueRef coro_id_fn_val; - LLVMValueRef coro_alloc_fn_val; - LLVMValueRef coro_size_fn_val; - LLVMValueRef coro_begin_fn_val; - LLVMValueRef coro_suspend_fn_val; - LLVMValueRef coro_end_fn_val; - LLVMValueRef coro_free_fn_val; - LLVMValueRef coro_resume_fn_val; - LLVMValueRef coro_save_fn_val; - LLVMValueRef coro_promise_fn_val; - LLVMValueRef coro_alloc_helper_fn_val; - LLVMValueRef coro_frame_fn_val; - LLVMValueRef merge_err_ret_traces_fn_val; - LLVMValueRef add_error_return_trace_addr_fn_val; - LLVMValueRef stacksave_fn_val; - LLVMValueRef stackrestore_fn_val; - LLVMValueRef write_register_fn_val; - bool error_during_imports; - - LLVMValueRef sp_md_node; - - const char **clang_argv; - size_t clang_argv_len; - ZigList lib_dirs; - - const char **llvm_argv; - size_t llvm_argv_len; - - ZigList test_fns; - ZigType *test_fn_type; - - bool each_lib_rpath; - - ZigType *err_tag_type; - ZigList err_enumerators; - ZigList errors_by_index; - bool generate_error_name_table; - LLVMValueRef err_name_table; - size_t largest_err_name_len; - LLVMValueRef safety_crash_err_fn; - - LLVMValueRef return_err_fn; - - IrInstruction *invalid_instruction; - ConstExprValue const_void_val; - - ConstExprValue panic_msg_vals[PanicMsgIdCount]; - - Buf global_asm; - ZigList link_objects; - ZigList assembly_files; - - Buf *test_filter; - Buf *test_name_prefix; - - ZigList timing_events; - - Buf cache_dir; - Buf *out_h_path; - - ZigList inline_fns; - ZigList tld_ref_source_node_stack; - - ZigType *align_amt_type; - ZigType *stack_trace_type; - ZigType *ptr_to_stack_trace_type; - - ZigList error_di_types; - - ZigList forbidden_libs; - - bool no_rosegment_workaround; }; enum VarLinkage { diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index a0c43bc810..b6aebdac67 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -37,6 +37,20 @@ void cache_int(CacheHash *ch, int x) { blake2b_update(&ch->blake, buf, sizeof(int) + 1); } +void cache_usize(CacheHash *ch, size_t x) { + assert(ch->manifest_file_path == nullptr); + // + 1 to include the null byte + uint8_t buf[sizeof(size_t) + 1]; + memcpy(buf, &x, sizeof(size_t)); + buf[sizeof(size_t)] = 0; + blake2b_update(&ch->blake, buf, sizeof(size_t) + 1); +} + +void cache_bool(CacheHash *ch, bool x) { + assert(ch->manifest_file_path == nullptr); + blake2b_update(&ch->blake, &x, 1); +} + void cache_buf(CacheHash *ch, Buf *buf) { assert(ch->manifest_file_path == nullptr); assert(buf != nullptr); @@ -74,6 +88,26 @@ void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len) { cache_str(ch, ""); } +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + Buf *buf = ptr[i]; + cache_file(ch, buf); + } + cache_str(ch, ""); +} + +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len) { + assert(ch->manifest_file_path == nullptr); + + for (size_t i = 0; i < len; i += 1) { + const char *s = ptr[i]; + cache_str(ch, s); + } + cache_str(ch, ""); +} + void cache_file(CacheHash *ch, Buf *file_path) { assert(ch->manifest_file_path == nullptr); assert(file_path != nullptr); @@ -154,9 +188,13 @@ static Error base64_decode(Slice dest, Slice source) { return ErrorNone; } -static Error hash_file(uint8_t *digest, OsFile handle) { +static Error hash_file(uint8_t *digest, OsFile handle, Buf *contents) { Error err; + if (contents) { + buf_resize(contents, 0); + } + blake2b_state blake; int rc = blake2b_init(&blake, 48); assert(rc == 0); @@ -172,10 +210,13 @@ static Error hash_file(uint8_t *digest, OsFile handle) { return ErrorNone; } blake2b_update(&blake, buf, amt); + if (contents) { + buf_append_mem(contents, (char*)buf, amt); + } } } -static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { +static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf, Buf *contents) { Error err; assert(chf->path != nullptr); @@ -189,7 +230,7 @@ static Error populate_file_hash(CacheHash *ch, CacheHashFile *chf) { return err; } - if ((err = hash_file(chf->bin_digest, this_file))) { + if ((err = hash_file(chf->bin_digest, this_file, contents))) { os_file_close(this_file); return err; } @@ -323,7 +364,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { chf->mtime = actual_mtime; uint8_t actual_digest[48]; - if ((err = hash_file(actual_digest, this_file))) { + if ((err = hash_file(actual_digest, this_file, nullptr))) { os_file_close(this_file); os_file_close(ch->manifest_file); return err; @@ -344,7 +385,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { ch->manifest_dirty = true; for (; file_i < input_file_count; file_i += 1) { CacheHashFile *chf = &ch->files.at(file_i); - if ((err = populate_file_hash(ch, chf))) { + if ((err = populate_file_hash(ch, chf, nullptr))) { os_file_close(ch->manifest_file); return err; } @@ -355,13 +396,13 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { return cache_final(ch, out_digest); } -Error cache_add_file(CacheHash *ch, Buf *path) { +Error cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents) { Error err; assert(ch->manifest_file_path != nullptr); CacheHashFile *chf = ch->files.add_one(); - chf->path = path; - if ((err = populate_file_hash(ch, chf))) { + chf->path = resolved_path; + if ((err = populate_file_hash(ch, chf, contents))) { os_file_close(ch->manifest_file); return err; } @@ -369,6 +410,12 @@ Error cache_add_file(CacheHash *ch, Buf *path) { return ErrorNone; } +Error cache_add_file(CacheHash *ch, Buf *path) { + Buf *resolved_path = buf_alloc(); + *resolved_path = os_path_resolve(&path, 1); + return cache_add_file_fetch(ch, resolved_path, nullptr); +} + static Error write_manifest_file(CacheHash *ch) { Error err; Buf contents = BUF_INIT; @@ -398,7 +445,8 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { buf_ptr(ch->manifest_file_path), err_str(err)); } } - os_file_close(ch->manifest_file); + // We don't close the manifest file yet, because we want to + // keep it locked until the API user is done using it. uint8_t bin_digest[48]; int rc = blake2b_final(&ch->blake, bin_digest, 48); @@ -408,3 +456,7 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { return ErrorNone; } + +void cache_release(CacheHash *ch) { + os_file_close(ch->manifest_file); +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index 3c2d77abc0..ede7344c75 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -17,6 +17,7 @@ struct CacheHashFile { Buf *path; OsTimeStamp mtime; uint8_t bin_digest[48]; + Buf *contents; }; struct CacheHash { @@ -34,24 +35,36 @@ void cache_init(CacheHash *ch, Buf *manifest_dir); // Next, use the hash population functions to add the initial parameters. void cache_str(CacheHash *ch, const char *ptr); void cache_int(CacheHash *ch, int x); +void cache_bool(CacheHash *ch, bool x); +void cache_usize(CacheHash *ch, size_t x); void cache_buf(CacheHash *ch, Buf *buf); void cache_buf_opt(CacheHash *ch, Buf *buf); void cache_list_of_link_lib(CacheHash *ch, LinkLib **ptr, size_t len); void cache_list_of_buf(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_file(CacheHash *ch, Buf **ptr, size_t len); +void cache_list_of_str(CacheHash *ch, const char **ptr, size_t len); void cache_file(CacheHash *ch, Buf *path); void cache_file_opt(CacheHash *ch, Buf *path); // Then call cache_hit when you're ready to see if you can skip the next step. -// out_b64_digest will be left unchanged if it was a cache miss +// out_b64_digest will be left unchanged if it was a cache miss. +// If you got a cache hit, the next step is cache_release. +// From this point on, there is a lock on the input params. Release +// the lock with cache_release. Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest); -// If you got a cache hit, the flow is done. No more functions to call. -// Next call this function for every file that is depended on. +// If you did not get a cache hit, call this function for every file +// that is depended on, and then finish with cache_final. Error ATTRIBUTE_MUST_USE cache_add_file(CacheHash *ch, Buf *path); -// If you did not get a cache hit, use the hash population functions again -// and do all the actual work. When done use cache_final to save the cache -// for next time. -Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_digest); +// This variant of cache_add_file returns the file contents. +// Also the file path argument must be already resolved. +Error ATTRIBUTE_MUST_USE cache_add_file_fetch(CacheHash *ch, Buf *resolved_path, Buf *contents); + +// out_b64_digest will be the same thing that cache_hit returns if you got a cache hit +Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); + +// Until this function is called, no one will be able to get a lock on your input params. +void cache_release(CacheHash *ch); #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index a18b26bb18..1f53bb4c9d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -13,7 +13,6 @@ #include "error.hpp" #include "hash_map.hpp" #include "ir.hpp" -#include "link.hpp" #include "os.hpp" #include "translate_c.hpp" #include "target.hpp" @@ -188,6 +187,10 @@ void codegen_set_output_h_path(CodeGen *g, Buf *h_path) { g->out_h_path = h_path; } +void codegen_set_output_path(CodeGen *g, Buf *path) { + g->wanted_output_file_path = path; +} + void codegen_set_clang_argv(CodeGen *g, const char **args, size_t len) { g->clang_argv = args; g->clang_argv_len = len; @@ -5756,8 +5759,6 @@ static void validate_inline_fns(CodeGen *g) { static void do_code_gen(CodeGen *g) { assert(!g->errors.length); - codegen_add_time_event(g, "Code Generation"); - { // create debug type for error sets assert(g->err_enumerators.length == g->errors_by_index.length); @@ -6068,38 +6069,10 @@ static void do_code_gen(CodeGen *g) { codegen_add_time_event(g, "LLVM Emit Output"); - char *err_msg = nullptr; - Buf *o_basename = buf_create_from_buf(g->root_out_name); - - switch (g->emit_file_type) { - case EmitFileTypeBinary: - { - const char *o_ext = target_o_file_ext(&g->zig_target); - buf_append_str(o_basename, o_ext); - break; - } - case EmitFileTypeAssembly: - { - const char *asm_ext = target_asm_file_ext(&g->zig_target); - buf_append_str(o_basename, asm_ext); - break; - } - case EmitFileTypeLLVMIr: - { - const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); - buf_append_str(o_basename, llvm_ir_ext); - break; - } - default: - zig_unreachable(); - } - - Buf *output_path = buf_alloc(); - os_path_join(&g->cache_dir, o_basename, output_path); - ensure_cache_dir(g); - bool is_small = g->build_mode == BuildModeSmallRelease; + Buf *output_path = &g->o_file_output_path; + char *err_msg = nullptr; switch (g->emit_file_type) { case EmitFileTypeBinary: if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path), @@ -6118,7 +6091,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; case EmitFileTypeLLVMIr: @@ -6128,7 +6100,6 @@ static void do_code_gen(CodeGen *g) { zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg); } validate_inline_fns(g); - g->link_objects.append(output_path); break; default: @@ -7035,8 +7006,8 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package Buf *resolved_path = buf_alloc(); *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, false))) { + Error err; + if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7131,6 +7102,8 @@ static void gen_root_source(CodeGen *g) { Buf *source_code = buf_alloc(); int err; + // No need for using the caching system for this file fetch because it is handled + // separately. if ((err = os_fetch_file_path(resolved_path, source_code, true))) { fprintf(stderr, "unable to open '%s': %s\n", buf_ptr(resolved_path), err_str(err)); exit(1); @@ -7197,6 +7170,8 @@ static void gen_global_asm(CodeGen *g) { int err; for (size_t i = 0; i < g->assembly_files.length; i += 1) { Buf *asm_file = g->assembly_files.at(i); + // No need to use the caching system for these fetches because they + // are handled separately. if ((err = os_fetch_file_path(asm_file, &contents, false))) { zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err)); } @@ -7460,14 +7435,9 @@ static Buf *preprocessor_mangle(Buf *src) { } static void gen_h_file(CodeGen *g) { - if (!g->want_h_file) - return; - GenH gen_h_data = {0}; GenH *gen_h = &gen_h_data; - codegen_add_time_event(g, "Generate .h"); - assert(!g->is_test_build); if (!g->out_h_path) { @@ -7675,6 +7645,24 @@ void codegen_add_time_event(CodeGen *g, const char *name) { g->timing_events.append({os_get_time(), name}); } +static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { + if (buf_len(&pkg->root_src_path) == 0) + return; + + Buf *rel_full_path = buf_alloc(); + os_path_join(&pkg->root_src_dir, &pkg->root_src_path, rel_full_path); + cache_file(ch, rel_full_path); + + auto it = pkg->package_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + cache_buf(ch, entry->key); + add_cache_pkg(g, ch, entry->value); + } +} // Called before init() static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { @@ -7683,16 +7671,46 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); + add_cache_pkg(g, ch, g->root_package); + if (g->linker_script != nullptr) { + cache_file(ch, buf_create_from_str(g->linker_script)); + } cache_buf(ch, g->compiler_id); cache_buf(ch, g->root_out_name); - cache_file(ch, get_resolved_root_src_path(g)); // Root source file cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); cache_list_of_buf(ch, g->rpath_list.items, g->rpath_list.length); + cache_list_of_buf(ch, g->forbidden_libs.items, g->forbidden_libs.length); + cache_list_of_file(ch, g->link_objects.items, g->link_objects.length); + cache_list_of_file(ch, g->assembly_files.items, g->assembly_files.length); cache_int(ch, g->emit_file_type); cache_int(ch, g->build_mode); cache_int(ch, g->out_type); - // TODO the rest of the struct CodeGen fields + cache_int(ch, g->zig_target.arch.arch); + cache_int(ch, g->zig_target.arch.sub_arch); + cache_int(ch, g->zig_target.vendor); + cache_int(ch, g->zig_target.os); + cache_int(ch, g->zig_target.env_type); + cache_int(ch, g->zig_target.oformat); + cache_bool(ch, g->is_static); + cache_bool(ch, g->strip_debug_symbols); + cache_bool(ch, g->is_test_build); + cache_bool(ch, g->is_native_target); + cache_bool(ch, g->windows_subsystem_windows); + cache_bool(ch, g->windows_subsystem_console); + cache_bool(ch, g->linker_rdynamic); + cache_bool(ch, g->no_rosegment_workaround); + cache_bool(ch, g->each_lib_rpath); + cache_buf_opt(ch, g->mmacosx_version_min); + cache_buf_opt(ch, g->mios_version_min); + cache_usize(ch, g->version_major); + cache_usize(ch, g->version_minor); + cache_usize(ch, g->version_patch); + cache_buf_opt(ch, g->test_filter); + cache_buf_opt(ch, g->test_name_prefix); + cache_list_of_str(ch, g->llvm_argv, g->llvm_argv_len); + cache_list_of_str(ch, g->clang_argv, g->clang_argv_len); + cache_list_of_str(ch, g->lib_dirs.items, g->lib_dirs.length); buf_resize(digest, 0); if ((err = cache_hit(ch, digest))) @@ -7701,10 +7719,53 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { return ErrorNone; } -void codegen_build(CodeGen *g) { +static void resolve_out_paths(CodeGen *g) { + Buf *o_basename = buf_create_from_buf(g->root_out_name); + + switch (g->emit_file_type) { + case EmitFileTypeBinary: + { + const char *o_ext = target_o_file_ext(&g->zig_target); + buf_append_str(o_basename, o_ext); + break; + } + case EmitFileTypeAssembly: + { + const char *asm_ext = target_asm_file_ext(&g->zig_target); + buf_append_str(o_basename, asm_ext); + break; + } + case EmitFileTypeLLVMIr: + { + const char *llvm_ir_ext = target_llvm_ir_file_ext(&g->zig_target); + buf_append_str(o_basename, llvm_ir_ext); + break; + } + default: + zig_unreachable(); + } + + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + + if (g->out_type == OutTypeObj) { + buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); + } else if (g->out_type == OutTypeExe) { + assert(g->root_out_name); + + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } +} + + +void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); + codegen_add_time_event(g, "Check Cache"); + Buf app_data_dir = BUF_INIT; if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); @@ -7721,25 +7782,45 @@ void codegen_build(CodeGen *g) { fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); exit(1); } + + Buf *artifact_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + if (buf_len(&digest) != 0) { - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + os_path_join(artifact_dir, &digest, &g->artifact_dir); + resolve_out_paths(g); + } else { + init(g); - Buf *this_artifact_dir = buf_alloc(); - os_path_join(artifact_dir, &digest, this_artifact_dir); + codegen_add_time_event(g, "Semantic Analysis"); - fprintf(stderr, "copy artifacts from %s\n", buf_ptr(this_artifact_dir)); - return; + gen_global_asm(g); + gen_root_source(g); + + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + if ((err = os_make_path(&g->artifact_dir))) { + fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); + exit(1); + } + resolve_out_paths(g); + + codegen_add_time_event(g, "Code Generation"); + do_code_gen(g); + if (g->want_h_file) { + codegen_add_time_event(g, "Generate .h"); + gen_h_file(g); + } + if (g->out_type != OutTypeObj) { + codegen_link(g); + } } + // TODO hard link output_file_path to wanted_output_file_path - init(g); - - codegen_add_time_event(g, "Semantic Analysis"); - - gen_global_asm(g); - gen_root_source(g); - do_code_gen(g); - gen_h_file(g); + codegen_add_time_event(g, "Done"); } PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path) { diff --git a/src/codegen.hpp b/src/codegen.hpp index 649cf06856..1e7fafa28a 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -48,9 +48,11 @@ void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); +void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); void codegen_print_timing_report(CodeGen *g, FILE *f); -void codegen_build(CodeGen *g); +void codegen_link(CodeGen *g); +void codegen_build_and_link(CodeGen *g); PackageTableEntry *codegen_create_package(CodeGen *g, const char *root_src_dir, const char *root_src_path); void codegen_add_assembly(CodeGen *g, Buf *path); diff --git a/src/error.cpp b/src/error.cpp index d9da6e6c52..d503eaa18b 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -29,6 +29,7 @@ const char *err_str(int err) { case ErrorCCompileErrors: return "C compile errors"; case ErrorEndOfFile: return "end of file"; case ErrorIsDir: return "is directory"; + case ErrorUnsupportedOperatingSystem: return "unsupported operating system"; } return "(invalid error)"; } diff --git a/src/error.hpp b/src/error.hpp index c99c6e6b12..8c1f8b9aa5 100644 --- a/src/error.hpp +++ b/src/error.hpp @@ -29,6 +29,7 @@ enum Error { ErrorCCompileErrors, ErrorEndOfFile, ErrorIsDir, + ErrorUnsupportedOperatingSystem, }; const char *err_str(int err); diff --git a/src/ir.cpp b/src/ir.cpp index 0269c29b1a..22d9a9bc49 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16232,6 +16232,8 @@ static ZigType *ir_analyze_instruction_union_tag(IrAnalyze *ira, IrInstructionUn } static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImport *import_instruction) { + Error err; + IrInstruction *name_value = import_instruction->name->other; Buf *import_target_str = ir_resolve_str(ira, name_value); if (!import_target_str) @@ -16275,8 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - int err; - if ((err = os_fetch_file_path(resolved_path, import_code, true))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -16287,6 +16288,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_invalid; } } + ImportTableEntry *target_import = add_source_file(ira->codegen, target_package, resolved_path, import_code); scan_import(ira->codegen, target_import); @@ -18106,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = os_fetch_file_path(&file_path, file_contents, false))) { + if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; @@ -18116,9 +18118,6 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE } } - // TODO add dependency on the file we embedded so that we know if it changes - // we'll have to invalidate the cache - ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); init_const_str_lit(ira->codegen, out_val, file_contents); diff --git a/src/link.cpp b/src/link.cpp index c0eb25bd55..8c250fbbe1 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -5,7 +5,6 @@ * See http://opensource.org/licenses/MIT */ -#include "link.hpp" #include "os.hpp" #include "config.h" #include "codegen.hpp" @@ -13,7 +12,6 @@ struct LinkJob { CodeGen *codegen; - Buf out_file; ZigList args; bool link_in_crt; HashMap rpath_table; @@ -62,14 +60,8 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } - codegen_build(child_gen); - const char *o_ext = target_o_file_ext(&child_gen->zig_target); - Buf *o_out_name = buf_sprintf("%s%s", oname, o_ext); - Buf *output_path = buf_alloc(); - os_path_join(&parent_gen->cache_dir, o_out_name, output_path); - codegen_link(child_gen, buf_ptr(output_path)); - - return output_path; + codegen_build_and_link(child_gen); + return &child_gen->output_file_path; } static Buf *build_o(CodeGen *parent_gen, const char *oname) { @@ -237,15 +229,15 @@ static void construct_linker_job_elf(LinkJob *lj) { } else if (shared) { lj->args.append("-shared"); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.so.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } soname = buf_sprintf("lib%s.so.%" ZIG_PRI_usize "", buf_ptr(g->root_out_name), g->version_major); } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); if (lj->link_in_crt) { const char *crt1o; @@ -397,7 +389,7 @@ static void construct_linker_job_wasm(LinkJob *lj) { lj->args.append("--relocatable"); // So lld doesn't look for _start. lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); // .o files for (size_t i = 0; i < g->link_objects.length; i += 1) { @@ -478,7 +470,7 @@ static void construct_linker_job_coff(LinkJob *lj) { // } //} - lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&lj->out_file)))); + lj->args.append(buf_ptr(buf_sprintf("-OUT:%s", buf_ptr(&g->output_file_path)))); if (g->libc_link_lib != nullptr) { lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->msvc_lib_dir)))); @@ -585,11 +577,11 @@ static void construct_linker_job_coff(LinkJob *lj) { buf_appendf(def_contents, "\n"); Buf *def_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.def", buf_ptr(link_lib->name)), def_path); os_write_file(def_path, def_contents); Buf *generated_lib_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); + os_path_join(&g->artifact_dir, buf_sprintf("%s.lib", buf_ptr(link_lib->name)), generated_lib_path); gen_lib_args.resize(0); gen_lib_args.append("link"); @@ -797,8 +789,8 @@ static void construct_linker_job_macho(LinkJob *lj) { //lj->args.append("-install_name"); //lj->args.append(buf_ptr(dylib_install_name)); - if (buf_len(&lj->out_file) == 0) { - buf_appendf(&lj->out_file, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", + if (buf_len(&g->output_file_path) == 0) { + buf_appendf(&g->output_file_path, "lib%s.%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".%" ZIG_PRI_usize ".dylib", buf_ptr(g->root_out_name), g->version_major, g->version_minor, g->version_patch); } } @@ -832,13 +824,13 @@ static void construct_linker_job_macho(LinkJob *lj) { } lj->args.append("-o"); - lj->args.append(buf_ptr(&lj->out_file)); + lj->args.append(buf_ptr(&g->output_file_path)); for (size_t i = 0; i < g->rpath_list.length; i += 1) { Buf *rpath = g->rpath_list.at(i); add_rpath(lj, rpath); } - add_rpath(lj, &lj->out_file); + add_rpath(lj, &g->output_file_path); if (shared) { lj->args.append("-headerpad_max_install_names"); @@ -942,7 +934,8 @@ static void construct_linker_job(LinkJob *lj) { } } -void codegen_link(CodeGen *g, const char *out_file) { +void codegen_link(CodeGen *g) { + assert(g->out_type != OutTypeObj); codegen_add_time_event(g, "Build Dependencies"); LinkJob lj = {0}; @@ -953,11 +946,6 @@ void codegen_link(CodeGen *g, const char *out_file) { lj.rpath_table.init(4); lj.codegen = g; - if (out_file) { - buf_init_from_str(&lj.out_file, out_file); - } else { - buf_resize(&lj.out_file, 0); - } if (g->verbose_llvm_ir) { fprintf(stderr, "\nOptimization:\n"); @@ -966,35 +954,9 @@ void codegen_link(CodeGen *g, const char *out_file) { LLVMDumpModule(g->module); } - bool override_out_file = (buf_len(&lj.out_file) != 0); - if (!override_out_file) { - assert(g->root_out_name); - - buf_init_from_buf(&lj.out_file, g->root_out_name); - if (g->out_type == OutTypeExe) { - buf_append_str(&lj.out_file, target_exe_file_ext(&g->zig_target)); - } - } - - if (g->out_type == OutTypeObj) { - if (override_out_file) { - assert(g->link_objects.length == 1); - Buf *o_file_path = g->link_objects.at(0); - int err; - if ((err = os_rename(o_file_path, &lj.out_file))) { - zig_panic("unable to rename object file %s into final output %s: %s", buf_ptr(o_file_path), buf_ptr(&lj.out_file), err_str(err)); - } - } - return; - } - if (g->out_type == OutTypeLib && g->is_static) { - // invoke `ar` - // example: - // # static link into libfoo.a - // ar rcs libfoo.a foo1.o foo2.o - zig_panic("TODO invoke ar"); - return; + fprintf(stderr, "Zig does not yet support creating static libraries\nSee https://github.com/ziglang/zig/issues/1493\n"); + exit(1); } lj.link_in_crt = (g->libc_link_lib != nullptr && g->out_type == OutTypeExe); @@ -1017,6 +979,4 @@ void codegen_link(CodeGen *g, const char *out_file) { fprintf(stderr, "%s\n", buf_ptr(&diag)); exit(1); } - - codegen_add_time_event(g, "Done"); } diff --git a/src/link.hpp b/src/link.hpp deleted file mode 100644 index 9f978c28d9..0000000000 --- a/src/link.hpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2015 Andrew Kelley - * - * This file is part of zig, which is MIT licensed. - * See http://opensource.org/licenses/MIT - */ - -#ifndef ZIG_LINK_HPP -#define ZIG_LINK_HPP - -#include "all_types.hpp" - -void codegen_link(CodeGen *g, const char *out_file); - - -#endif - diff --git a/src/main.cpp b/src/main.cpp index 1520ae7d67..ff2c061a83 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,7 +10,6 @@ #include "codegen.hpp" #include "config.h" #include "error.hpp" -#include "link.hpp" #include "os.hpp" #include "target.hpp" #include "cache_hash.hpp" @@ -385,8 +384,7 @@ int main(int argc, char **argv) { CliPkg *cur_pkg = allocate(1); BuildMode build_mode = BuildModeDebug; ZigList test_exec_args = {0}; - int comptime_args_end = 0; - int runtime_args_start = argc; + int runtime_args_start = -1; bool no_rosegment_workaround = false; if (argc >= 2 && strcmp(argv[1], "build") == 0) { @@ -454,8 +452,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - Buf *path_to_build_exe = buf_alloc(); - os_path_join(&full_cache_dir, buf_create_from_str("build"), path_to_build_exe); codegen_set_cache_dir(g, full_cache_dir); args.items[1] = buf_ptr(&build_file_dirname); @@ -525,14 +521,13 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); - codegen_build(g); - codegen_link(g, buf_ptr(path_to_build_exe)); + codegen_build_and_link(g); Termination term; - os_spawn_process(buf_ptr(path_to_build_exe), args, &term); + os_spawn_process(buf_ptr(&g->output_file_path), args, &term); if (term.how != TerminationIdClean || term.code != 0) { fprintf(stderr, "\nBuild failed. The following command failed:\n"); - fprintf(stderr, "%s", buf_ptr(path_to_build_exe)); + fprintf(stderr, "%s", buf_ptr(&g->output_file_path)); for (size_t i = 0; i < args.length; i += 1) { fprintf(stderr, " %s", args.at(i)); } @@ -541,15 +536,11 @@ int main(int argc, char **argv) { return (term.how == TerminationIdClean) ? term.code : -1; } - for (int i = 1; i < argc; i += 1, comptime_args_end += 1) { + for (int i = 1; i < argc; i += 1) { char *arg = argv[i]; if (arg[0] == '-') { - if (strcmp(arg, "--") == 0) { - // ignore -- from both compile and runtime arg sets - runtime_args_start = i + 1; - break; - } else if (strcmp(arg, "--release-fast") == 0) { + if (strcmp(arg, "--release-fast") == 0) { build_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { build_mode = BuildModeSafeRelease; @@ -746,6 +737,10 @@ int main(int argc, char **argv) { case CmdTest: if (!in_file) { in_file = arg; + if (cmd == CmdRun) { + runtime_args_start = i + 1; + break; // rest of the args are for the program + } } else { fprintf(stderr, "Unexpected extra parameter: %s\n", arg); return usage(arg0); @@ -856,19 +851,10 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; Buf full_cache_dir = BUF_INIT; - Buf *run_exec_path = buf_alloc(); - if (cmd == CmdRun) { - if (buf_out_name == nullptr) { - buf_out_name = buf_create_from_str("run"); - } - - Buf *global_cache_dir = buf_alloc(); - os_get_global_cache_directory(global_cache_dir); - os_path_join(global_cache_dir, buf_out_name, run_exec_path); - full_cache_dir = os_path_resolve(&global_cache_dir, 1); - - out_file = buf_ptr(run_exec_path); - } else { + if (cmd == CmdRun && buf_out_name == nullptr) { + buf_out_name = buf_create_from_str("run"); + } + { Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); full_cache_dir = os_path_resolve(&resolve_paths, 1); } @@ -957,6 +943,8 @@ int main(int argc, char **argv) { codegen_set_test_name_prefix(g, buf_create_from_str(test_name_prefix)); } + if (out_file) + codegen_set_output_path(g, buf_create_from_str(out_file)); if (out_file_h) codegen_set_output_h_path(g, buf_create_from_str(out_file_h)); @@ -976,8 +964,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); - codegen_build(g); - codegen_link(g, out_file); + codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -987,8 +974,14 @@ int main(int argc, char **argv) { args.append(argv[i]); } + const char *exec_path = buf_ptr(&g->output_file_path); + args.append(nullptr); + + os_execv(exec_path, args.items); + + args.pop(); Termination term; - os_spawn_process(buf_ptr(run_exec_path), args, &term); + os_spawn_process(exec_path, args, &term); return term.code; } @@ -1005,11 +998,8 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); - ZigTarget *non_null_target = target ? target : &native; - - Buf *test_exe_name = buf_sprintf("test%s", target_exe_file_ext(non_null_target)); - Buf *test_exe_path = buf_alloc(); - os_path_join(&full_cache_dir, test_exe_name, test_exe_path); + codegen_build_and_link(g); + Buf *test_exe_path = &g->output_file_path; for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { @@ -1017,8 +1007,6 @@ int main(int argc, char **argv) { } } - codegen_build(g); - codegen_link(g, buf_ptr(test_exe_path)); if (!target_can_exec(&native, target)) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", diff --git a/src/os.cpp b/src/os.cpp index 4ee4618184..8502e72715 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -972,6 +972,22 @@ static int os_exec_process_windows(const char *exe, ZigList &args, } #endif +Error os_execv(const char *exe, const char **argv) { +#if defined(ZIG_OS_WINDOWS) + return ErrorUnsupportedOperatingSystem; +#else + execv(exe, (char *const *)argv); + switch (errno) { + case ENOMEM: + return ErrorSystemResources; + case EIO: + return ErrorFileSystem; + default: + return ErrorUnexpected; + } +#endif +} + int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout) { @@ -1238,44 +1254,6 @@ int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { #endif } -#if defined(ZIG_OS_POSIX) -int os_get_global_cache_directory(Buf *out_tmp_path) { - const char *tmp_dir = getenv("TMPDIR"); - if (!tmp_dir) { - tmp_dir = P_tmpdir; - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - -#if defined(ZIG_OS_WINDOWS) -int os_get_global_cache_directory(Buf *out_tmp_path) { - char tmp_dir[MAX_PATH + 1]; - if (GetTempPath(MAX_PATH, tmp_dir) == 0) { - zig_panic("GetTempPath failed"); - } - - Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); - Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); - - buf_resize(out_tmp_path, 0); - os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); - - buf_deinit(tmp_dir_buf); - buf_deinit(cache_dirname_buf); - return 0; -} -#endif - int os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index c64eccf8d4..fc2a34326f 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -87,6 +87,7 @@ int os_init(void); void os_spawn_process(const char *exe, ZigList &args, Termination *term); int os_exec_process(const char *exe, ZigList &args, Termination *term, Buf *out_stderr, Buf *out_stdout); +Error os_execv(const char *exe, const char **argv); void os_path_dirname(Buf *full_path, Buf *out_dirname); void os_path_split(Buf *full_path, Buf *out_dirname, Buf *out_basename); @@ -96,8 +97,6 @@ int os_path_real(Buf *rel_path, Buf *out_abs_path); Buf os_path_resolve(Buf **paths_ptr, size_t paths_len); bool os_path_is_absolute(Buf *path); -int os_get_global_cache_directory(Buf *out_tmp_path); - Error ATTRIBUTE_MUST_USE os_make_path(Buf *path); Error ATTRIBUTE_MUST_USE os_make_dir(Buf *path); From 5ee5933ade09c535bd1806d91cb606f49d07acea Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 10 Sep 2018 17:30:45 -0400 Subject: [PATCH 08/20] stage1 caching: zig no longer uses zig-cache --- CMakeLists.txt | 1 + src/all_types.hpp | 2 - src/cache_hash.cpp | 7 +++ src/codegen.cpp | 110 ++++++++++++++++++++++++++++++--------------- src/codegen.hpp | 3 +- src/compiler.cpp | 66 +++++++++++++++++++++++++++ src/compiler.hpp | 17 +++++++ src/link.cpp | 4 +- src/main.cpp | 78 ++------------------------------ src/os.cpp | 22 ++++----- src/os.hpp | 6 +-- 11 files changed, 187 insertions(+), 129 deletions(-) create mode 100644 src/compiler.cpp create mode 100644 src/compiler.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8339be71b9..11bb31892c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,6 +411,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp" "${CMAKE_SOURCE_DIR}/src/cache_hash.cpp" "${CMAKE_SOURCE_DIR}/src/codegen.cpp" + "${CMAKE_SOURCE_DIR}/src/compiler.cpp" "${CMAKE_SOURCE_DIR}/src/errmsg.cpp" "${CMAKE_SOURCE_DIR}/src/error.cpp" "${CMAKE_SOURCE_DIR}/src/ir.cpp" diff --git a/src/all_types.hpp b/src/all_types.hpp index 6d11244a25..4a15162076 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1671,7 +1671,6 @@ struct CodeGen { Buf triple_str; Buf global_asm; Buf *out_h_path; - Buf cache_dir; Buf artifact_dir; Buf output_file_path; Buf o_file_output_path; @@ -1731,7 +1730,6 @@ struct CodeGen { ZigList assembly_files; ZigList lib_dirs; - Buf *compiler_id; size_t version_major; size_t version_minor; size_t version_patch; diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b6aebdac67..b302946310 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -248,6 +248,12 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) { int rc = blake2b_final(&ch->blake, bin_digest, 48); assert(rc == 0); + if (ch->files.length == 0) { + buf_resize(out_digest, 64); + base64_encode(buf_to_slice(out_digest), {bin_digest, 48}); + return ErrorNone; + } + Buf b64_digest = BUF_INIT; buf_resize(&b64_digest, 64); base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48}); @@ -458,5 +464,6 @@ Error cache_final(CacheHash *ch, Buf *out_digest) { } void cache_release(CacheHash *ch) { + assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 1f53bb4c9d..578fb314a8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8,6 +8,7 @@ #include "analyze.hpp" #include "ast_render.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "errmsg.hpp" #include "error.hpp" @@ -87,13 +88,12 @@ static const char *symbols_that_llvm_depends_on[] = { }; CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id) + Buf *zig_lib_dir) { CodeGen *g = allocate(1); codegen_add_time_event(g, "Initialize"); - g->compiler_id = compiler_id; g->zig_lib_dir = zig_lib_dir; g->zig_std_dir = buf_alloc(); @@ -243,10 +243,6 @@ void codegen_set_out_name(CodeGen *g, Buf *out_name) { g->root_out_name = out_name; } -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir) { - g->cache_dir = cache_dir; -} - void codegen_set_libc_lib_dir(CodeGen *g, Buf *libc_lib_dir) { g->libc_lib_dir = libc_lib_dir; } @@ -5728,13 +5724,6 @@ static LLVMValueRef build_alloca(CodeGen *g, ZigType *type_entry, const char *na return result; } -static void ensure_cache_dir(CodeGen *g) { - int err; - if ((err = os_make_path(&g->cache_dir))) { - zig_panic("unable to make cache dir: %s", err_str(err)); - } -} - static void report_errors_and_maybe_exit(CodeGen *g) { if (g->errors.length != 0) { for (size_t i = 0; i < g->errors.length; i += 1) { @@ -6824,36 +6813,84 @@ Buf *codegen_generate_builtin_source(CodeGen *g) { return contents; } -static void define_builtin_compile_vars(CodeGen *g) { +static Error define_builtin_compile_vars(CodeGen *g) { if (g->std_package == nullptr) - return; + return ErrorNone; + + Error err; + + Buf *manifest_dir = buf_alloc(); + os_path_join(get_stage1_cache_path(), buf_create_from_str("builtin"), manifest_dir); + + CacheHash cache_hash; + cache_init(&cache_hash, manifest_dir); + + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + + // Only a few things affect builtin.zig + cache_buf(&cache_hash, compiler_id); + cache_int(&cache_hash, g->build_mode); + cache_bool(&cache_hash, g->is_test_build); + cache_int(&cache_hash, g->zig_target.arch.arch); + cache_int(&cache_hash, g->zig_target.arch.sub_arch); + cache_int(&cache_hash, g->zig_target.vendor); + cache_int(&cache_hash, g->zig_target.os); + cache_int(&cache_hash, g->zig_target.env_type); + cache_int(&cache_hash, g->zig_target.oformat); + cache_bool(&cache_hash, g->have_err_ret_tracing); + cache_bool(&cache_hash, g->libc_link_lib != nullptr); + + Buf digest = BUF_INIT; + buf_resize(&digest, 0); + if ((err = cache_hit(&cache_hash, &digest))) + return err; + + // We should always get a cache hit because there are no + // files in the input hash. + assert(buf_len(&digest) != 0); + + Buf *this_dir = buf_alloc(); + os_path_join(manifest_dir, &digest, this_dir); + + if ((err = os_make_path(this_dir))) + return err; const char *builtin_zig_basename = "builtin.zig"; Buf *builtin_zig_path = buf_alloc(); - os_path_join(&g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); + os_path_join(this_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); - Buf *contents = codegen_generate_builtin_source(g); - ensure_cache_dir(g); - os_write_file(builtin_zig_path, contents); - - Buf *resolved_path = buf_alloc(); - Buf *resolve_paths[] = {builtin_zig_path}; - *resolved_path = os_path_resolve(resolve_paths, 1); + bool hit; + if ((err = os_file_exists(builtin_zig_path, &hit))) + return err; + Buf *contents; + if (hit) { + contents = buf_alloc(); + if ((err = os_fetch_file_path(builtin_zig_path, contents, false))) { + fprintf(stderr, "Unable to open '%s': %s\n", buf_ptr(builtin_zig_path), err_str(err)); + exit(1); + } + } else { + contents = codegen_generate_builtin_source(g); + os_write_file(builtin_zig_path, contents); + } assert(g->root_package); assert(g->std_package); - g->compile_var_package = new_package(buf_ptr(&g->cache_dir), builtin_zig_basename); + g->compile_var_package = new_package(buf_ptr(this_dir), builtin_zig_basename); g->root_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); g->std_package->package_table.put(buf_create_from_str("builtin"), g->compile_var_package); - g->compile_var_import = add_source_file(g, g->compile_var_package, resolved_path, contents); + g->compile_var_import = add_source_file(g, g->compile_var_package, builtin_zig_path, contents); scan_import(g, g->compile_var_import); + + return ErrorNone; } static void init(CodeGen *g) { if (g->module) return; - if (g->llvm_argv_len > 0) { const char **args = allocate_nonzero(g->llvm_argv_len + 2); args[0] = "zig (LLVM option parsing)"; @@ -6960,7 +6997,11 @@ static void init(CodeGen *g) { g->have_err_ret_tracing = g->build_mode != BuildModeFastRelease && g->build_mode != BuildModeSmallRelease; define_builtin_fns(g); - define_builtin_compile_vars(g); + Error err; + if ((err = define_builtin_compile_vars(g))) { + fprintf(stderr, "Unable to create builtin.zig: %s\n", err_str(err)); + exit(1); + } } void codegen_translate_c(CodeGen *g, Buf *full_path) { @@ -7668,6 +7709,10 @@ static void add_cache_pkg(CodeGen *g, CacheHash *ch, PackageTableEntry *pkg) { static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { Error err; + Buf *compiler_id; + if ((err = get_compiler_id(&compiler_id))) + return err; + CacheHash *ch = &g->cache_hash; cache_init(ch, manifest_dir); @@ -7675,7 +7720,7 @@ static Error check_cache(CodeGen *g, Buf *manifest_dir, Buf *digest) { if (g->linker_script != nullptr) { cache_file(ch, buf_create_from_str(g->linker_script)); } - cache_buf(ch, g->compiler_id); + cache_buf(ch, compiler_id); cache_buf(ch, g->root_out_name); cache_list_of_link_lib(ch, g->link_libs_list.items, g->link_libs_list.length); cache_list_of_buf(ch, g->darwin_frameworks.items, g->darwin_frameworks.length); @@ -7766,13 +7811,7 @@ void codegen_build_and_link(CodeGen *g) { codegen_add_time_event(g, "Check Cache"); - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) { - fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); - exit(1); - } - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); + Buf *stage1_dir = get_stage1_cache_path(); Buf *manifest_dir = buf_alloc(); os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); @@ -7820,6 +7859,7 @@ void codegen_build_and_link(CodeGen *g) { } // TODO hard link output_file_path to wanted_output_file_path + cache_release(&g->cache_hash); codegen_add_time_event(g, "Done"); } diff --git a/src/codegen.hpp b/src/codegen.hpp index 1e7fafa28a..203e2d2b94 100644 --- a/src/codegen.hpp +++ b/src/codegen.hpp @@ -15,7 +15,7 @@ #include CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out_type, BuildMode build_mode, - Buf *zig_lib_dir, Buf *compiler_id); + Buf *zig_lib_dir); void codegen_set_clang_argv(CodeGen *codegen, const char **args, size_t len); void codegen_set_llvm_argv(CodeGen *codegen, const char **args, size_t len); @@ -46,7 +46,6 @@ void codegen_set_linker_script(CodeGen *g, const char *linker_script); void codegen_set_test_filter(CodeGen *g, Buf *filter); void codegen_set_test_name_prefix(CodeGen *g, Buf *prefix); void codegen_set_lib_version(CodeGen *g, size_t major, size_t minor, size_t patch); -void codegen_set_cache_dir(CodeGen *g, Buf cache_dir); void codegen_set_output_h_path(CodeGen *g, Buf *h_path); void codegen_set_output_path(CodeGen *g, Buf *path); void codegen_add_time_event(CodeGen *g, const char *name); diff --git a/src/compiler.cpp b/src/compiler.cpp new file mode 100644 index 0000000000..dd02b541dd --- /dev/null +++ b/src/compiler.cpp @@ -0,0 +1,66 @@ +#include "cache_hash.hpp" + +#include + +static Buf saved_compiler_id = BUF_INIT; +static Buf saved_app_data_dir = BUF_INIT; +static Buf saved_stage1_path = BUF_INIT; + +Buf *get_stage1_cache_path() { + if (saved_stage1_path.list.length != 0) { + return &saved_stage1_path; + } + Error err; + if ((err = os_get_app_data_dir(&saved_app_data_dir, "zig"))) { + fprintf(stderr, "Unable to get app data dir: %s\n", err_str(err)); + exit(1); + } + os_path_join(&saved_app_data_dir, buf_create_from_str("stage1"), &saved_stage1_path); + return &saved_stage1_path; +} + +Error get_compiler_id(Buf **result) { + if (saved_compiler_id.list.length != 0) { + *result = &saved_compiler_id; + return ErrorNone; + } + + Error err; + Buf *stage1_dir = get_stage1_cache_path(); + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); + + CacheHash cache_hash; + CacheHash *ch = &cache_hash; + cache_init(ch, manifest_dir); + Buf self_exe_path = BUF_INIT; + if ((err = os_self_exe_path(&self_exe_path))) + return err; + + cache_file(ch, &self_exe_path); + + buf_resize(&saved_compiler_id, 0); + if ((err = cache_hit(ch, &saved_compiler_id))) + return err; + if (buf_len(&saved_compiler_id) != 0) { + cache_release(ch); + *result = &saved_compiler_id; + return ErrorNone; + } + ZigList lib_paths = {}; + if ((err = os_self_exe_shared_libs(lib_paths))) + return err; + for (size_t i = 0; i < lib_paths.length; i += 1) { + Buf *lib_path = lib_paths.at(i); + if ((err = cache_add_file(ch, lib_path))) + return err; + } + if ((err = cache_final(ch, &saved_compiler_id))) + return err; + + cache_release(ch); + + *result = &saved_compiler_id; + return ErrorNone; +} + diff --git a/src/compiler.hpp b/src/compiler.hpp new file mode 100644 index 0000000000..b95e4ceda7 --- /dev/null +++ b/src/compiler.hpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2018 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_COMPILER_HPP +#define ZIG_COMPILER_HPP + +#include "buffer.hpp" +#include "error.hpp" + +Buf *get_stage1_cache_path(); +Error get_compiler_id(Buf **result); + +#endif diff --git a/src/link.cpp b/src/link.cpp index 8c250fbbe1..8d7b8b4d5f 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -32,7 +32,7 @@ static const char *get_libc_static_file(CodeGen *g, const char *file) { static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) { ZigTarget *child_target = parent_gen->is_native_target ? nullptr : &parent_gen->zig_target; CodeGen *child_gen = codegen_create(full_path, child_target, OutTypeObj, parent_gen->build_mode, - parent_gen->zig_lib_dir, parent_gen->compiler_id); + parent_gen->zig_lib_dir); child_gen->want_h_file = false; child_gen->verbose_tokenize = parent_gen->verbose_tokenize; @@ -42,8 +42,6 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) child_gen->verbose_llvm_ir = parent_gen->verbose_llvm_ir; child_gen->verbose_cimport = parent_gen->verbose_cimport; - codegen_set_cache_dir(child_gen, parent_gen->cache_dir); - codegen_set_strip(child_gen, parent_gen->strip_debug_symbols); codegen_set_is_static(child_gen, parent_gen->is_static); diff --git a/src/main.cpp b/src/main.cpp index ff2c061a83..10b789a6e1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,11 +8,11 @@ #include "ast_render.hpp" #include "buffer.hpp" #include "codegen.hpp" +#include "compiler.hpp" #include "config.h" #include "error.hpp" #include "os.hpp" #include "target.hpp" -#include "cache_hash.hpp" #include @@ -257,53 +257,6 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } -static Buf saved_compiler_id = BUF_INIT; -static Error get_compiler_id(Buf **result) { - if (saved_compiler_id.list.length != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - - Error err; - Buf app_data_dir = BUF_INIT; - if ((err = os_get_app_data_dir(&app_data_dir, "zig"))) - return err; - Buf *stage1_dir = buf_alloc(); - os_path_join(&app_data_dir, buf_create_from_str("stage1"), stage1_dir); - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("exe"), manifest_dir); - - CacheHash cache_hash; - CacheHash *ch = &cache_hash; - cache_init(ch, manifest_dir); - Buf self_exe_path = BUF_INIT; - if ((err = os_self_exe_path(&self_exe_path))) - return err; - - cache_file(ch, &self_exe_path); - - buf_resize(&saved_compiler_id, 0); - if ((err = cache_hit(ch, &saved_compiler_id))) - return err; - if (buf_len(&saved_compiler_id) != 0) { - *result = &saved_compiler_id; - return ErrorNone; - } - ZigList lib_paths = {}; - if ((err = os_self_exe_shared_libs(lib_paths))) - return err; - for (size_t i = 0; i < lib_paths.length; i += 1) { - Buf *lib_path = lib_paths.at(i); - if ((err = cache_add_file(ch, lib_path))) - return err; - } - if ((err = cache_final(ch, &saved_compiler_id))) - return err; - - *result = &saved_compiler_id; - return ErrorNone; -} - int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -428,14 +381,7 @@ int main(int argc, char **argv) { Buf *build_runner_path = buf_alloc(); os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf); codegen_set_out_name(g, buf_create_from_str("build")); Buf *build_file_buf = buf_create_from_str(build_file); @@ -452,8 +398,6 @@ int main(int argc, char **argv) { full_cache_dir = os_path_resolve(&cache_dir_buf, 1); } - codegen_set_cache_dir(g, full_cache_dir); - args.items[1] = buf_ptr(&build_file_dirname); args.items[2] = buf_ptr(&full_cache_dir); @@ -795,7 +739,7 @@ int main(int argc, char **argv) { switch (cmd) { case CmdBuiltin: { Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf, nullptr); + CodeGen *g = codegen_create(nullptr, target, out_type, build_mode, zig_lib_dir_buf); Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); @@ -850,30 +794,16 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; - Buf full_cache_dir = BUF_INIT; if (cmd == CmdRun && buf_out_name == nullptr) { buf_out_name = buf_create_from_str("run"); } - { - Buf *resolve_paths = buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir); - full_cache_dir = os_path_resolve(&resolve_paths, 1); - } - Buf *zig_lib_dir_buf = resolve_zig_lib_dir(); - Buf *compiler_id; - if ((err = get_compiler_id(&compiler_id))) { - fprintf(stderr, "Unable to determine compiler id: %s\n", err_str(err)); - return EXIT_FAILURE; - } - - CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf, - compiler_id); + CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf); codegen_set_out_name(g, buf_out_name); codegen_set_lib_version(g, ver_major, ver_minor, ver_patch); codegen_set_is_test(g, cmd == CmdTest); codegen_set_linker_script(g, linker_script); - codegen_set_cache_dir(g, full_cache_dir); if (each_lib_rpath) codegen_set_each_lib_rpath(g, each_lib_rpath); diff --git a/src/os.cpp b/src/os.cpp index 8502e72715..3950711c56 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -721,7 +721,7 @@ Buf os_path_resolve(Buf **paths_ptr, size_t paths_len) { #endif } -int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { +Error os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { static const ssize_t buf_size = 0x2000; buf_resize(out_buf, buf_size); ssize_t actual_buf_len = 0; @@ -757,7 +757,7 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { if (amt_read != buf_size) { if (feof(f)) { buf_resize(out_buf, actual_buf_len); - return 0; + return ErrorNone; } else { return ErrorFileSystem; } @@ -769,13 +769,13 @@ int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { zig_unreachable(); } -int os_file_exists(Buf *full_path, bool *result) { +Error os_file_exists(Buf *full_path, bool *result) { #if defined(ZIG_OS_WINDOWS) *result = GetFileAttributes(buf_ptr(full_path)) != INVALID_FILE_ATTRIBUTES; - return 0; + return ErrorNone; #else *result = access(buf_ptr(full_path), F_OK) != -1; - return 0; + return ErrorNone; #endif } @@ -834,13 +834,15 @@ static int os_exec_process_posix(const char *exe, ZigList &args, FILE *stdout_f = fdopen(stdout_pipe[0], "rb"); FILE *stderr_f = fdopen(stderr_pipe[0], "rb"); - os_fetch_file(stdout_f, out_stdout, false); - os_fetch_file(stderr_f, out_stderr, false); + Error err1 = os_fetch_file(stdout_f, out_stdout, false); + Error err2 = os_fetch_file(stderr_f, out_stderr, false); fclose(stdout_f); fclose(stderr_f); - return 0; + if (err1) return err1; + if (err2) return err2; + return ErrorNone; } } #endif @@ -1064,7 +1066,7 @@ int os_copy_file(Buf *src_path, Buf *dest_path) { } } -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { +Error os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { FILE *f = fopen(buf_ptr(full_path), "rb"); if (!f) { switch (errno) { @@ -1083,7 +1085,7 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { return ErrorFileSystem; } } - int result = os_fetch_file(f, out_contents, skip_shebang); + Error result = os_fetch_file(f, out_contents, skip_shebang); fclose(f); return result; } diff --git a/src/os.hpp b/src/os.hpp index fc2a34326f..ac422fbd21 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -111,8 +111,8 @@ void os_file_close(OsFile file); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); -int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); -int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +Error ATTRIBUTE_MUST_USE os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); int os_get_cwd(Buf *out_cwd); @@ -122,7 +122,7 @@ void os_stderr_set_color(TermColor color); int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path); int os_delete_file(Buf *path); -int os_file_exists(Buf *full_path, bool *result); +Error ATTRIBUTE_MUST_USE os_file_exists(Buf *full_path, bool *result); int os_rename(Buf *src_path, Buf *dest_path); double os_get_time(void); From 67735c6f1557092efe6e8c1712445c30655fe283 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 00:32:40 -0400 Subject: [PATCH 09/20] ability to disable cache. off by default except for... ...zig run, zig build, compiler_rt.a, and builtin.a --- src/all_types.hpp | 1 + src/analyze.cpp | 8 ++++ src/analyze.hpp | 3 ++ src/cache_hash.cpp | 8 ++++ src/cache_hash.hpp | 5 +++ src/codegen.cpp | 105 ++++++++++++++++++++++++++++++++------------- src/ir.cpp | 4 +- src/link.cpp | 1 + src/main.cpp | 50 +++++++++++++++++++-- src/target.cpp | 16 +++++++ src/target.hpp | 1 + 11 files changed, 167 insertions(+), 35 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 4a15162076..cdc290268a 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1718,6 +1718,7 @@ struct CodeGen { bool verbose_cimport; bool error_during_imports; bool generate_error_name_table; + bool enable_cache; //////////////////////////// Participates in Input Parameter Cache Hash ZigList link_libs_list; diff --git a/src/analyze.cpp b/src/analyze.cpp index aa4fda7624..7fe7a48245 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -6367,3 +6367,11 @@ not_integer: } return nullptr; } + +Error file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents) { + if (g->enable_cache) { + return cache_add_file_fetch(&g->cache_hash, resolved_path, contents); + } else { + return os_fetch_file_path(resolved_path, contents, false); + } +} diff --git a/src/analyze.hpp b/src/analyze.hpp index 41cc50916e..9f036c409d 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -210,4 +210,7 @@ ZigType *get_primitive_type(CodeGen *g, Buf *name); bool calling_convention_allows_zig_types(CallingConvention cc); const char *calling_convention_name(CallingConvention cc); + +Error ATTRIBUTE_MUST_USE file_fetch(CodeGen *g, Buf *resolved_path, Buf *contents); + #endif diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp index b302946310..65e3e62ae0 100644 --- a/src/cache_hash.cpp +++ b/src/cache_hash.cpp @@ -467,3 +467,11 @@ void cache_release(CacheHash *ch) { assert(ch->manifest_file_path != nullptr); os_file_close(ch->manifest_file); } + +Buf *get_random_basename() { + Buf *result = buf_alloc(); + for (size_t i = 0; i < 16; i += 1) { + buf_append_char(result, base64_fs_alphabet[rand() % 64]); + } + return result; +} diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp index ede7344c75..6acd805be6 100644 --- a/src/cache_hash.hpp +++ b/src/cache_hash.hpp @@ -67,4 +67,9 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest); // Until this function is called, no one will be able to get a lock on your input params. void cache_release(CacheHash *ch); + + +// Completely independent function. Just returns a random filename safe basename. +Buf *get_random_basename(); + #endif diff --git a/src/codegen.cpp b/src/codegen.cpp index 578fb314a8..1e0fd7fec3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7048,7 +7048,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package *resolved_path = os_path_resolve(resolve_paths, 1); Buf *import_code = buf_alloc(); Error err; - if ((err = cache_add_file_fetch(&g->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(g, resolved_path, import_code))) { zig_panic("unable to open '%s': %s\n", buf_ptr(&path_to_code_src), err_str(err)); } @@ -7480,10 +7480,7 @@ static void gen_h_file(CodeGen *g) { GenH *gen_h = &gen_h_data; assert(!g->is_test_build); - - if (!g->out_h_path) { - g->out_h_path = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); - } + assert(g->out_h_path != nullptr); FILE *out_h = fopen(buf_ptr(g->out_h_path), "wb"); if (!out_h) @@ -7790,17 +7787,56 @@ static void resolve_out_paths(CodeGen *g) { zig_unreachable(); } - os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + if (g->enable_cache || g->out_type != OutTypeObj) { + os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path); + } else { + buf_init_from_buf(&g->o_file_output_path, o_basename); + } if (g->out_type == OutTypeObj) { buf_init_from_buf(&g->output_file_path, &g->o_file_output_path); } else if (g->out_type == OutTypeExe) { - assert(g->root_out_name); + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + assert(g->root_out_name); - Buf basename = BUF_INIT; - buf_init_from_buf(&basename, g->root_out_name); - buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); - os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_exe_file_ext(&g->zig_target)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else if (g->out_type == OutTypeLib) { + if (!g->enable_cache && g->wanted_output_file_path != nullptr) { + buf_init_from_buf(&g->output_file_path, g->wanted_output_file_path); + } else { + Buf basename = BUF_INIT; + buf_init_from_buf(&basename, g->root_out_name); + buf_append_str(&basename, target_lib_file_ext(&g->zig_target, g->is_static, + g->version_major, g->version_minor, g->version_patch)); + if (g->enable_cache) { + os_path_join(&g->artifact_dir, &basename, &g->output_file_path); + } else { + buf_init_from_buf(&g->output_file_path, &basename); + } + } + } else { + zig_unreachable(); + } + + if (g->want_h_file && !g->out_h_path) { + assert(g->root_out_name); + Buf *h_basename = buf_sprintf("%s.h", buf_ptr(g->root_out_name)); + if (g->enable_cache) { + g->out_h_path = buf_alloc(); + os_path_join(&g->artifact_dir, h_basename, g->out_h_path); + } else { + g->out_h_path = h_basename; + } } } @@ -7809,23 +7845,26 @@ void codegen_build_and_link(CodeGen *g) { Error err; assert(g->out_type != OutTypeUnknown); - codegen_add_time_event(g, "Check Cache"); - Buf *stage1_dir = get_stage1_cache_path(); - - Buf *manifest_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); - + Buf *artifact_dir = buf_alloc(); Buf digest = BUF_INIT; - if ((err = check_cache(g, manifest_dir, &digest))) { - fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); - exit(1); + if (g->enable_cache) { + codegen_add_time_event(g, "Check Cache"); + + Buf *manifest_dir = buf_alloc(); + os_path_join(stage1_dir, buf_create_from_str("build"), manifest_dir); + + if ((err = check_cache(g, manifest_dir, &digest))) { + fprintf(stderr, "Unable to check cache: %s\n", err_str(err)); + exit(1); + } + + os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); + } else { + os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir); } - Buf *artifact_dir = buf_alloc(); - os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir); - - if (buf_len(&digest) != 0) { + if (g->enable_cache && buf_len(&digest) != 0) { os_path_join(artifact_dir, &digest, &g->artifact_dir); resolve_out_paths(g); } else { @@ -7836,11 +7875,16 @@ void codegen_build_and_link(CodeGen *g) { gen_global_asm(g); gen_root_source(g); - if ((err = cache_final(&g->cache_hash, &digest))) { - fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); - exit(1); + if (g->enable_cache) { + if ((err = cache_final(&g->cache_hash, &digest))) { + fprintf(stderr, "Unable to finalize cache hash: %s\n", err_str(err)); + exit(1); + } + os_path_join(artifact_dir, &digest, &g->artifact_dir); + } else { + Buf *tmp_basename = get_random_basename(); + os_path_join(artifact_dir, tmp_basename, &g->artifact_dir); } - os_path_join(artifact_dir, &digest, &g->artifact_dir); if ((err = os_make_path(&g->artifact_dir))) { fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err)); exit(1); @@ -7857,9 +7901,10 @@ void codegen_build_and_link(CodeGen *g) { codegen_link(g); } } - // TODO hard link output_file_path to wanted_output_file_path - cache_release(&g->cache_hash); + if (g->enable_cache) { + cache_release(&g->cache_hash); + } codegen_add_time_event(g, "Done"); } diff --git a/src/ir.cpp b/src/ir.cpp index 22d9a9bc49..bfe21f974d 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16277,7 +16277,7 @@ static ZigType *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructionImpor return ira->codegen->builtin_types.entry_namespace; } - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, resolved_path, import_code))) { + if ((err = file_fetch(ira->codegen, resolved_path, import_code))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -18108,7 +18108,7 @@ static ZigType *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstructionE // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = cache_add_file_fetch(&ira->codegen->cache_hash, &file_path, file_contents))) { + if ((err = file_fetch(ira->codegen, &file_path, file_contents))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; diff --git a/src/link.cpp b/src/link.cpp index 8d7b8b4d5f..aa0edde61b 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -58,6 +58,7 @@ static Buf *build_o_raw(CodeGen *parent_gen, const char *oname, Buf *full_path) new_link_lib->provided_explicitly = link_lib->provided_explicitly; } + child_gen->enable_cache = true; codegen_build_and_link(child_gen); return &child_gen->output_file_path; } diff --git a/src/main.cpp b/src/main.cpp index 10b789a6e1..e4019f10df 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,6 +34,7 @@ static int usage(const char *arg0) { "Compile Options:\n" " --assembly [source] add assembly file to build\n" " --cache-dir [path] override the cache directory\n" + " --cache [auto|off|on] build to the global cache and print output path to stdout\n" " --color [auto|off|on] enable or disable colored error messages\n" " --emit [asm|bin|llvm-ir] emit a specific file format as compilation output\n" " --enable-timing-info print timing diagnostics\n" @@ -257,6 +258,24 @@ static void add_package(CodeGen *g, CliPkg *cli_pkg, PackageTableEntry *pkg) { } } +enum CacheOpt { + CacheOptAuto, + CacheOptOn, + CacheOptOff, +}; + +static bool get_cache_opt(CacheOpt opt, bool default_value) { + switch (opt) { + case CacheOptAuto: + return default_value; + case CacheOptOn: + return true; + case CacheOptOff: + return false; + } + zig_unreachable(); +} + int main(int argc, char **argv) { if (argc == 2 && strcmp(argv[1], "BUILD_INFO") == 0) { printf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n", @@ -301,6 +320,7 @@ int main(int argc, char **argv) { bool verbose_llvm_ir = false; bool verbose_cimport = false; ErrColor color = ErrColorAuto; + CacheOpt enable_cache = CacheOptAuto; const char *libc_lib_dir = nullptr; const char *libc_static_lib_dir = nullptr; const char *libc_include_dir = nullptr; @@ -465,6 +485,7 @@ int main(int argc, char **argv) { PackageTableEntry *build_pkg = codegen_create_package(g, buf_ptr(&build_file_dirname), buf_ptr(&build_file_basename)); g->root_package->package_table.put(buf_create_from_str("@build"), build_pkg); + g->enable_cache = get_cache_opt(enable_cache, true); codegen_build_and_link(g); Termination term; @@ -562,6 +583,17 @@ int main(int argc, char **argv) { fprintf(stderr, "--color options are 'auto', 'on', or 'off'\n"); return usage(arg0); } + } else if (strcmp(arg, "--cache") == 0) { + if (strcmp(argv[i], "auto") == 0) { + enable_cache = CacheOptAuto; + } else if (strcmp(argv[i], "on") == 0) { + enable_cache = CacheOptOn; + } else if (strcmp(argv[i], "off") == 0) { + enable_cache = CacheOptOff; + } else { + fprintf(stderr, "--cache options are 'auto', 'on', or 'off'\n"); + return usage(arg0); + } } else if (strcmp(arg, "--emit") == 0) { if (strcmp(argv[i], "asm") == 0) { emit_file_type = EmitFileTypeAssembly; @@ -894,6 +926,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); + g->enable_cache = get_cache_opt(enable_cache, cmd == CmdRun); codegen_build_and_link(g); if (timing_info) codegen_print_timing_report(g, stdout); @@ -913,9 +946,17 @@ int main(int argc, char **argv) { Termination term; os_spawn_process(exec_path, args, &term); return term.code; + } else if (cmd == CmdBuild) { + if (g->enable_cache) { + printf("%s\n", buf_ptr(&g->output_file_path)); + if (g->out_h_path != nullptr) { + printf("%s\n", buf_ptr(g->out_h_path)); + } + } + return EXIT_SUCCESS; + } else { + zig_unreachable(); } - - return EXIT_SUCCESS; } else if (cmd == CmdTranslateC) { codegen_translate_c(g, in_file_buf); ast_render(g, stdout, g->root_import->root, 4); @@ -928,8 +969,11 @@ int main(int argc, char **argv) { ZigTarget native; get_native_target(&native); + g->enable_cache = get_cache_opt(enable_cache, false); codegen_build_and_link(g); - Buf *test_exe_path = &g->output_file_path; + Buf *test_exe_path_unresolved = &g->output_file_path; + Buf *test_exe_path = buf_alloc(); + *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1); for (size_t i = 0; i < test_exec_args.length; i += 1) { if (test_exec_args.items[i] == nullptr) { diff --git a/src/target.cpp b/src/target.cpp index 91d36c5109..f657af8af3 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -813,6 +813,22 @@ const char *target_exe_file_ext(ZigTarget *target) { } } +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch) { + if (target->os == OsWindows) { + if (is_static) { + return ".lib"; + } else { + return ".dll"; + } + } else { + if (is_static) { + return ".a"; + } else { + return buf_ptr(buf_sprintf(".so.%zu", version_major)); + } + } +} + enum FloatAbi { FloatAbiHard, FloatAbiSoft, diff --git a/src/target.hpp b/src/target.hpp index 5a118f6d8d..c8658bf352 100644 --- a/src/target.hpp +++ b/src/target.hpp @@ -113,6 +113,7 @@ const char *target_o_file_ext(ZigTarget *target); const char *target_asm_file_ext(ZigTarget *target); const char *target_llvm_ir_file_ext(ZigTarget *target); const char *target_exe_file_ext(ZigTarget *target); +const char *target_lib_file_ext(ZigTarget *target, bool is_static, size_t version_major, size_t version_minor, size_t version_patch); Buf *target_dynamic_linker(ZigTarget *target); From 15c67d2d50aae11dd425ad2629ebc9770fb95f30 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 11 Sep 2018 16:52:50 -0400 Subject: [PATCH 10/20] fix docgen tests --- doc/docgen.zig | 7 +++++++ src/codegen.cpp | 2 ++ 2 files changed, 9 insertions(+) diff --git a/doc/docgen.zig b/doc/docgen.zig index c1158dc03f..5d77a1ea14 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -11,6 +11,7 @@ const max_doc_file_size = 10 * 1024 * 1024; const exe_ext = std.build.Target(std.build.Target.Native).exeFileExt(); const obj_ext = std.build.Target(std.build.Target.Native).oFileExt(); const tmp_dir_name = "docgen_tmp"; +const test_out_path = tmp_dir_name ++ os.path.sep_str ++ "test" ++ exe_ext; pub fn main() !void { var direct_allocator = std.heap.DirectAllocator.init(); @@ -821,6 +822,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var zig_exe, "test", tmp_source_file_name, + "--output", + test_out_path, }); try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -863,6 +866,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             "--color",
                             "on",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         try out.print("
$ zig test {}.zig", code.name);
                         switch (code.mode) {
@@ -918,6 +923,8 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var
                             zig_exe,
                             "test",
                             tmp_source_file_name,
+                            "--output",
+                            test_out_path,
                         });
                         switch (code.mode) {
                             builtin.Mode.Debug => {},
diff --git a/src/codegen.cpp b/src/codegen.cpp
index d1239eb23f..af42f324b4 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8122,6 +8122,8 @@ static void resolve_out_paths(CodeGen *g) {
 
     if (g->enable_cache || g->out_type != OutTypeObj) {
         os_path_join(&g->artifact_dir, o_basename, &g->o_file_output_path);
+    } else if (g->wanted_output_file_path != nullptr && g->out_type == OutTypeObj) {
+        buf_init_from_buf(&g->o_file_output_path, g->wanted_output_file_path);
     } else {
         buf_init_from_buf(&g->o_file_output_path, o_basename);
     }

From 9227315bf24238c05c0c01a9f516449aa8525e41 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:23:36 -0400
Subject: [PATCH 11/20] zig build: better placement of test exe artifact

---
 std/build.zig | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 4e323eaf7b..0ef6109bc8 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -1641,6 +1641,7 @@ pub const TestStep = struct {
     lib_paths: ArrayList([]const u8),
     object_files: ArrayList([]const u8),
     no_rosegment: bool,
+    output_path: ?[]const u8,
 
     pub fn init(builder: *Builder, root_src: []const u8) TestStep {
         const step_name = builder.fmt("test {}", root_src);
@@ -1659,6 +1660,7 @@ pub const TestStep = struct {
             .lib_paths = ArrayList([]const u8).init(builder.allocator),
             .object_files = ArrayList([]const u8).init(builder.allocator),
             .no_rosegment = false,
+            .output_path = null,
         };
     }
 
@@ -1682,6 +1684,24 @@ pub const TestStep = struct {
         self.build_mode = mode;
     }
 
+    pub fn setOutputPath(self: *TestStep, file_path: []const u8) void {
+        self.output_path = file_path;
+
+        // catch a common mistake
+        if (mem.eql(u8, self.builder.pathFromRoot(file_path), self.builder.pathFromRoot("."))) {
+            debug.panic("setOutputPath wants a file path, not a directory\n");
+        }
+    }
+
+    pub fn getOutputPath(self: *TestStep) []const u8 {
+        if (self.output_path) |output_path| {
+            return output_path;
+        } else {
+            const basename = self.builder.fmt("test{}", self.target.exeFileExt());
+            return os.path.join(self.builder.allocator, self.builder.cache_root, basename) catch unreachable;
+        }
+    }
+
     pub fn linkSystemLibrary(self: *TestStep, name: []const u8) void {
         self.link_libs.put(name) catch unreachable;
     }
@@ -1746,6 +1766,10 @@ pub const TestStep = struct {
             builtin.Mode.ReleaseSmall => try zig_args.append("--release-small"),
         }
 
+        const output_path = builder.pathFromRoot(self.getOutputPath());
+        try zig_args.append("--output");
+        try zig_args.append(output_path);
+
         switch (self.target) {
             Target.Native => {},
             Target.Cross => |cross_target| {

From a1132ffe0ff965ab48c160a658f11bae345bf2d2 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:29:18 -0400
Subject: [PATCH 12/20] stage1: build blake code with -std=c99

---
 CMakeLists.txt                 | 13 +++++++------
 src/{blake2b.cpp => blake2b.c} |  0
 2 files changed, 7 insertions(+), 6 deletions(-)
 rename src/{blake2b.cpp => blake2b.c} (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 11bb31892c..ee9f9c2817 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -426,8 +426,8 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/util.cpp"
     "${CMAKE_SOURCE_DIR}/src/translate_c.cpp"
 )
-set(ZIG_SOURCES_O3
-    "${CMAKE_SOURCE_DIR}/src/blake2b.cpp"
+set(BLAKE_SOURCES
+    "${CMAKE_SOURCE_DIR}/src/blake2b.c"
 )
 set(ZIG_CPP_SOURCES
     "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"
@@ -795,6 +795,7 @@ else()
   set(EXE_CFLAGS "${EXE_CFLAGS} -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D_GNU_SOURCE -fno-exceptions -fno-rtti  -Werror=strict-prototypes -Werror=old-style-definition -Werror=type-limits -Wno-missing-braces")
 endif()
 
+set(BLAKE_CFLAGS "-std=c99")
 
 set(EXE_LDFLAGS " ")
 if(MINGW)
@@ -816,9 +817,9 @@ set_target_properties(zig_cpp PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
 )
 
-add_library(zig_O3 STATIC ${ZIG_SOURCES_O3})
-set_target_properties(zig_O3 PROPERTIES
-    COMPILE_FLAGS "${EXE_CFLAGS} -O3"
+add_library(embedded_blake STATIC ${BLAKE_SOURCES})
+set_target_properties(embedded_blake PROPERTIES
+    COMPILE_FLAGS "${BLAKE_CFLAGS} -O3"
 )
 
 add_executable(zig ${ZIG_SOURCES})
@@ -829,7 +830,7 @@ set_target_properties(zig PROPERTIES
 
 target_link_libraries(zig LINK_PUBLIC
     zig_cpp
-    zig_O3
+    embedded_blake
     ${SOFTFLOAT_LIBRARIES}
     ${CLANG_LIBRARIES}
     ${LLD_LIBRARIES}
diff --git a/src/blake2b.cpp b/src/blake2b.c
similarity index 100%
rename from src/blake2b.cpp
rename to src/blake2b.c

From 1a4dcf10fecf3fed7db0573fc859718e05e93f11 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 17:42:03 -0400
Subject: [PATCH 13/20] darwin fixups

---
 src/os.cpp | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/os.cpp b/src/os.cpp
index f3569ca3bb..9970c04c72 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -1657,6 +1657,14 @@ Error os_self_exe_shared_libs(ZigList &paths) {
     paths.resize(0);
     dl_iterate_phdr(self_exe_shared_libs_callback, &paths);
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    paths.resize(0);
+    uint32_t img_count = _dyld_image_count();
+    for (uint32_t i = 0; i != img_count; i += 1) {
+        const char *name = _dyld_get_image_name(i);
+        paths.append(buf_create_from_str(name));
+    }
+    return ErrorNone;
 #else
 #error "unimplemented"
 #endif
@@ -1750,7 +1758,7 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
 #error unimplemented
-#else
+#elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
         return ErrorFileSystem;
@@ -1758,6 +1766,16 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
     mtime->sec = statbuf.st_mtim.tv_sec;
     mtime->nsec = statbuf.st_mtim.tv_nsec;
     return ErrorNone;
+#elif defined(ZIG_OS_DARWIN)
+    struct stat statbuf;
+    if (fstat(file, &statbuf) == -1)
+        return ErrorFileSystem;
+
+    mtime->sec = statbuf.st_mtimespec.tv_sec;
+    mtime->nsec = statbuf.st_mtimespec.tv_nsec;
+    return ErrorNone;
+#else
+#error unimplemented
 #endif
 }
 

From 04dc5cdacaa953ca5365245f32158db70213b90e Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 18:15:08 -0400
Subject: [PATCH 14/20] zig build: make the cache root dir before building

---
 std/build.zig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/std/build.zig b/std/build.zig
index 0ef6109bc8..5800e69c62 100644
--- a/std/build.zig
+++ b/std/build.zig
@@ -232,6 +232,8 @@ pub const Builder = struct {
     }
 
     pub fn make(self: *Builder, step_names: []const []const u8) !void {
+        try self.makePath(self.cache_root);
+
         var wanted_steps = ArrayList(*Step).init(self.allocator);
         defer wanted_steps.deinit();
 

From 7e9f25dd18b7da750e547de94a779e3c14c07d94 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 20:54:39 -0400
Subject: [PATCH 15/20] stage1: clean up timing report in test mode

---
 src/codegen.cpp | 7 +++++--
 src/main.cpp    | 8 +++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/codegen.cpp b/src/codegen.cpp
index af42f324b4..96e979f9d3 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6388,9 +6388,9 @@ static void do_code_gen(CodeGen *g) {
     char *error = nullptr;
     LLVMVerifyModule(g->module, LLVMAbortProcessAction, &error);
 #endif
+}
 
-    codegen_add_time_event(g, "LLVM Emit Output");
-
+static void zig_llvm_emit_output(CodeGen *g) {
     bool is_small = g->build_mode == BuildModeSmallRelease;
 
     Buf *output_path = &g->o_file_output_path;
@@ -8228,6 +8228,9 @@ void codegen_build_and_link(CodeGen *g) {
 
         codegen_add_time_event(g, "Code Generation");
         do_code_gen(g);
+        codegen_add_time_event(g, "LLVM Emit Output");
+        zig_llvm_emit_output(g);
+
         if (g->want_h_file) {
             codegen_add_time_event(g, "Generate .h");
             gen_h_file(g);
diff --git a/src/main.cpp b/src/main.cpp
index e4019f10df..1751fabfe5 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -971,6 +971,11 @@ int main(int argc, char **argv) {
 
                 g->enable_cache = get_cache_opt(enable_cache, false);
                 codegen_build_and_link(g);
+
+                if (timing_info) {
+                    codegen_print_timing_report(g, stdout);
+                }
+
                 Buf *test_exe_path_unresolved = &g->output_file_path;
                 Buf *test_exe_path = buf_alloc();
                 *test_exe_path = os_path_resolve(&test_exe_path_unresolved, 1);
@@ -981,7 +986,6 @@ int main(int argc, char **argv) {
                     }
                 }
 
-
                 if (!target_can_exec(&native, target)) {
                     fprintf(stderr, "Created %s but skipping execution because it is non-native.\n",
                             buf_ptr(test_exe_path));
@@ -1003,8 +1007,6 @@ int main(int argc, char **argv) {
                 if (term.how != TerminationIdClean || term.code != 0) {
                     fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n");
                     fprintf(stderr, "%s\n", buf_ptr(test_exe_path));
-                } else if (timing_info) {
-                    codegen_print_timing_report(g, stdout);
                 }
                 return (term.how == TerminationIdClean) ? term.code : -1;
             } else {

From ee263a15cc8fb52c2ac6053a29168fb15089839b Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:25:52 -0400
Subject: [PATCH 16/20] bring back zig-cache

we need somewhere to put .o files and leave them while the user
executes their program, so that stack traces on MacOS can find
the .o files and get at the DWARF info.

if we try to clean up old global tmp dir files, first of all that's
a hard and complicated problem, and secondly it's not clear how
that is better than dumping the .o file inside zig-cache locally.
---
 src/all_types.hpp  |  1 +
 src/cache_hash.cpp |  8 --------
 src/cache_hash.hpp |  4 ----
 src/codegen.cpp    |  5 +----
 src/main.cpp       | 13 +++++--------
 5 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index f9c50eeaac..3dfbc6ba7f 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1682,6 +1682,7 @@ struct CodeGen {
     Buf output_file_path;
     Buf o_file_output_path;
     Buf *wanted_output_file_path;
+    Buf cache_dir;
 
     IrInstruction *invalid_instruction;
 
diff --git a/src/cache_hash.cpp b/src/cache_hash.cpp
index 65e3e62ae0..b302946310 100644
--- a/src/cache_hash.cpp
+++ b/src/cache_hash.cpp
@@ -467,11 +467,3 @@ void cache_release(CacheHash *ch) {
     assert(ch->manifest_file_path != nullptr);
     os_file_close(ch->manifest_file);
 }
-
-Buf *get_random_basename() {
-    Buf *result = buf_alloc();
-    for (size_t i = 0; i < 16; i += 1) {
-        buf_append_char(result, base64_fs_alphabet[rand() % 64]);
-    }
-    return result;
-}
diff --git a/src/cache_hash.hpp b/src/cache_hash.hpp
index 6acd805be6..db1c42ec03 100644
--- a/src/cache_hash.hpp
+++ b/src/cache_hash.hpp
@@ -68,8 +68,4 @@ Error ATTRIBUTE_MUST_USE cache_final(CacheHash *ch, Buf *out_b64_digest);
 void cache_release(CacheHash *ch);
 
 
-
-// Completely independent function. Just returns a random filename safe basename.
-Buf *get_random_basename();
-
 #endif
diff --git a/src/codegen.cpp b/src/codegen.cpp
index 96e979f9d3..aeb2b6edc4 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -8195,8 +8195,6 @@ void codegen_build_and_link(CodeGen *g) {
         }
 
         os_path_join(stage1_dir, buf_create_from_str("artifact"), artifact_dir);
-    } else {
-        os_path_join(stage1_dir, buf_create_from_str("tmp"), artifact_dir);
     }
 
     if (g->enable_cache && buf_len(&digest) != 0) {
@@ -8217,8 +8215,7 @@ void codegen_build_and_link(CodeGen *g) {
             }
             os_path_join(artifact_dir, &digest, &g->artifact_dir);
         } else {
-            Buf *tmp_basename = get_random_basename(); 
-            os_path_join(artifact_dir, tmp_basename, &g->artifact_dir);
+            buf_init_from_buf(&g->artifact_dir, &g->cache_dir);
         }
         if ((err = os_make_path(&g->artifact_dir))) {
             fprintf(stderr, "Unable to create artifact directory: %s\n", err_str(err));
diff --git a/src/main.cpp b/src/main.cpp
index 1751fabfe5..f18dd949b0 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = nullptr;
+    const char *cache_dir = default_zig_cache_name;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -410,13 +411,8 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf full_cache_dir = BUF_INIT;
-        if (cache_dir == nullptr) {
-            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
-        } else {
-            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
-        }
+        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -832,6 +828,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);

From 014cc60a72ac2d64935bf424459b5fa13e281ed9 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:46:22 -0400
Subject: [PATCH 17/20] rename --enable-timing-info to -ftime-report to match
 clang

and have it print llvm's internal timing info
---
 src/all_types.hpp | 1 +
 src/codegen.cpp   | 9 ++++++---
 src/main.cpp      | 6 ++++--
 src/zig_llvm.cpp  | 9 ++++++++-
 src/zig_llvm.h    | 3 ++-
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/all_types.hpp b/src/all_types.hpp
index 3dfbc6ba7f..8aa2718d30 100644
--- a/src/all_types.hpp
+++ b/src/all_types.hpp
@@ -1727,6 +1727,7 @@ struct CodeGen {
     bool error_during_imports;
     bool generate_error_name_table;
     bool enable_cache;
+    bool enable_time_report;
 
     //////////////////////////// Participates in Input Parameter Cache Hash
     ZigList link_libs_list;
diff --git a/src/codegen.cpp b/src/codegen.cpp
index aeb2b6edc4..6f21ceecab 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -6398,7 +6398,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
     switch (g->emit_file_type) {
         case EmitFileTypeBinary:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitBinary, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write object file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6408,7 +6409,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeAssembly:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitAssembly, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write assembly file %s: %s", buf_ptr(output_path), err_msg);
             }
@@ -6417,7 +6419,8 @@ static void zig_llvm_emit_output(CodeGen *g) {
 
         case EmitFileTypeLLVMIr:
             if (ZigLLVMTargetMachineEmitToFile(g->target_machine, g->module, buf_ptr(output_path),
-                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small))
+                        ZigLLVM_EmitLLVMIr, &err_msg, g->build_mode == BuildModeDebug, is_small,
+                        g->enable_time_report))
             {
                 zig_panic("unable to write llvm-ir file %s: %s", buf_ptr(output_path), err_msg);
             }
diff --git a/src/main.cpp b/src/main.cpp
index f18dd949b0..fdc34a83ae 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -37,7 +37,7 @@ static int usage(const char *arg0) {
         "  --cache [auto|off|on]        build to the global cache and print output path to stdout\n"
         "  --color [auto|off|on]        enable or disable colored error messages\n"
         "  --emit [asm|bin|llvm-ir]     emit a specific file format as compilation output\n"
-        "  --enable-timing-info         print timing diagnostics\n"
+        "  -ftime-report                print timing diagnostics\n"
         "  --libc-include-dir [path]    directory where libc stdlib.h resides\n"
         "  --name [name]                override output name\n"
         "  --output [file]              override destination path\n"
@@ -402,6 +402,7 @@ int main(int argc, char **argv) {
         os_path_join(special_dir, buf_create_from_str("build_runner.zig"), build_runner_path);
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
+        g->enable_time_report = timing_info;
         buf_init_from_str(&g->cache_dir, cache_dir);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
@@ -533,7 +534,7 @@ int main(int argc, char **argv) {
                 no_rosegment_workaround = true;
             } else if (strcmp(arg, "--each-lib-rpath") == 0) {
                 each_lib_rpath = true;
-            } else if (strcmp(arg, "--enable-timing-info") == 0) {
+            } else if (strcmp(arg, "-ftime-report") == 0) {
                 timing_info = true;
             } else if (strcmp(arg, "--test-cmd-bin") == 0) {
                 test_exec_args.append(nullptr);
@@ -828,6 +829,7 @@ int main(int argc, char **argv) {
             Buf *zig_lib_dir_buf = resolve_zig_lib_dir();
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
+            g->enable_time_report = timing_info;
             buf_init_from_str(&g->cache_dir, cache_dir);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp
index a43d2d182c..61287f620c 100644
--- a/src/zig_llvm.cpp
+++ b/src/zig_llvm.cpp
@@ -30,6 +30,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -81,8 +82,11 @@ static const bool assertions_on = false;
 #endif
 
 bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small)
+        const char *filename, ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report)
 {
+    TimePassesIsEnabled = time_report;
+
     std::error_code EC;
     raw_fd_ostream dest(filename, EC, sys::fs::F_None);
     if (EC) {
@@ -182,6 +186,9 @@ bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMM
         }
     }
 
+    if (time_report) {
+        TimerGroup::printAll(errs());
+    }
     return false;
 }
 
diff --git a/src/zig_llvm.h b/src/zig_llvm.h
index 63d69bd23e..5cdc6cc041 100644
--- a/src/zig_llvm.h
+++ b/src/zig_llvm.h
@@ -55,7 +55,8 @@ enum ZigLLVM_EmitOutputType {
 };
 
 ZIG_EXTERN_C bool ZigLLVMTargetMachineEmitToFile(LLVMTargetMachineRef targ_machine_ref, LLVMModuleRef module_ref,
-        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug, bool is_small);
+        const char *filename, enum ZigLLVM_EmitOutputType output_type, char **error_message, bool is_debug,
+        bool is_small, bool time_report);
 
 ZIG_EXTERN_C LLVMTypeRef ZigLLVMTokenTypeInContext(LLVMContextRef context_ref);
 

From ff0b7fe29a49a5619b3c2f7fee0baccb979caf2d Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Tue, 11 Sep 2018 22:57:39 -0400
Subject: [PATCH 18/20] error messages for attempted cache when zig cannot
 perfectly do it

---
 src/analyze.cpp | 6 ++++++
 src/ir.cpp      | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/src/analyze.cpp b/src/analyze.cpp
index bae4b15f81..59be76518b 100644
--- a/src/analyze.cpp
+++ b/src/analyze.cpp
@@ -6289,6 +6289,12 @@ LinkLib *add_link_lib(CodeGen *g, Buf *name) {
     if (is_libc && g->libc_link_lib != nullptr)
         return g->libc_link_lib;
 
+    if (g->enable_cache && is_libc && g->zig_target.os != OsMacOSX && g->zig_target.os != OsIOS) {
+        fprintf(stderr, "TODO linking against libc is currently incompatible with `--cache on`.\n"
+        "Zig is not yet capable of determining whether the libc installation has changed on subsequent builds.\n");
+        exit(1);
+    }
+
     for (size_t i = 0; i < g->link_libs_list.length; i += 1) {
         LinkLib *existing_lib = g->link_libs_list.at(i);
         if (buf_eql_buf(existing_lib->name, name)) {
diff --git a/src/ir.cpp b/src/ir.cpp
index 858a6df33c..030f6627ad 100644
--- a/src/ir.cpp
+++ b/src/ir.cpp
@@ -17961,6 +17961,12 @@ static ZigType *ir_analyze_instruction_type_name(IrAnalyze *ira, IrInstructionTy
 }
 
 static ZigType *ir_analyze_instruction_c_import(IrAnalyze *ira, IrInstructionCImport *instruction) {
+    if (ira->codegen->enable_cache) {
+        ir_add_error(ira, &instruction->base,
+            buf_sprintf("TODO @cImport is incompatible with --cache on. The cache system currently is unable to detect subsequent changes in .h files."));
+        return ira->codegen->builtin_types.entry_invalid;
+    }
+
     AstNode *node = instruction->base.source_node;
     assert(node->type == NodeTypeFnCallExpr);
     AstNode *block_node = node->data.fn_call_expr.params.at(0);

From 1caa48c2df63d61b8d2501b3e3ee7c85ee8d89cb Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:33:26 -0400
Subject: [PATCH 19/20] windows os.cpp implementations

---
 src/error.cpp |   2 +
 src/error.hpp |   2 +
 src/os.cpp    | 207 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/os.hpp    |   2 +-
 4 files changed, 203 insertions(+), 10 deletions(-)

diff --git a/src/error.cpp b/src/error.cpp
index d503eaa18b..7a9bd963bb 100644
--- a/src/error.cpp
+++ b/src/error.cpp
@@ -30,6 +30,8 @@ const char *err_str(int err) {
         case ErrorEndOfFile: return "end of file";
         case ErrorIsDir: return "is directory";
         case ErrorUnsupportedOperatingSystem: return "unsupported operating system";
+        case ErrorSharingViolation: return "sharing violation";
+        case ErrorPipeBusy: return "pipe busy";
     }
     return "(invalid error)";
 }
diff --git a/src/error.hpp b/src/error.hpp
index 8c1f8b9aa5..0996a41d58 100644
--- a/src/error.hpp
+++ b/src/error.hpp
@@ -30,6 +30,8 @@ enum Error {
     ErrorEndOfFile,
     ErrorIsDir,
     ErrorUnsupportedOperatingSystem,
+    ErrorSharingViolation,
+    ErrorPipeBusy,
 };
 
 const char *err_str(int err);
diff --git a/src/os.cpp b/src/os.cpp
index 9970c04c72..6e46b96e41 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -24,6 +24,7 @@
 #endif
 
 #include 
+#include 
 #include 
 #include 
 
@@ -1393,7 +1394,7 @@ Error os_self_exe_path(Buf *out_path) {
         }
         if (copied_amt < buf_len(out_path)) {
             buf_resize(out_path, copied_amt);
-            return 0;
+            return ErrorNone;
         }
         buf_resize(out_path, buf_len(out_path) * 2);
     }
@@ -1616,10 +1617,118 @@ int os_get_win32_kern32_path(ZigWindowsSDK *sdk, Buf* output_buf, ZigLLVM_ArchTy
 #endif
 }
 
-// Ported from std/os/get_app_data_dir.zig
+#if defined(ZIG_OS_WINDOWS)
+// Ported from std/unicode.zig
+struct Utf16LeIterator {
+    uint8_t *bytes;
+    size_t i;
+};
+
+// Ported from std/unicode.zig
+static Utf16LeIterator Utf16LeIterator_init(WCHAR *ptr) {
+    return {(uint8_t*)ptr, 0};
+}
+
+// Ported from std/unicode.zig
+static Optional Utf16LeIterator_nextCodepoint(Utf16LeIterator *it) {
+    if (it->bytes[it->i] == 0 && it->bytes[it->i + 1] == 0)
+        return {};
+    uint32_t c0 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+    if (c0 & ~((uint32_t)0x03ff) == 0xd800) {
+        // surrogate pair
+        it->i += 2;
+        assert(it->bytes[it->i] != 0 || it->bytes[it->i + 1] != 0);
+        uint32_t c1 = ((uint32_t)it->bytes[it->i]) | (((uint32_t)it->bytes[it->i + 1]) << 8);
+        assert(c1 & ~((uint32_t)0x03ff) == 0xdc00);
+        it->i += 2;
+        return Optional::some(0x10000 + (((c0 & 0x03ff) << 10) | (c1 & 0x03ff)));
+    } else {
+        assert(c0 & ~((uint32_t)0x03ff) != 0xdc00);
+        it->i += 2;
+        return Optional::some(c0);
+    }
+}
+
+// Ported from std/unicode.zig
+static uint8_t utf8CodepointSequenceLength(uint32_t c) {
+    if (c < 0x80) return 1;
+    if (c < 0x800) return 2;
+    if (c < 0x10000) return 3;
+    if (c < 0x110000) return 4;
+    zig_unreachable();
+}
+
+// Ported from std/unicode.zig
+static size_t utf8Encode(uint32_t c, Slice out) {
+    size_t length = utf8CodepointSequenceLength(c);
+    assert(out.len >= length);
+    switch (length) {
+        // The pattern for each is the same
+        // - Increasing the initial shift by 6 each time
+        // - Each time after the first shorten the shifted
+        //   value to a max of 0b111111 (63)
+        case 1:
+            out.ptr[0] = c; // Can just do 0 + codepoint for initial range
+            break;
+        case 2:
+            out.ptr[0] = 0b11000000 | (c >> 6);
+            out.ptr[1] = 0b10000000 | (c & 0b111111);
+            break;
+        case 3:
+            assert(!(0xd800 <= c && c <= 0xdfff));
+            out.ptr[0] = 0b11100000 | (c >> 12);
+            out.ptr[1] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[2] = 0b10000000 | (c & 0b111111);
+            break;
+        case 4:
+            out.ptr[0] = 0b11110000 | (c >> 18);
+            out.ptr[1] = 0b10000000 | ((c >> 12) & 0b111111);
+            out.ptr[2] = 0b10000000 | ((c >> 6) & 0b111111);
+            out.ptr[3] = 0b10000000 | (c & 0b111111);
+            break;
+        default:
+            zig_unreachable();
+    }
+    return length;
+}
+
+// Ported from std.unicode.utf16leToUtf8Alloc
+static void utf16le_ptr_to_utf8(Buf *out, WCHAR *utf16le) {
+    // optimistically guess that it will all be ascii.
+    buf_resize(out, 0);
+    size_t out_index = 0;
+    Utf16LeIterator it = Utf16LeIterator_init(utf16le);
+    for (;;) {
+        Optional opt_codepoint = Utf16LeIterator_nextCodepoint(&it);
+        if (!opt_codepoint.is_some) break;
+        uint32_t codepoint = opt_codepoint.value;
+
+        size_t utf8_len = utf8CodepointSequenceLength(codepoint);
+        buf_resize(out, buf_len(out) + utf8_len);
+        utf8Encode(codepoint, {(uint8_t*)buf_ptr(out)+out_index, buf_len(out)-out_index});
+        out_index += utf8_len;
+    }
+}
+#endif
+
+// Ported from std.os.getAppDataDir
 Error os_get_app_data_dir(Buf *out_path, const char *appname) {
 #if defined(ZIG_OS_WINDOWS)
-#error "Unimplemented"
+    Error err;
+    WCHAR *dir_path_ptr;
+    switch (SHGetKnownFolderPath(FOLDERID_LocalAppData, KF_FLAG_CREATE, nullptr, &dir_path_ptr)) {
+        case S_OK:
+            // defer os.windows.CoTaskMemFree(@ptrCast(*c_void, dir_path_ptr));
+            utf16le_ptr_to_utf8(out_path, dir_path_ptr);
+            CoTaskMemFree(dir_path_ptr);
+            buf_appendf(out_path, "\\%s", appname);
+            return ErrorNone;
+        case E_OUTOFMEMORY:
+            return ErrorNoMem;
+        default:
+            return ErrorUnexpected;
+    }
+    zig_unreachable();
 #elif defined(ZIG_OS_DARWIN)
     const char *home_dir = getenv("HOME");
     if (home_dir == nullptr) {
@@ -1665,14 +1774,44 @@ Error os_self_exe_shared_libs(ZigList &paths) {
         paths.append(buf_create_from_str(name));
     }
     return ErrorNone;
+#elif defined(ZIG_OS_WINDOWS)
+    // zig is built statically on windows, so we can return an empty list
+    paths.resize(0);
+    return ErrorNone;
 #else
-#error "unimplemented"
+#error unimplemented
 #endif
 }
 
 Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    // TODO use CreateFileW
+    HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+    if (result == INVALID_HANDLE_VALUE) {
+        DWORD err = GetLastError();
+        switch (err) {
+            case ERROR_SHARING_VIOLATION:
+                return ErrorSharingViolation;
+            case ERROR_ALREADY_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_EXISTS:
+                return ErrorPathAlreadyExists;
+            case ERROR_FILE_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_PATH_NOT_FOUND:
+                return ErrorFileNotFound;
+            case ERROR_ACCESS_DENIED:
+                return ErrorAccess;
+            case ERROR_PIPE_BUSY:
+                return ErrorPipeBusy;
+            default:
+                return ErrorUnexpected;
+        }
+    }
+
+    *out_file = result;
+    return ErrorNone;
 #else
     for (;;) {
         int fd = open(buf_ptr(full_path), O_RDONLY|O_CLOEXEC);
@@ -1702,7 +1841,36 @@ Error os_file_open_r(Buf *full_path, OsFile *out_file) {
 
 Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 #if defined(ZIG_OS_WINDOWS)
-#error "unimplemented"
+    for (;;) {
+        HANDLE result = CreateFileA(buf_ptr(full_path), GENERIC_READ | GENERIC_WRITE,
+            0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
+
+        if (result == INVALID_HANDLE_VALUE) {
+            DWORD err = GetLastError();
+            switch (err) {
+                case ERROR_SHARING_VIOLATION:
+                    // TODO wait for the lock instead of sleeping
+                    Sleep(10);
+                    continue;
+                case ERROR_ALREADY_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_EXISTS:
+                    return ErrorPathAlreadyExists;
+                case ERROR_FILE_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_PATH_NOT_FOUND:
+                    return ErrorFileNotFound;
+                case ERROR_ACCESS_DENIED:
+                    return ErrorAccess;
+                case ERROR_PIPE_BUSY:
+                    return ErrorPipeBusy;
+                default:
+                    return ErrorUnexpected;
+            }
+        }
+        *out_file = result;
+        return ErrorNone;
+    }
 #else
     int fd;
     for (;;) {
@@ -1757,7 +1925,12 @@ Error os_file_open_lock_rw(Buf *full_path, OsFile *out_file) {
 
 Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    FILETIME last_write_time;
+    if (!GetFileTime(file, nullptr, nullptr, &last_write_time))
+        return ErrorUnexpected;
+    mtime->sec = last_write_time.dwLowDateTime | (last_write_time.dwHighDateTime << 32);
+    mtime->nsec = 0;
+    return ErrorNone;
 #elif defined(ZIG_OS_LINUX)
     struct stat statbuf;
     if (fstat(file, &statbuf) == -1)
@@ -1781,7 +1954,11 @@ Error os_file_mtime(OsFile file, OsTimeStamp *mtime) {
 
 Error os_file_read(OsFile file, void *ptr, size_t *len) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    DWORD amt_read;
+    if (ReadFile(file, ptr, *len, &amt_read, nullptr) == 0)
+        return ErrorUnexpected;
+    *len = amt_read;
+    return ErrorNone;
 #else
     for (;;) {
         ssize_t rc = read(file, ptr, *len);
@@ -1830,10 +2007,18 @@ Error os_file_read_all(OsFile file, Buf *contents) {
 
 Error os_file_overwrite(OsFile file, Buf *contents) {
 #if defined(ZIG_OS_WINDOWS)
-#error unimplemented
+    if (SetFilePointer(file, 0, nullptr, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
+        return ErrorFileSystem;
+    if (!SetEndOfFile(file))
+        return ErrorFileSystem;
+    if (!WriteFile(file, buf_ptr(contents), buf_len(contents), nullptr, nullptr))
+        return ErrorFileSystem;
+    return ErrorNone;
 #else
     if (lseek(file, 0, SEEK_SET) == -1)
         return ErrorFileSystem;
+    if (ftruncate(file, 0) == -1)
+        return ErrorFileSystem;
     for (;;) {
         if (write(file, buf_ptr(contents), buf_len(contents)) == -1) {
             switch (errno) {
@@ -1853,5 +2038,9 @@ Error os_file_overwrite(OsFile file, Buf *contents) {
 }
 
 void os_file_close(OsFile file) {
+#if defined(ZIG_OS_WINDOWS)
+    CloseHandle(file);
+#else
     close(file);
+#endif
 }
diff --git a/src/os.hpp b/src/os.hpp
index ac422fbd21..1054bf24a7 100644
--- a/src/os.hpp
+++ b/src/os.hpp
@@ -72,7 +72,7 @@ struct Termination {
 };
 
 #if defined(ZIG_OS_WINDOWS)
-#define OsFile (void *)
+#define OsFile void *
 #else
 #define OsFile int
 #endif

From 3a49d115cf38154f0094d9615334529890059006 Mon Sep 17 00:00:00 2001
From: Andrew Kelley 
Date: Wed, 12 Sep 2018 11:49:46 -0400
Subject: [PATCH 20/20] fix zig build cache dir path

---
 build.zig    |  3 ++-
 src/main.cpp | 16 +++++++++++-----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/build.zig b/build.zig
index d4a159e5e6..e886377139 100644
--- a/build.zig
+++ b/build.zig
@@ -16,11 +16,12 @@ pub fn build(b: *Builder) !void {
     var docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
 
     const rel_zig_exe = try os.path.relative(b.allocator, b.build_root, b.zig_exe);
+    const langref_out_path = os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable;
     var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8{
         docgen_exe.getOutputPath(),
         rel_zig_exe,
         "doc" ++ os.path.sep_str ++ "langref.html.in",
-        os.path.join(b.allocator, b.cache_root, "langref.html") catch unreachable,
+        langref_out_path,
     });
     docgen_cmd.step.dependOn(&docgen_exe.step);
 
diff --git a/src/main.cpp b/src/main.cpp
index fdc34a83ae..bf6c142975 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -353,7 +353,7 @@ int main(int argc, char **argv) {
     size_t ver_minor = 0;
     size_t ver_patch = 0;
     bool timing_info = false;
-    const char *cache_dir = default_zig_cache_name;
+    const char *cache_dir = nullptr;
     CliPkg *cur_pkg = allocate(1);
     BuildMode build_mode = BuildModeDebug;
     ZigList test_exec_args = {0};
@@ -403,7 +403,7 @@ int main(int argc, char **argv) {
 
         CodeGen *g = codegen_create(build_runner_path, nullptr, OutTypeExe, BuildModeDebug, zig_lib_dir_buf);
         g->enable_time_report = timing_info;
-        buf_init_from_str(&g->cache_dir, cache_dir);
+        buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
         codegen_set_out_name(g, buf_create_from_str("build"));
 
         Buf *build_file_buf = buf_create_from_str(build_file);
@@ -412,8 +412,14 @@ int main(int argc, char **argv) {
         Buf build_file_dirname = BUF_INIT;
         os_path_split(&build_file_abs, &build_file_dirname, &build_file_basename);
 
-        Buf *cache_dir_buf = buf_create_from_str(cache_dir);
-        Buf full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+
+        Buf full_cache_dir = BUF_INIT;
+        if (cache_dir == nullptr) {
+            os_path_join(&build_file_dirname, buf_create_from_str(default_zig_cache_name), &full_cache_dir);
+        } else {
+            Buf *cache_dir_buf = buf_create_from_str(cache_dir);
+            full_cache_dir = os_path_resolve(&cache_dir_buf, 1);
+        }
 
         args.items[1] = buf_ptr(&build_file_dirname);
         args.items[2] = buf_ptr(&full_cache_dir);
@@ -830,7 +836,7 @@ int main(int argc, char **argv) {
 
             CodeGen *g = codegen_create(zig_root_source_file, target, out_type, build_mode, zig_lib_dir_buf);
             g->enable_time_report = timing_info;
-            buf_init_from_str(&g->cache_dir, cache_dir);
+            buf_init_from_str(&g->cache_dir, cache_dir ? cache_dir : default_zig_cache_name);
             codegen_set_out_name(g, buf_out_name);
             codegen_set_lib_version(g, ver_major, ver_minor, ver_patch);
             codegen_set_is_test(g, cmd == CmdTest);