started work on asset packer

This commit is contained in:
Matthew 2025-04-06 20:34:58 +10:00
parent ff07cbc572
commit 43d89bee84
33 changed files with 59866 additions and 96 deletions

BIN
assets/patamon.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

BIN
assets/pattermon.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 340 KiB

BIN
assets/purplemon.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 204 KiB

View File

@ -19,12 +19,15 @@ if [ ! -v render_flag ]; then render_flag="-DSTG_VULKAN_RENDERER"; vulka
# source files # source files
source_files="../src/entry_linux.c" source_files="../src/entry_linux.c"
packer_source_files="../src/packer.c"
# includes # includes
include_flags="-I../src/ -I../external/ -L. -I../external/dxvk/include/ -L../external/dxvk/lib/" include_flags="-I../src/ -I../external/ -L. -I../external/dxvk/include/ -L../external/dxvk/lib/"
packer_include_flags="-I../src/ -I../external/"
# executable name # executable name
out_name="Gears" out_name="Gears"
packer_out_name="Packer"
# vma flags # vma flags
vma_source_files="../external/vma/vma.cpp" vma_source_files="../external/vma/vma.cpp"
@ -76,6 +79,8 @@ mkdir -p build
cd build cd build
if [ ! -v packer ]; then
mkdir -p ../src/file_data mkdir -p ../src/file_data
if [ -v vulkan ]; then if [ -v vulkan ]; then
@ -124,4 +129,6 @@ ar rcs libvma.a vma.o
$compile $source_files $compile_link $link_os_gfx $out $out_name $compile $source_files $compile_link $link_os_gfx $out $out_name
elif [ -v packer ]; then
$compile $packer_source_files $compile_link $link_os_gfx $packer_include_flags $out $packer_out_name
fi

551
external/fastlz/fastlz.c vendored Normal file
View File

@ -0,0 +1,551 @@
/*
FastLZ - lightning-fast lossless compression library
Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#if !defined(FASTLZ__COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR)
/*
* Always check for bound when decompressing.
* Generally it is best to leave it defined.
*/
#define FASTLZ_SAFE
/*
* Give hints to the compiler for branch prediction optimization.
*/
#if defined(__GNUC__) && (__GNUC__ > 2)
#define FASTLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1))
#define FASTLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0))
#else
#define FASTLZ_EXPECT_CONDITIONAL(c) (c)
#define FASTLZ_UNEXPECT_CONDITIONAL(c) (c)
#endif
/*
* Use inlined functions for supported systems.
*/
#if defined(__GNUC__) || defined(__DMC__) || defined(__POCC__) || defined(__WATCOMC__) || defined(__SUNPRO_C)
#define FASTLZ_INLINE inline
#elif defined(__BORLANDC__) || defined(_MSC_VER) || defined(__LCC__)
#define FASTLZ_INLINE __inline
#else
#define FASTLZ_INLINE
#endif
/*
* Prevent accessing more than 8-bit at once, except on x86 architectures.
*/
#if !defined(FASTLZ_STRICT_ALIGN)
#define FASTLZ_STRICT_ALIGN
#if defined(__i386__) || defined(__386) /* GNU C, Sun Studio */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */
#undef FASTLZ_STRICT_ALIGN
#elif defined(_M_IX86) /* Intel, MSVC */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__386)
#undef FASTLZ_STRICT_ALIGN
#elif defined(_X86_) /* MinGW */
#undef FASTLZ_STRICT_ALIGN
#elif defined(__I86__) /* Digital Mars */
#undef FASTLZ_STRICT_ALIGN
#endif
#endif
/*
* FIXME: use preprocessor magic to set this on different platforms!
*/
typedef unsigned char flzuint8;
typedef unsigned short flzuint16;
typedef unsigned int flzuint32;
/* prototypes */
int fastlz_compress(const void* input, int length, void* output);
int fastlz_compress_level(int level, const void* input, int length, void* output);
int fastlz_decompress(const void* input, int length, void* output, int maxout);
#define MAX_COPY 32
#define MAX_LEN 264 /* 256 + 8 */
#define MAX_DISTANCE 8192
#if !defined(FASTLZ_STRICT_ALIGN)
#define FASTLZ_READU16(p) *((const flzuint16*)(p))
#else
#define FASTLZ_READU16(p) ((p)[0] | (p)[1]<<8)
#endif
#define HASH_LOG 13
#define HASH_SIZE (1<< HASH_LOG)
#define HASH_MASK (HASH_SIZE-1)
#define HASH_FUNCTION(v,p) { v = FASTLZ_READU16(p); v ^= FASTLZ_READU16(p+1)^(v>>(16-HASH_LOG));v &= HASH_MASK; }
#undef FASTLZ_LEVEL
#define FASTLZ_LEVEL 1
#undef FASTLZ_COMPRESSOR
#undef FASTLZ_DECOMPRESSOR
#define FASTLZ_COMPRESSOR fastlz1_compress
#define FASTLZ_DECOMPRESSOR fastlz1_decompress
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output);
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout);
#include "fastlz.c"
#undef FASTLZ_LEVEL
#define FASTLZ_LEVEL 2
#undef MAX_DISTANCE
#define MAX_DISTANCE 8191
#define MAX_FARDISTANCE (65535+MAX_DISTANCE-1)
#undef FASTLZ_COMPRESSOR
#undef FASTLZ_DECOMPRESSOR
#define FASTLZ_COMPRESSOR fastlz2_compress
#define FASTLZ_DECOMPRESSOR fastlz2_decompress
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output);
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout);
#include "fastlz.c"
int fastlz_compress(const void* input, int length, void* output)
{
/* for short block, choose fastlz1 */
if(length < 65536)
return fastlz1_compress(input, length, output);
/* else... */
return fastlz2_compress(input, length, output);
}
int fastlz_decompress(const void* input, int length, void* output, int maxout)
{
/* magic identifier for compression level */
int level = ((*(const flzuint8*)input) >> 5) + 1;
if(level == 1)
return fastlz1_decompress(input, length, output, maxout);
if(level == 2)
return fastlz2_decompress(input, length, output, maxout);
/* unknown level, trigger error */
return 0;
}
int fastlz_compress_level(int level, const void* input, int length, void* output)
{
if(level == 1)
return fastlz1_compress(input, length, output);
if(level == 2)
return fastlz2_compress(input, length, output);
return 0;
}
#else /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */
static FASTLZ_INLINE int FASTLZ_COMPRESSOR(const void* input, int length, void* output)
{
const flzuint8* ip = (const flzuint8*) input;
const flzuint8* ip_bound = ip + length - 2;
const flzuint8* ip_limit = ip + length - 12;
flzuint8* op = (flzuint8*) output;
const flzuint8* htab[HASH_SIZE];
const flzuint8** hslot;
flzuint32 hval;
flzuint32 copy;
/* sanity check */
if(FASTLZ_UNEXPECT_CONDITIONAL(length < 4))
{
if(length)
{
/* create literal copy only */
*op++ = length-1;
ip_bound++;
while(ip <= ip_bound)
*op++ = *ip++;
return length+1;
}
else
return 0;
}
/* initializes hash table */
for (hslot = htab; hslot < htab + HASH_SIZE; hslot++)
*hslot = ip;
/* we start with literal copy */
copy = 2;
*op++ = MAX_COPY-1;
*op++ = *ip++;
*op++ = *ip++;
/* main loop */
while(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
{
const flzuint8* ref;
flzuint32 distance;
/* minimum match length */
flzuint32 len = 3;
/* comparison starting-point */
const flzuint8* anchor = ip;
/* check for a run */
#if FASTLZ_LEVEL==2
if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1))
{
distance = 1;
ip += 3;
ref = anchor - 1 + 3;
goto match;
}
#endif
/* find potential match */
HASH_FUNCTION(hval,ip);
hslot = htab + hval;
ref = htab[hval];
/* calculate distance to the match */
distance = anchor - ref;
/* update hash table */
*hslot = anchor;
/* is this a match? check the first 3 bytes */
if(distance==0 ||
#if FASTLZ_LEVEL==1
(distance >= MAX_DISTANCE) ||
#else
(distance >= MAX_FARDISTANCE) ||
#endif
*ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++)
goto literal;
#if FASTLZ_LEVEL==2
/* far, needs at least 5-byte match */
if(distance >= MAX_DISTANCE)
{
if(*ip++ != *ref++ || *ip++!= *ref++)
goto literal;
len += 2;
}
match:
#endif
/* last matched byte */
ip = anchor + len;
/* distance is biased */
distance--;
if(!distance)
{
/* zero distance means a run */
flzuint8 x = ip[-1];
while(ip < ip_bound)
if(*ref++ != x) break; else ip++;
}
else
for(;;)
{
/* safe because the outer check against ip limit */
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
if(*ref++ != *ip++) break;
while(ip < ip_bound)
if(*ref++ != *ip++) break;
break;
}
/* if we have copied something, adjust the copy count */
if(copy)
/* copy is biased, '0' means 1 byte copy */
*(op-copy-1) = copy-1;
else
/* back, to overwrite the copy count */
op--;
/* reset literal counter */
copy = 0;
/* length is biased, '1' means a match of 3 bytes */
ip -= 3;
len = ip - anchor;
/* encode the match */
#if FASTLZ_LEVEL==2
if(distance < MAX_DISTANCE)
{
if(len < 7)
{
*op++ = (len << 5) + (distance >> 8);
*op++ = (distance & 255);
}
else
{
*op++ = (7 << 5) + (distance >> 8);
for(len-=7; len >= 255; len-= 255)
*op++ = 255;
*op++ = len;
*op++ = (distance & 255);
}
}
else
{
/* far away, but not yet in the another galaxy... */
if(len < 7)
{
distance -= MAX_DISTANCE;
*op++ = (len << 5) + 31;
*op++ = 255;
*op++ = distance >> 8;
*op++ = distance & 255;
}
else
{
distance -= MAX_DISTANCE;
*op++ = (7 << 5) + 31;
for(len-=7; len >= 255; len-= 255)
*op++ = 255;
*op++ = len;
*op++ = 255;
*op++ = distance >> 8;
*op++ = distance & 255;
}
}
#else
if(FASTLZ_UNEXPECT_CONDITIONAL(len > MAX_LEN-2))
while(len > MAX_LEN-2)
{
*op++ = (7 << 5) + (distance >> 8);
*op++ = MAX_LEN - 2 - 7 -2;
*op++ = (distance & 255);
len -= MAX_LEN-2;
}
if(len < 7)
{
*op++ = (len << 5) + (distance >> 8);
*op++ = (distance & 255);
}
else
{
*op++ = (7 << 5) + (distance >> 8);
*op++ = len - 7;
*op++ = (distance & 255);
}
#endif
/* update the hash at match boundary */
HASH_FUNCTION(hval,ip);
htab[hval] = ip++;
HASH_FUNCTION(hval,ip);
htab[hval] = ip++;
/* assuming literal copy */
*op++ = MAX_COPY-1;
continue;
literal:
*op++ = *anchor++;
ip = anchor;
copy++;
if(FASTLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY))
{
copy = 0;
*op++ = MAX_COPY-1;
}
}
/* left-over as literal copy */
ip_bound++;
while(ip <= ip_bound)
{
*op++ = *ip++;
copy++;
if(copy == MAX_COPY)
{
copy = 0;
*op++ = MAX_COPY-1;
}
}
/* if we have copied something, adjust the copy length */
if(copy)
*(op-copy-1) = copy-1;
else
op--;
#if FASTLZ_LEVEL==2
/* marker for fastlz2 */
*(flzuint8*)output |= (1 << 5);
#endif
return op - (flzuint8*)output;
}
static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void* output, int maxout)
{
const flzuint8* ip = (const flzuint8*) input;
const flzuint8* ip_limit = ip + length;
flzuint8* op = (flzuint8*) output;
flzuint8* op_limit = op + maxout;
flzuint32 ctrl = (*ip++) & 31;
int loop = 1;
do
{
const flzuint8* ref = op;
flzuint32 len = ctrl >> 5;
flzuint32 ofs = (ctrl & 31) << 8;
if(ctrl >= 32)
{
#if FASTLZ_LEVEL==2
flzuint8 code;
#endif
len--;
ref -= ofs;
if (len == 7-1)
#if FASTLZ_LEVEL==1
len += *ip++;
ref -= *ip++;
#else
do
{
code = *ip++;
len += code;
} while (code==255);
code = *ip++;
ref -= code;
/* match from 16-bit distance */
if(FASTLZ_UNEXPECT_CONDITIONAL(code==255))
if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8)))
{
ofs = (*ip++) << 8;
ofs += *ip++;
ref = op - ofs - MAX_DISTANCE;
}
#endif
#ifdef FASTLZ_SAFE
if (FASTLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit))
return 0;
if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output))
return 0;
#endif
if(FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit))
ctrl = *ip++;
else
loop = 0;
if(ref == op)
{
/* optimize copy for a run */
flzuint8 b = ref[-1];
*op++ = b;
*op++ = b;
*op++ = b;
for(; len; --len)
*op++ = b;
}
else
{
#if !defined(FASTLZ_STRICT_ALIGN)
const flzuint16* p;
flzuint16* q;
#endif
/* copy from reference */
ref--;
*op++ = *ref++;
*op++ = *ref++;
*op++ = *ref++;
#if !defined(FASTLZ_STRICT_ALIGN)
/* copy a byte, so that now it's word aligned */
if(len & 1)
{
*op++ = *ref++;
len--;
}
/* copy 16-bit at once */
q = (flzuint16*) op;
op += len;
p = (const flzuint16*) ref;
for(len>>=1; len > 4; len-=4)
{
*q++ = *p++;
*q++ = *p++;
*q++ = *p++;
*q++ = *p++;
}
for(; len; --len)
*q++ = *p++;
#else
for(; len; --len)
*op++ = *ref++;
#endif
}
}
else
{
ctrl++;
#ifdef FASTLZ_SAFE
if (FASTLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit))
return 0;
if (FASTLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit))
return 0;
#endif
*op++ = *ip++;
for(--ctrl; ctrl; ctrl--)
*op++ = *ip++;
loop = FASTLZ_EXPECT_CONDITIONAL(ip < ip_limit);
if(loop)
ctrl = *ip++;
}
}
while(FASTLZ_EXPECT_CONDITIONAL(loop));
return op - (flzuint8*)output;
}
#endif /* !defined(FASTLZ_COMPRESSOR) && !defined(FASTLZ_DECOMPRESSOR) */

100
external/fastlz/fastlz.h vendored Normal file
View File

@ -0,0 +1,100 @@
/*
FastLZ - lightning-fast lossless compression library
Copyright (C) 2007 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2006 Ariya Hidayat (ariya@kde.org)
Copyright (C) 2005 Ariya Hidayat (ariya@kde.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef FASTLZ_H
#define FASTLZ_H
#define FASTLZ_VERSION 0x000100
#define FASTLZ_VERSION_MAJOR 0
#define FASTLZ_VERSION_MINOR 0
#define FASTLZ_VERSION_REVISION 0
#define FASTLZ_VERSION_STRING "0.1.0"
#if defined (__cplusplus)
extern "C" {
#endif
/**
Compress a block of data in the input buffer and returns the size of
compressed block. The size of input buffer is specified by length. The
minimum input buffer size is 16.
The output buffer must be at least 5% larger than the input buffer
and can not be smaller than 66 bytes.
If the input is not compressible, the return value might be larger than
length (input buffer size).
The input buffer and the output buffer can not overlap.
*/
int fastlz_compress(const void* input, int length, void* output);
/**
Decompress a block of compressed data and returns the size of the
decompressed block. If error occurs, e.g. the compressed data is
corrupted or the output buffer is not large enough, then 0 (zero)
will be returned instead.
The input buffer and the output buffer can not overlap.
Decompression is memory safe and guaranteed not to write the output buffer
more than what is specified in maxout.
*/
int fastlz_decompress(const void* input, int length, void* output, int maxout);
/**
Compress a block of data in the input buffer and returns the size of
compressed block. The size of input buffer is specified by length. The
minimum input buffer size is 16.
The output buffer must be at least 5% larger than the input buffer
and can not be smaller than 66 bytes.
If the input is not compressible, the return value might be larger than
length (input buffer size).
The input buffer and the output buffer can not overlap.
Compression level can be specified in parameter level. At the moment,
only level 1 and level 2 are supported.
Level 1 is the fastest compression and generally useful for short data.
Level 2 is slightly slower but it gives better compression ratio.
Note that the compressed data, regardless of the level, can always be
decompressed using the function fastlz_decompress above.
*/
int fastlz_compress_level(int level, const void* input, int length, void* output);
#if defined (__cplusplus)
}
#endif
#endif /* FASTLZ_H */

26
external/xxhash/LICENSE vendored Normal file
View File

@ -0,0 +1,26 @@
xxHash Library
Copyright (c) 2012-2021 Yann Collet
All rights reserved.
BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

274
external/xxhash/README.md vendored Normal file
View File

@ -0,0 +1,274 @@
xxHash - Extremely fast hash algorithm
======================================
xxHash is an Extremely fast Hash algorithm, processing at RAM speed limits.
Code is highly portable, and produces hashes identical across all platforms (little / big endian).
The library includes the following algorithms :
- XXH32 : generates 32-bit hashes, using 32-bit arithmetic
- XXH64 : generates 64-bit hashes, using 64-bit arithmetic
- XXH3 (since `v0.8.0`): generates 64 or 128-bit hashes, using vectorized arithmetic.
The 128-bit variant is called XXH128.
All variants successfully complete the [SMHasher](https://code.google.com/p/smhasher/wiki/SMHasher) test suite
which evaluates the quality of hash functions (collision, dispersion and randomness).
Additional tests, which evaluate more thoroughly speed and collision properties of 64-bit hashes, [are also provided](https://github.com/Cyan4973/xxHash/tree/dev/tests).
|Branch |Status |
|------------|---------|
|release | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=release)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Arelease+) |
|dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) |
Benchmarks
-------------------------
The benchmarked reference system uses an Intel i7-9700K cpu, and runs Ubuntu x64 20.04.
The [open source benchmark program] is compiled with `clang` v10.0 using `-O3` flag.
| Hash Name | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment |
| --------- | ----- | ---------------- | ----- | --- | --- |
| __XXH3__ (SSE2) | 64 | 31.5 GB/s | 133.1 | 10
| __XXH128__ (SSE2) | 128 | 29.6 GB/s | 118.1 | 10
| _RAM sequential read_ | N/A | 28.0 GB/s | N/A | N/A | _for reference_
| City64 | 64 | 22.0 GB/s | 76.6 | 10
| T1ha2 | 64 | 22.0 GB/s | 99.0 | 9 | Slightly worse [collisions]
| City128 | 128 | 21.7 GB/s | 57.7 | 10
| __XXH64__ | 64 | 19.4 GB/s | 71.0 | 10
| SpookyHash | 64 | 19.3 GB/s | 53.2 | 10
| Mum | 64 | 18.0 GB/s | 67.0 | 9 | Slightly worse [collisions]
| __XXH32__ | 32 | 9.7 GB/s | 71.9 | 10
| City32 | 32 | 9.1 GB/s | 66.0 | 10
| Murmur3 | 32 | 3.9 GB/s | 56.1 | 10
| SipHash | 64 | 3.0 GB/s | 43.2 | 10
| FNV64 | 64 | 1.2 GB/s | 62.7 | 5 | Poor avalanche properties
| Blake2 | 256 | 1.1 GB/s | 5.1 | 10 | Cryptographic
| SHA1 | 160 | 0.8 GB/s | 5.6 | 10 | Cryptographic but broken
| MD5 | 128 | 0.6 GB/s | 7.8 | 10 | Cryptographic but broken
[open source benchmark program]: https://github.com/Cyan4973/xxHash/tree/release/tests/bench
[collisions]: https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison#collision-study
note 1: Small data velocity is a _rough_ evaluation of algorithm's efficiency on small data. For more detailed analysis, please refer to next paragraph.
note 2: some algorithms feature _faster than RAM_ speed. In which case, they can only reach their full speed potential when input is already in CPU cache (L3 or better). Otherwise, they max out on RAM speed limit.
### Small data
Performance on large data is only one part of the picture.
Hashing is also very useful in constructions like hash tables and bloom filters.
In these use cases, it's frequent to hash a lot of small data (starting at a few bytes).
Algorithm's performance can be very different for such scenarios, since parts of the algorithm,
such as initialization or finalization, become fixed cost.
The impact of branch mis-prediction also becomes much more present.
XXH3 has been designed for excellent performance on both long and small inputs,
which can be observed in the following graph:
![XXH3, latency, random size](https://user-images.githubusercontent.com/750081/61976089-aedeab00-af9f-11e9-9239-e5375d6c080f.png)
For a more detailed analysis, please visit the wiki :
https://github.com/Cyan4973/xxHash/wiki/Performance-comparison#benchmarks-concentrating-on-small-data-
Quality
-------------------------
Speed is not the only property that matters.
Produced hash values must respect excellent dispersion and randomness properties,
so that any sub-section of it can be used to maximally spread out a table or index,
as well as reduce the amount of collisions to the minimal theoretical level, following the [birthday paradox].
`xxHash` has been tested with Austin Appleby's excellent SMHasher test suite,
and passes all tests, ensuring reasonable quality levels.
It also passes extended tests from [newer forks of SMHasher], featuring additional scenarios and conditions.
Finally, xxHash provides its own [massive collision tester](https://github.com/Cyan4973/xxHash/tree/dev/tests/collisions),
able to generate and compare billions of hashes to test the limits of 64-bit hash algorithms.
On this front too, xxHash features good results, in line with the [birthday paradox].
A more detailed analysis is documented [in the wiki](https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison).
[birthday paradox]: https://en.wikipedia.org/wiki/Birthday_problem
[newer forks of SMHasher]: https://github.com/rurban/smhasher
### Build modifiers
The following macros can be set at compilation time to modify `libxxhash`'s behavior. They are generally disabled by default.
- `XXH_INLINE_ALL`: Make all functions `inline`, implementation is directly included within `xxhash.h`.
Inlining functions is beneficial for speed, notably for small keys.
It's _extremely effective_ when key's length is expressed as _a compile time constant_,
with performance improvements observed in the +200% range .
See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details.
- `XXH_PRIVATE_API`: same outcome as `XXH_INLINE_ALL`. Still available for legacy support.
The name underlines that `XXH_*` symbol names will not be exported.
- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation.
Incompatible with dynamic linking, due to risks of ABI changes.
- `XXH_NAMESPACE`: Prefixes all symbols with the value of `XXH_NAMESPACE`.
This macro can only use compilable character set.
Useful to evade symbol naming collisions,
in case of multiple inclusions of xxHash's source code.
Client applications still use the regular function names,
as symbols are automatically translated through `xxhash.h`.
- `XXH_FORCE_ALIGN_CHECK`: Use a faster direct read path when input is aligned.
This option can result in dramatic performance improvement on architectures unable to load memory from unaligned addresses
when input to hash happens to be aligned on 32 or 64-bit boundaries.
It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses).
This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms.
- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation.
Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets.
Method `2` forces unaligned reads, which is not standard compliant, but might sometimes be the only way to extract better read performance.
Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction.
- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time.
If, for some reason, the compiler cannot simplify the runtime test, it can cost performance.
It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1.
Setting it to 0 states big-endian.
- `XXH_ENABLE_AUTOVECTORIZE`: Auto-vectorization may be triggered for XXH32 and XXH64, depending on cpu vector capabilities and compiler version.
Note: auto-vectorization tends to be triggered more easily with recent versions of `clang`.
For XXH32, SSE4.1 or equivalent (NEON) is enough, while XXH64 requires AVX512.
Unfortunately, auto-vectorization is generally detrimental to XXH performance.
For this reason, the xxhash source code tries to prevent auto-vectorization by default.
That being said, systems evolve, and this conclusion is not forthcoming.
For example, it has been reported that recent Zen4 cpus are more likely to improve performance with vectorization.
Therefore, should you prefer or want to test vectorized code, you can enable this flag:
it will remove the no-vectorization protection code, thus making it more likely for XXH32 and XXH64 to be auto-vectorized.
- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump.
This is generally undesirable for performance, especially when hashing inputs of random sizes.
But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default.
- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors.
- `XXH_NO_STDLIB`: Disable invocation of `<stdlib.h>` functions, notably `malloc()` and `free()`.
`libxxhash`'s `XXH*_createState()` will always fail and return `NULL`.
But one-shot hashing (like `XXH32()`) or streaming using statically allocated states
still work as expected.
This build flag is useful for embedded environments without dynamic allocation.
- `XXH_memcpy`, `XXH_memset`, `XXH_memcmp` : redirect `memcpy()`, `memset()` and `memcmp()` to some user-selected symbol at compile time.
Redirecting all 3 removes the need to include `<string.h>` standard library.
- `XXH_NO_EXTERNC_GUARD`: When `xxhash.h` is compiled in C++ mode, removes the `extern "C" { .. }` block guard.
- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements.
This (slightly) slows down execution, but may help finding bugs during debugging sessions.
#### Binary size control
- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary.
`XXH3` is by far the largest contributor to `libxxhash` size,
so it's useful to reduce binary size for applications which do not employ `XXH3`.
- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit `long long` types
which include `XXH3` and `XXH64`.
Only `XXH32` will be compiled.
Useful for targets (architectures and compilers) without 64-bit support.
- `XXH_NO_STREAM`: Disables the streaming API, limiting the library to single shot variants only.
- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size.
Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not.
This is very useful when optimizing for smallest binary size,
and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang.
It may also be required to successfully compile using `-Og`, depending on compiler version.
- `XXH_SIZE_OPT`: `0`: default, optimize for speed
`1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization
`2`: makes code as small as possible, performance may cry
#### Build modifiers specific for XXH3
- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on x86_64 requires `-mavx2` for `AVX2`, or `-mavx512f` for `AVX512`).
- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only.
- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only.
#### Makefile variables
When compiling the Command Line Interface `xxhsum` using `make`, the following environment variables can also be set :
- `DISPATCH=1` : use `xxh_x86dispatch.c`, select at runtime between `scalar`, `sse2`, `avx2` or `avx512` instruction set. This option is only valid for `x86`/`x64` systems. It is enabled by default when target `x86`/`x64` is detected. It can be forcefully turned off using `DISPATCH=0`.
- `LIBXXH_DISPATCH=1` : same idea, implemented a runtime vector extension detector, but within `libxxhash`. This parameter is disabled by default. When enabled (only valid for `x86`/`x64` systems), new symbols published in `xxh_x86dispatch.h` become accessible. At the time of this writing, it's required to include `xxh_x86dispatch.h` in order to access the symbols with runtime vector extension detection.
- `XXH_1ST_SPEED_TARGET` : select an initial speed target, expressed in MB/s, for the first speed test in benchmark mode. Benchmark will adjust the target at subsequent iterations, but the first test is made "blindly" by targeting this speed. Currently conservatively set to 10 MB/s, to support very slow (emulated) platforms.
- `NODE_JS=1` : When compiling `xxhsum` for Node.js with Emscripten, this links the `NODERAWFS` library for unrestricted filesystem access and patches `isatty` to make the command line utility correctly detect the terminal. This does make the binary specific to Node.js.
### Building xxHash - Using vcpkg
You can download and install xxHash using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager:
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
./bootstrap-vcpkg.sh
./vcpkg integrate install
./vcpkg install xxhash
The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
### Example
The simplest example calls xxhash 64-bit variant as a one-shot function
generating a hash value from a single buffer, and invoked from a C/C++ program:
```C
#include "xxhash.h"
(...)
XXH64_hash_t hash = XXH64(buffer, size, seed);
}
```
Streaming variant is more involved, but makes it possible to provide data incrementally:
```C
#include "stdlib.h" /* abort() */
#include "xxhash.h"
XXH64_hash_t calcul_hash_streaming(FileHandler fh)
{
/* create a hash state */
XXH64_state_t* const state = XXH64_createState();
if (state==NULL) abort();
size_t const bufferSize = SOME_SIZE;
void* const buffer = malloc(bufferSize);
if (buffer==NULL) abort();
/* Initialize state with selected seed */
XXH64_hash_t const seed = 0; /* or any other value */
if (XXH64_reset(state, seed) == XXH_ERROR) abort();
/* Feed the state with input data, any size, any number of times */
(...)
while ( /* some data left */ ) {
size_t const length = get_more_data(buffer, bufferSize, fh);
if (XXH64_update(state, buffer, length) == XXH_ERROR) abort();
(...)
}
(...)
/* Produce the final hash value */
XXH64_hash_t const hash = XXH64_digest(state);
/* State could be re-used; but in this example, it is simply freed */
free(buffer);
XXH64_freeState(state);
return hash;
}
```
### License
The library files `xxhash.c` and `xxhash.h` are BSD licensed.
The utility `xxhsum` is GPL licensed.
### Other programming languages
Beyond the C reference version,
xxHash is also available from many different programming languages,
thanks to great contributors.
They are [listed here](http://www.xxhash.com/#other-languages).
### Packaging status
Many distributions bundle a package manager
which allows easy xxhash installation as both a `libxxhash` library
and `xxhsum` command line interface.
[![Packaging status](https://repology.org/badge/vertical-allrepos/xxhash.svg)](https://repology.org/project/xxhash/versions)
### Special Thanks
- Takayuki Matsuoka, aka @t-mat, for creating `xxhsum -c` and great support during early xxh releases
- Mathias Westerdahl, aka @JCash, for introducing the first version of `XXH64`
- Devin Hussey, aka @easyaspi314, for incredible low-level optimizations on `XXH3` and `XXH128`

13
external/xxhash/SECURITY.md vendored Normal file
View File

@ -0,0 +1,13 @@
# Security Policy
## Supported Versions
Security updates are applied only to the latest release.
## Reporting a Vulnerability
If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
Please disclose it at [security advisory](https://github.com/Cyan4973/xxHash/security/advisories/new).
This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure.

55
external/xxhash/xxh3.h vendored Normal file
View File

@ -0,0 +1,55 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Development source file for `xxh3`
* Copyright (C) 2019-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
/*
* Note: This file used to host the source code of XXH3_* variants.
* during the development period.
* The source code is now properly integrated within xxhash.h.
*
* xxh3.h is no longer useful,
* but it is still provided for compatibility with source code
* which used to include it directly.
*
* Programs are now highly discouraged to include xxh3.h.
* Include `xxhash.h` instead, which is the officially supported interface.
*
* In the future, xxh3.h will start to generate warnings, then errors,
* then it will be removed from source package and from include directory.
*/
/* Simulate the same impact as including the old xxh3.h source file */
#define XXH_INLINE_ALL
#include "xxhash.h"

821
external/xxhash/xxh_x86dispatch.c vendored Normal file
View File

@ -0,0 +1,821 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2020-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
/*!
* @file xxh_x86dispatch.c
*
* Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
*
* Optional add-on.
*
* **Compile this file with the default flags for your target.**
* Note that compiling with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`
* will make the resulting binary incompatible with cpus not supporting the requested instruction set.
*
* @defgroup dispatch x86 Dispatcher
* @{
*/
#if defined (__cplusplus)
extern "C" {
#endif
#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64))
# error "Dispatching is currently only supported on x86 and x86_64."
#endif
/*! @cond Doxygen ignores this part */
#ifndef XXH_HAS_INCLUDE
# ifdef __has_include
/*
* Not defined as XXH_HAS_INCLUDE(x) (function-like) because
* this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
*/
# define XXH_HAS_INCLUDE __has_include
# else
# define XXH_HAS_INCLUDE(x) 0
# endif
#endif
/*! @endcond */
/*!
* @def XXH_DISPATCH_SCALAR
* @brief Enables/dispatching the scalar code path.
*
* If this is defined to 0, SSE2 support is assumed. This reduces code size
* when the scalar path is not needed.
*
* This is automatically defined to 0 when...
* - SSE2 support is enabled in the compiler
* - Targeting x86_64
* - Targeting Android x86
* - Targeting macOS
*/
#ifndef XXH_DISPATCH_SCALAR
# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
|| defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
|| defined(__ANDROID__) || defined(__APPLE__) /* Android or macOS */
# define XXH_DISPATCH_SCALAR 0 /* disable */
# else
# define XXH_DISPATCH_SCALAR 1
# endif
#endif
/*!
* @def XXH_DISPATCH_AVX2
* @brief Enables/disables dispatching for AVX2.
*
* This is automatically detected if it is not defined.
* - GCC 4.7 and later are known to support AVX2, but >4.9 is required for
* to get the AVX2 intrinsics and typedefs without -mavx -mavx2.
* - Visual Studio 2013 Update 2 and later are known to support AVX2.
* - The GCC/Clang internal header `<avx2intrin.h>` is detected. While this is
* not allowed to be included directly, it still appears in the builtin
* include path and is detectable with `__has_include`.
*
* @see XXH_AVX2
*/
#ifndef XXH_DISPATCH_AVX2
# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \
|| (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \
|| (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \
|| XXH_HAS_INCLUDE(<avx2intrin.h>) /* GCC/Clang internal header */
# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */
# else
# define XXH_DISPATCH_AVX2 0
# endif
#endif /* XXH_DISPATCH_AVX2 */
/*!
* @def XXH_DISPATCH_AVX512
* @brief Enables/disables dispatching for AVX512.
*
* Automatically detected if one of the following conditions is met:
* - GCC 4.9 and later are known to support AVX512.
* - Visual Studio 2017 and later are known to support AVX2.
* - The GCC/Clang internal header `<avx512fintrin.h>` is detected. While this
* is not allowed to be included directly, it still appears in the builtin
* include path and is detectable with `__has_include`.
*
* @see XXH_AVX512
*/
#ifndef XXH_DISPATCH_AVX512
# if (defined(__GNUC__) \
&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \
|| (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \
|| XXH_HAS_INCLUDE(<avx512fintrin.h>) /* GCC/Clang internal header */
# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */
# else
# define XXH_DISPATCH_AVX512 0
# endif
#endif /* XXH_DISPATCH_AVX512 */
/*!
* @def XXH_TARGET_SSE2
* @brief Allows a function to be compiled with SSE2 intrinsics.
*
* Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used
* even with `-mno-sse2`.
*
* @def XXH_TARGET_AVX2
* @brief Like @ref XXH_TARGET_SSE2, but for AVX2.
*
* @def XXH_TARGET_AVX512
* @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
*
*/
#if defined(__GNUC__)
# include <emmintrin.h> /* SSE2 */
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
# include <immintrin.h> /* AVX2, AVX512F */
# endif
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
#elif defined(__clang__) && defined(_MSC_VER) /* clang-cl.exe */
# include <emmintrin.h> /* SSE2 */
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
# include <immintrin.h> /* AVX2, AVX512F */
# include <smmintrin.h>
# include <avxintrin.h>
# include <avx2intrin.h>
# include <avx512fintrin.h>
# endif
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
#elif defined(_MSC_VER)
# include <intrin.h>
# define XXH_TARGET_SSE2
# define XXH_TARGET_AVX2
# define XXH_TARGET_AVX512
#else
# error "Dispatching is currently not supported for your compiler."
#endif
/*! @cond Doxygen ignores this part */
#ifdef XXH_DISPATCH_DEBUG
/* debug logging */
# include <stdio.h>
# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); }
#else
# define XXH_debugPrint(str) ((void)0)
# undef NDEBUG /* avoid redefinition */
# define NDEBUG
#endif
/*! @endcond */
#include <assert.h>
#ifndef XXH_DOXYGEN
#define XXH_INLINE_ALL
#define XXH_X86DISPATCH
#include "xxhash.h"
#endif
/*! @cond Doxygen ignores this part */
#ifndef XXH_HAS_ATTRIBUTE
# ifdef __has_attribute
# define XXH_HAS_ATTRIBUTE(...) __has_attribute(__VA_ARGS__)
# else
# define XXH_HAS_ATTRIBUTE(...) 0
# endif
#endif
/*! @endcond */
/*! @cond Doxygen ignores this part */
#if XXH_HAS_ATTRIBUTE(constructor)
# define XXH_CONSTRUCTOR __attribute__((constructor))
# define XXH_DISPATCH_MAYBE_NULL 0
#else
# define XXH_CONSTRUCTOR
# define XXH_DISPATCH_MAYBE_NULL 1
#endif
/*! @endcond */
/*! @cond Doxygen ignores this part */
/*
* Support both AT&T and Intel dialects
*
* GCC doesn't convert AT&T syntax to Intel syntax, and will error out if
* compiled with -masm=intel. Instead, it supports dialect switching with
* curly braces: { AT&T syntax | Intel syntax }
*
* Clang's integrated assembler automatically converts AT&T syntax to Intel if
* needed, making the dialect switching useless (it isn't even supported).
*
* Note: Comments are written in the inline assembly itself.
*/
#ifdef __clang__
# define XXH_I_ATT(intel, att) att "\n\t"
#else
# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
#endif
/*! @endcond */
/*!
* @private
* @brief Runs CPUID.
*
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
* @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
*/
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
{
#if defined(_MSC_VER)
__cpuidex((int*)abcd, eax, ecx);
#else
xxh_u32 ebx, edx;
# if defined(__i386__) && defined(__PIC__)
__asm__(
"# Call CPUID\n\t"
"#\n\t"
"# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t"
"# EBX, so we use EDI instead.\n\t"
XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi")
XXH_I_ATT("cpuid", "cpuid" )
XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi")
: "=D" (ebx),
# else
__asm__(
"# Call CPUID\n\t"
XXH_I_ATT("cpuid", "cpuid")
: "=b" (ebx),
# endif
"+a" (eax), "+c" (ecx), "=d" (edx));
abcd[0] = eax;
abcd[1] = ebx;
abcd[2] = ecx;
abcd[3] = edx;
#endif
}
/*
* Modified version of Intel's guide
* https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family
*/
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
/*!
* @private
* @brief Runs `XGETBV`.
*
* While the CPU may support AVX2, the operating system might not properly save
* the full YMM/ZMM registers.
*
* xgetbv is used for detecting this: Any compliant operating system will define
* a set of flags in the xcr0 register indicating how it saves the AVX registers.
*
* You can manually disable this flag on Windows by running, as admin:
*
* bcdedit.exe /set xsavedisable 1
*
* and rebooting. Run the same command with 0 to re-enable it.
*/
static xxh_u64 XXH_xgetbv(void)
{
#if defined(_MSC_VER)
return _xgetbv(0); /* min VS2010 SP1 compiler is required */
#else
xxh_u32 xcr0_lo, xcr0_hi;
__asm__(
"# Call XGETBV\n\t"
"#\n\t"
"# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t"
"# the XGETBV opcode, so we encode it by hand instead.\n\t"
"# See <https://github.com/asmjit/asmjit/issues/78> for details.\n\t"
".byte 0x0f, 0x01, 0xd0\n\t"
: "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0));
return xcr0_lo | ((xxh_u64)xcr0_hi << 32);
#endif
}
#endif
/*! @cond Doxygen ignores this part */
#define XXH_SSE2_CPUID_MASK (1 << 26)
#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
#define XXH_AVX2_CPUID_MASK (1 << 5)
#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
#define XXH_AVX512F_CPUID_MASK (1 << 16)
#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
/*! @endcond */
/*!
* @private
* @brief Returns the best XXH3 implementation.
*
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
*
* @return The best @ref XXH_VECTOR implementation.
* @see XXH_VECTOR_TYPES
*/
int XXH_featureTest(void)
{
xxh_u32 abcd[4];
xxh_u32 max_leaves;
int best = XXH_SCALAR;
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
xxh_u64 xgetbv_val;
#endif
#if defined(__GNUC__) && defined(__i386__)
xxh_u32 cpuid_supported;
__asm__(
"# For the sake of ruthless backwards compatibility, check if CPUID\n\t"
"# is supported in the EFLAGS on i386.\n\t"
"# This is not necessary on x86_64 - CPUID is mandatory.\n\t"
"# The ID flag (bit 21) in the EFLAGS register indicates support\n\t"
"# for the CPUID instruction. If a software procedure can set and\n\t"
"# clear this flag, the processor executing the procedure supports\n\t"
"# the CPUID instruction.\n\t"
"# <https://c9x.me/x86/html/file_module_x86_id_45.html>\n\t"
"#\n\t"
"# Routine is from <https://wiki.osdev.org/CPUID>.\n\t"
"# Save EFLAGS\n\t"
XXH_I_ATT("pushfd", "pushfl" )
"# Store EFLAGS\n\t"
XXH_I_ATT("pushfd", "pushfl" )
"# Invert the ID bit in stored EFLAGS\n\t"
XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)")
"# Load stored EFLAGS (with ID bit inverted)\n\t"
XXH_I_ATT("popfd", "popfl" )
"# Store EFLAGS again (ID bit may or not be inverted)\n\t"
XXH_I_ATT("pushfd", "pushfl" )
"# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t"
XXH_I_ATT("pop eax", "popl %%eax" )
"# eax = whichever bits were changed\n\t"
XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" )
"# Restore original EFLAGS\n\t"
XXH_I_ATT("popfd", "popfl" )
"# eax = zero if ID bit can't be changed, else non-zero\n\t"
XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" )
: "=a" (cpuid_supported) :: "cc");
if (XXH_unlikely(!cpuid_supported)) {
XXH_debugPrint("CPUID support is not detected!");
return best;
}
#endif
/* Check how many CPUID pages we have */
XXH_cpuid(0, 0, abcd);
max_leaves = abcd[0];
/* Shouldn't happen on hardware, but happens on some QEMU configs. */
if (XXH_unlikely(max_leaves == 0)) {
XXH_debugPrint("Max CPUID leaves == 0!");
return best;
}
/* Check for SSE2, OSXSAVE and xgetbv */
XXH_cpuid(1, 0, abcd);
/*
* Test for SSE2. The check is redundant on x86_64, but it doesn't hurt.
*/
if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK))
return best;
XXH_debugPrint("SSE2 support detected.");
best = XXH_SSE2;
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
/* Make sure we have enough leaves */
if (XXH_unlikely(max_leaves < 7))
return best;
/* Test for OSXSAVE and XGETBV */
if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK)
return best;
/* CPUID check for AVX features */
XXH_cpuid(7, 0, abcd);
xgetbv_val = XXH_xgetbv();
#if XXH_DISPATCH_AVX2
/* Validate that AVX2 is supported by the CPU */
if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK)
return best;
/* Validate that the OS supports YMM registers */
if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) {
XXH_debugPrint("AVX2 supported by the CPU, but not the OS.");
return best;
}
/* AVX2 supported */
XXH_debugPrint("AVX2 support detected.");
best = XXH_AVX2;
#endif
#if XXH_DISPATCH_AVX512
/* Check if AVX512F is supported by the CPU */
if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) {
XXH_debugPrint("AVX512F not supported by CPU");
return best;
}
/* Validate that the OS supports ZMM registers */
if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) {
XXH_debugPrint("AVX512F supported by the CPU, but not the OS.");
return best;
}
/* AVX512F supported */
XXH_debugPrint("AVX512F support detected.");
best = XXH_AVX512;
#endif
#endif
return best;
}
/* === Vector implementations === */
/*! @cond PRIVATE */
/*!
* @private
* @brief Defines the various dispatch functions.
*
* TODO: Consolidate?
*
* @param suffix The suffix for the functions, e.g. sse2 or scalar
* @param target XXH_TARGET_* or empty.
*/
#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \
\
/* === XXH3, default variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len) \
{ \
return XXH3_hashLong_64b_internal( \
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH3, Seeded variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len, \
XXH64_hash_t seed) \
{ \
return XXH3_hashLong_64b_withSeed_internal( \
input, len, seed, XXH3_accumulate_##suffix, \
XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
); \
} \
\
/* === XXH3, Secret variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len, XXH_NOESCAPE const void* secret, \
size_t secretLen) \
{ \
return XXH3_hashLong_64b_internal( \
input, len, secret, secretLen, \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH3 update variants === */ \
\
XXH_NO_INLINE target XXH_errorcode \
XXH3_update_##suffix(XXH_NOESCAPE XXH3_state_t* state, \
XXH_NOESCAPE const void* input, size_t len) \
{ \
return XXH3_update(state, (const xxh_u8*)input, len, \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
} \
\
/* === XXH128 default variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len) \
{ \
return XXH3_hashLong_128b_internal( \
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH128 Secret variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len, \
XXH_NOESCAPE const void* XXH_RESTRICT secret, \
size_t secretLen) \
{ \
return XXH3_hashLong_128b_internal( \
input, len, (const xxh_u8*)secret, secretLen, \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
} \
\
/* === XXH128 Seeded variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len,\
XXH64_hash_t seed) \
{ \
return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix, \
XXH3_initCustomSecret_##suffix); \
}
/*! @endcond */
/* End XXH_DEFINE_DISPATCH_FUNCS */
/*! @cond Doxygen ignores this part */
#if XXH_DISPATCH_SCALAR
XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
#endif
XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2)
#if XXH_DISPATCH_AVX2
XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
#endif
#if XXH_DISPATCH_AVX512
XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
#endif
#undef XXH_DEFINE_DISPATCH_FUNCS
/*! @endcond */
/* ==== Dispatchers ==== */
/*! @cond Doxygen ignores this part */
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH_NOESCAPE XXH3_state_t*, XXH_NOESCAPE const void*, size_t);
typedef struct {
XXH3_dispatchx86_hashLong64_default hashLong64_default;
XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed;
XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret;
XXH3_dispatchx86_update update;
} XXH_dispatchFunctions_s;
#define XXH_NB_DISPATCHES 4
/*! @endcond */
/*!
* @private
* @brief Table of dispatchers for @ref XXH3_64bits().
*
* @pre The indices must match @ref XXH_VECTOR_TYPE.
*/
static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
#if XXH_DISPATCH_SCALAR
/* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar },
#else
/* Scalar */ { NULL, NULL, NULL, NULL },
#endif
/* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 },
#if XXH_DISPATCH_AVX2
/* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 },
#else
/* AVX2 */ { NULL, NULL, NULL, NULL },
#endif
#if XXH_DISPATCH_AVX512
/* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 }
#else
/* AVX512 */ { NULL, NULL, NULL, NULL }
#endif
};
/*!
* @private
* @brief The selected dispatch table for @ref XXH3_64bits().
*/
static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
/*! @cond Doxygen ignores this part */
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef struct {
XXH3_dispatchx86_hashLong128_default hashLong128_default;
XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed;
XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
XXH3_dispatchx86_update update;
} XXH_dispatch128Functions_s;
/*! @endcond */
/*!
* @private
* @brief Table of dispatchers for @ref XXH3_128bits().
*
* @pre The indices must match @ref XXH_VECTOR_TYPE.
*/
static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
#if XXH_DISPATCH_SCALAR
/* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar },
#else
/* Scalar */ { NULL, NULL, NULL, NULL },
#endif
/* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 },
#if XXH_DISPATCH_AVX2
/* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 },
#else
/* AVX2 */ { NULL, NULL, NULL, NULL },
#endif
#if XXH_DISPATCH_AVX512
/* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 }
#else
/* AVX512 */ { NULL, NULL, NULL, NULL }
#endif
};
/*!
* @private
* @brief The selected dispatch table for @ref XXH3_64bits().
*/
static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
/*!
* @private
* @brief Runs a CPUID check and sets the correct dispatch tables.
*/
static XXH_CONSTRUCTOR void XXH_setDispatch(void)
{
int vecID = XXH_featureTest();
XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512);
#if !XXH_DISPATCH_SCALAR
assert(vecID != XXH_SCALAR);
#endif
#if !XXH_DISPATCH_AVX512
assert(vecID != XXH_AVX512);
#endif
#if !XXH_DISPATCH_AVX2
assert(vecID != XXH_AVX2);
#endif
XXH_g_dispatch = XXH_kDispatch[vecID];
XXH_g_dispatch128 = XXH_kDispatch128[vecID];
}
/* ==== XXH3 public functions ==== */
/*! @cond Doxygen ignores this part */
static XXH64_hash_t
XXH3_hashLong_64b_defaultSecret_selection(const void* XXH_RESTRICT input, size_t len,
XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)seed64; (void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_default == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_default(input, len);
}
XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
{
return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
}
static XXH64_hash_t
XXH3_hashLong_64b_withSeed_selection(const void* XXH_RESTRICT input, size_t len,
XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_seed == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
}
XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
}
static XXH64_hash_t
XXH3_hashLong_64b_withSecret_selection(const void* XXH_RESTRICT input, size_t len,
XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)seed64;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_secret == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
}
XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
{
return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
}
XXH_errorcode
XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.update == NULL)
XXH_setDispatch();
return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
}
/*! @endcond */
/* ==== XXH128 public functions ==== */
/*! @cond Doxygen ignores this part */
static XXH128_hash_t
XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)seed64; (void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_default == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_default(input, len);
}
XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
{
return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
}
static XXH128_hash_t
XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_seed == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
}
XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
}
static XXH128_hash_t
XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)seed64;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_secret == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
}
XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
{
return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
}
XXH_errorcode
XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.update == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
}
/*! @endcond */
#if defined (__cplusplus)
}
#endif
/*! @} */

93
external/xxhash/xxh_x86dispatch.h vendored Normal file
View File

@ -0,0 +1,93 @@
/*
* xxHash - XXH3 Dispatcher for x86-based targets
* Copyright (C) 2020-2024 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#ifndef XXH_X86DISPATCH_H_13563687684
#define XXH_X86DISPATCH_H_13563687684
#include "xxhash.h" /* XXH64_hash_t, XXH3_state_t */
#if defined (__cplusplus)
extern "C" {
#endif
/*!
* @brief Returns the best XXH3 implementation for x86
*
* @return The best @ref XXH_VECTOR implementation.
* @see XXH_VECTOR_TYPES
*/
XXH_PUBLIC_API int XXH_featureTest(void);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
#if defined (__cplusplus)
}
#endif
/* automatic replacement of XXH3 functions.
* can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */
#ifndef XXH_DISPATCH_DISABLE_REPLACE
# undef XXH3_64bits
# define XXH3_64bits XXH3_64bits_dispatch
# undef XXH3_64bits_withSeed
# define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch
# undef XXH3_64bits_withSecret
# define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch
# undef XXH3_64bits_update
# define XXH3_64bits_update XXH3_64bits_update_dispatch
# undef XXH128
# define XXH128 XXH3_128bits_withSeed_dispatch
# undef XXH3_128bits
# define XXH3_128bits XXH3_128bits_dispatch
# undef XXH3_128bits_withSeed
# define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch
# undef XXH3_128bits_withSecret
# define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch
# undef XXH3_128bits_update
# define XXH3_128bits_update XXH3_128bits_update_dispatch
#endif /* XXH_DISPATCH_DISABLE_REPLACE */
#endif /* XXH_X86DISPATCH_H_13563687684 */

42
external/xxhash/xxhash.c vendored Normal file
View File

@ -0,0 +1,42 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2012-2023 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
/*
* xxhash.c instantiates functions defined in xxhash.h
*/
#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
#define XXH_IMPLEMENTATION /* access definitions */
#include "xxhash.h"

7343
external/xxhash/xxhash.h vendored Normal file

File diff suppressed because it is too large Load Diff

View File

@ -4,7 +4,7 @@
#define ARENA_HEADER_SIZE 64 #define ARENA_HEADER_SIZE 64
typedef struct typedef struct Arena_t
{ {
u8 *buffer; u8 *buffer;
isize length; isize length;
@ -12,7 +12,7 @@ typedef struct
u32 init_line_no; u32 init_line_no;
} Arena; } Arena;
typedef struct typedef struct TempArena_t
{ {
Arena *arena; Arena *arena;
u64 pos; u64 pos;
@ -28,7 +28,7 @@ static Arena *CreateArenaDebug(rawptr buffer, isize length, u32 init_line_no);
// ::Allocator::FreeList::Header:: // ::Allocator::FreeList::Header::
typedef enum u8 typedef enum FLNodeColor_e
{ {
RB_RED, RB_RED,
RB_BLACK, RB_BLACK,

0
src/assets.c Normal file
View File

54
src/assets.h Normal file
View File

@ -0,0 +1,54 @@
#pragma once
typedef enum AssetType_e
{
SHADER_ASSET,
TEXTURE_ASSET,
SOUND_ASSET,
MODEL_ASSET,
ASSET_TYPE_MAX,
} AssetType;
typedef enum ShaderAsset_e
{
QUAD_FRAG_SPIRV_SHADER,
QUAD_VERT_SPIRV_SHADER,
GUI_FRAG_SPIRV_SHADER,
GUI_VERT_SPIRV_SHADER,
SHADER_ASSET_MAX,
} ShaderAsset;
typedef enum TextureAsset_e
{
PATTERMON_OBESE,
PATTERMON_NIGGER,
PATTERMON_ORIENTAL,
TEXTURE_ASSET_MAX
} TextureAsset;
typedef enum TextureAssetTag_e
{
TEXTURE_ASSET_TAG_MAX,
} TextureAssetTag;
typedef enum SoundAsset_e
{
SOUND_ASSET_MAX,
} SoundAsset;
typedef enum SoundAssetTag_e
{
SOUND_ASSET_TAG_MAX,
} SoundAssetTag;
typedef enum ModelAsset_e
{
MODEL_ASSET_MAX,
} ModelAsset;
typedef enum ModelAssetTag_e
{
MODEL_ASSET_TAG_MAX,
} ModelAssetTag;

View File

@ -2,6 +2,10 @@
#include "entry_linux.h" #include "entry_linux.h"
// ::ThirdParty::Include::
#include "xxhash/xxhash.c"
#include "fastlz/fastlz.c"
#include "platform/platform.c" #include "platform/platform.c"
#include "ds.c" #include "ds.c"
#include "util.c" #include "util.c"
@ -9,63 +13,6 @@
#include "renderer.c" #include "renderer.c"
#include "game.c" #include "game.c"
const char *strs[10] = {
"String 1",
"String 2",
"String 3",
"String 4",
"String 5",
"String 6",
"String 7",
"String 8",
"String 9",
"String 10",
};
u32 volatile str_index = 0;
pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
pthread_cond_t cond1 = PTHREAD_COND_INITIALIZER;
pthread_cond_t cond2 = PTHREAD_COND_INITIALIZER;
#include <unistd.h>
void *ThreadFunc(void *i)
{
pthread_mutex_t mut;
u32 val = *(u32 *)i;
pthread_mutex_init(&mut, NULL);
Printfln("Thread %d Started", *(u32 *)i);
pthread_cond_t *c = val == 1 ? &cond1 : &cond2;
pthread_mutex_lock(&mut);
pthread_cond_wait(c, &mut);
Printfln("Thread %d woken up", *(u32 *)i);
pthread_cond_wait(&cond, &mut);
Printfln("Thread %d global wake up", val);
pthread_mutex_unlock(&mut);
pthread_exit(NULL);
}
void RunThreadFunc(pthread_t *th, u32 *u, void *func)
{
pthread_create(th, NULL, func, u);
}
#define BIL 1000000000L
u64 GetDiff(struct timespec *start, struct timespec *end)
{
return BIL * (end->tv_sec - start->tv_sec) + end->tv_nsec - start->tv_nsec;
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
u8 *mem = (u8 *)MemAllocZeroed(MB(64)); u8 *mem = (u8 *)MemAllocZeroed(MB(64));
@ -80,9 +27,9 @@ int main(int argc, char **argv)
rawptr game_mem = ArenaAlloc(arena, game_mem_size); rawptr game_mem = ArenaAlloc(arena, game_mem_size);
Arena *game_arena = CreateArenaDebug(game_mem, game_mem_size, 3); Arena *game_arena = CreateArenaDebug(game_mem, game_mem_size, 3);
Assert(CreatePlatformWindow(), "Failed to initialize the window"); Assert(CreatePlatformWindow(WINDOW_NAME), "Failed to initialize the window");
GameInput *inputs = ArenaAlloc(arena, sizeof(GameInput) * 10); GameInput *inputs = MakeArray(arena, GameInput, 10);
u32 i_count = 0; u32 i_count = 0;
WaitForWindowEvent(inputs); WaitForWindowEvent(inputs);

View File

@ -8,7 +8,10 @@
#define WINDOW_NAME "Video Game" #define WINDOW_NAME "Video Game"
// ::ThirdParty::Include::Header::
#include "stb/stb_sprintf.h" #include "stb/stb_sprintf.h"
#include "xxhash/xxhash.h"
#include "fastlz/fastlz.h"
#include "shared_types.h" #include "shared_types.h"
#include "ds.h" #include "ds.h"

View File

@ -31,9 +31,9 @@ int CALLBACK WinMain(HINSTANCE instance, HINSTANCE prev_instance, LPSTR cmd_line
rawptr game_mem = ArenaAlloc(arena, game_mem_size); rawptr game_mem = ArenaAlloc(arena, game_mem_size);
Arena *game_arena = CreateArenaDebug(game_mem, game_mem_size, 3); Arena *game_arena = CreateArenaDebug(game_mem, game_mem_size, 3);
Assert(CreatePlatformWindow(), "Failed to initialize window"); Assert(CreatePlatformWindow(WINDOW_NAME), "Failed to initialize window");
GameInput *inputs = ArenaAlloc(arena, sizeof(GameInput) * 10); GameInput *inputs = MakeArray(arena, GameInput, 10);
u32 i_count = 0; u32 i_count = 0;
InitializeGame(renderer_arena); InitializeGame(renderer_arena);

49979
src/file_data/images.c Normal file

File diff suppressed because it is too large Load Diff

View File

@ -2,16 +2,16 @@ static void InitializeGame(Arena *arena, GameContext *ctx, Arena *ctx_arena)
{ {
Assert(InitRenderer(arena), "Failed to initialize the renderer"); Assert(InitRenderer(arena), "Failed to initialize the renderer");
ctx->gui.vertices = ArenaAlloc(ctx_arena, sizeof(GUIVertex) * 128); ctx->gui.vertices = MakeArray(ctx_arena, GUIVertex, 128);
ctx->gui.vertices_len = 0; ctx->gui.vertices_len = 0;
ctx->gui.indices = ArenaAlloc(ctx_arena, sizeof(u32) * 768); ctx->gui.indices = MakeArray(ctx_arena, u32, 768);
ctx->gui.indices_len = 0; ctx->gui.indices_len = 0;
ctx->gui.instance_count = 0; ctx->gui.instance_count = 0;
ctx->windows = ArenaAlloc(ctx_arena, sizeof(GUIWindow) * 32); ctx->windows = MakeArray(ctx_arena, GUIWindow, 32);
ctx->window_len = 0; ctx->window_len = 0;
ctx->buttons = ArenaAlloc(ctx_arena, sizeof(GUIButton) * 64); ctx->buttons = MakeArray(ctx_arena, GUIButton, 64);
ctx->btn_len = 0; ctx->btn_len = 0;
ctx->arena = arena; ctx->arena = arena;
@ -261,11 +261,11 @@ static void RunCycle(GameContext *ctx, GameInput *inputs, u32 i_count)
GetViewportSize(&ctx->pc.res); GetViewportSize(&ctx->pc.res);
RenderBuffer *vertex_buffer = ArenaAlloc(ctx->arena, sizeof(RenderBuffer)); RenderBuffer *vertex_buffer = MakeArray(ctx->arena, RenderBuffer, 1);
vertex_buffer->type = RENDER_BUFFER_TYPE_VERTEX; vertex_buffer->type = RENDER_BUFFER_TYPE_VERTEX;
vertex_buffer->size = sizeof(GUIVertex) * ctx->gui.vertices_len; vertex_buffer->size = sizeof(GUIVertex) * ctx->gui.vertices_len;
RenderBuffer *index_buffer = ArenaAlloc(ctx->arena, sizeof(RenderBuffer)); RenderBuffer *index_buffer = MakeArray(ctx->arena, RenderBuffer, 1);
index_buffer->type = RENDER_BUFFER_TYPE_INDEX, index_buffer->type = RENDER_BUFFER_TYPE_INDEX,
index_buffer->size = sizeof(u32) * ctx->gui.indices_len, index_buffer->size = sizeof(u32) * ctx->gui.indices_len,

263
src/packer.c Normal file
View File

@ -0,0 +1,263 @@
#include "packer.h"
#include "xxhash/xxhash.c"
#include "fastlz/fastlz.c"
#include "platform/platform.c"
#include "ds.c"
#include "util.c"
#include "allocators.c"
#include "renderer.c"
#include "game.c"
#define FWrite(buf, size, count, file) ((size) * (fwrite(buf, size, count, file)))
#if __linux__
b32 ChangeDir(c8 *dir)
{
return chdir(dir);
}
u8 **GetFileNamesInDir(Arena *arena, u32 *count)
{
struct dirent *dir;
DIR *d = opendir(".");
*count = 0;
if (d)
{
while ((dir = readdir(d)) != NULL)
{
if (!StrEq(dir->d_name, ".") && !StrEq(dir->d_name, ".."))
*count += 1;
}
}
c8 **file_names = MakeArray(arena, u8*, *count);
d = opendir(".");
*count = 0;
if (d)
{
while ((dir = readdir(d)) != NULL)
{
if (!StrEq(dir->d_name, ".") && !StrEq(dir->d_name, ".."))
{
i32 str_len = StrLen(dir->d_name);
file_names[*count] = MakeArray(arena, u8, str_len);
MemCpy(file_names[*count], dir->d_name, str_len);
*count += 1;
}
}
}
return (u8 **)file_names;
}
b8 DirVisible(c8 *dir_name)
{
b8 found = false;
struct dirent *dir;
DIR *d = opendir(".");
if (d)
{
while ((dir = readdir(d)) != NULL)
{
if (StrEq(dir_name, dir->d_name) && dir->d_type == DT_DIR)
{
found = true;
break;
}
}
}
return found;
}
#elif _WIN32
# error Not yet implemented
#endif
void SetArrayLookups()
{
for (i32 i = 0; i < Len(g_Shader_File_Map); i++)
{
FileMapping file_mapping = g_Shader_File_Map[i];
g_Shader_File_Names[file_mapping.ix] = file_mapping.file_name;
}
for (int i = 0; i < SHADER_ASSET_MAX; i++)
{
Assert(g_Shader_File_Names[i] != NULL, "Spirv shader file name is null");
}
for (i32 i = 0; i < Len(g_Texture_File_Map); i++)
{
FileMapping file_mapping = g_Texture_File_Map[i];
g_Texture_File_Names[file_mapping.ix] = file_mapping.file_name;
}
for (int i = 0; i < TEXTURE_ASSET_MAX; i++)
{
Assert(g_Texture_File_Names[i] != NULL, "Texture file name is null");
}
}
i32 WriteHeader(FILE *file, FileHeader *header)
{
i32 offset = 0;
return offset;
}
void MoveToShaderDir(c8 *return_dir)
{
if (DirVisible("build"))
{
Assert(ChangeDir("./build/shaders/glsl") == 0, "Unable to change to shader directory");
return_dir = "../../..";
}
else if (DirVisible("shaders"))
{
Assert(ChangeDir("./shaders/glsl") == 0 , "Unable to change to shader directory");
return_dir = "../..";
}
else
Assert(false, "Unable to find shader directory");
}
void MoveToTextureDir(c8 *return_dir)
{
if (DirVisible("assets"))
{
Assert(ChangeDir("./assets") == 0, "Unable to change to assets directory");
return_dir = "..";
}
if (DirVisible("shaders"))
{
Assert(ChangeDir("../assets") == 0, "Unable to change to assets directory");
return_dir = "../build";
}
else
Assert(false, "Unable to find assets directory");
}
void InitHeader(FileHeader *header)
{
Assert(header != NULL, "File header is null");
header->magic_num = CreateMagicValue('s', 't', 'e', 'g');
header->version = FILE_VERSION;
header->asset_counts[SHADER_ASSET] = SHADER_ASSET_MAX;
header->tag_counts[SHADER_ASSET] = 0;
header->asset_counts[TEXTURE_ASSET] = TEXTURE_ASSET_MAX;
header->tag_counts[TEXTURE_ASSET] = TEXTURE_ASSET_MAX;
header->asset_counts[SOUND_ASSET] = SOUND_ASSET_MAX;
header->tag_counts[SOUND_ASSET] = SOUND_ASSET_TAG_MAX;
header->asset_counts[MODEL_ASSET] = MODEL_ASSET_MAX;
header->tag_counts[MODEL_ASSET] = MODEL_ASSET_TAG_MAX;
u64 offset = sizeof(FileHeader);
for (u32 i = 0; i < ASSET_TYPE_MAX; i++)
{
if (header->tag_counts[i] > 0)
{
header->tag_offsets[i] = offset;
offset += sizeof(AssetTag) * header->tag_counts[i];
}
else
header->tag_offsets[i] = 0;
if (header->asset_counts[i] > 0)
{
header->asset_offsets[i] = offset;
offset += sizeof(AssetFile) * header->asset_counts[i];
}
else
header->asset_offsets[i] = 0;
}
}
void PackFiles(Arena *arena, FileHeader *header)
{
FILE *file = fopen("assets.sgp", "w+");
Assert(file != NULL, "File is null");
Assert(fwrite(header, sizeof(FileHeader), 1, file) == 1, "Unable to write file header");
u64 data_offset = 0;
for (u32 i = 0; i < ASSET_TYPE_MAX; i++)
{
u64 tag_offset = header->tag_offsets[i] + (header->tag_counts[i] * sizeof(AssetTag));
if (tag_offset > data_offset)
data_offset = tag_offset;
u64 asset_offset = header->asset_offsets[i] + (header->asset_counts[i] * sizeof(AssetFile));
if (asset_offset > data_offset)
data_offset = asset_offset;
}
c8 *return_dir = "";
u32 file_count;
MoveToShaderDir(return_dir);
AssetFile *shader_assets = MakeArray(arena, AssetFile, SHADER_ASSET_MAX);
for (u32 i = 0; i < SHADER_ASSET_MAX; i++)
{
c8 *asset_name = g_Shader_File_Names[i];
Printfln("Packing file: %s...", asset_name);
FILE *asset_file = fopen(asset_name, "r");
Assert(asset_file != NULL, "Asset file is null");
int ch = getc(asset_file);
Assert(ch != EOF, "File begins with EOF");
while (ch != EOF)
{
putc(ch, file);
ch = getc(asset_file);
}
shader_assets[i].data_offset = data_offset;
shader_assets[i].asset_id = i;
data_offset = (u64)ftell(file);
}
}
int main(int argc, c8 **argv)
{
SetArrayLookups();
void *mem = MemAllocZeroed(GB(1));
Arena *arena = CreateArenaDebug(mem, GB(1), __LINE__);
FILE *file = fopen("assets.sgp", "w+");
Assert(file != NULL, "File is null");
FileHeader header = {};
InitHeader(&header);
c8 *return_dir;
c8 **asset_names[ASSET_TYPE_MAX];
Assert(fseek(file, 0, SEEK_SET) == 0, "error seeking file");
Assert(fclose(file) != EOF, "error closing file");
}

94
src/packer.h Normal file
View File

@ -0,0 +1,94 @@
#pragma once
#if __linux__
# define _GNU_SOURCE
#elif _WIN32
#endif
#define STG_NO_RENDERER
#define STB_SPRINTF_IMPLEMENTATION
#include "stb/stb_sprintf.h"
#include "xxhash/xxhash.h"
#include "fastlz/fastlz.h"
#include "assets.h"
#include "shared_types.h"
#include "ds.h"
#include "platform/platform.h"
#include "util.h"
#include "allocators.h"
#include "renderer.h"
#include "game.h"
#include <stdio.h>
#if __linux__
# include <unistd.h>
# include <dirent.h>
#endif
#define CreateMagicValue(a, b, c, d) (u32)(a << 24) | (u32)(a << 16) | (u32)(a << 8) | (u32)(a)
#define FILE_VERSION 0
typedef struct AssetTag_t
{
u32 tag_id;
f32 value;
} AssetTag;
typedef struct AssetFile_t
{
u64 data_offset;
u32 asset_id;
} AssetFile;
typedef struct AssetHeader_t
{
AssetTag *tags;
AssetFile *assets;
u32 tag_count;
u32 asset_count;
} AssetHeader;
typedef struct FileHeader_t
{
u32 magic_num;
u32 version;
u32 tag_counts[ASSET_TYPE_MAX];
u32 asset_counts[ASSET_TYPE_MAX];
u64 tag_offsets[ASSET_TYPE_MAX];
u64 asset_offsets[ASSET_TYPE_MAX];
} FileHeader;
typedef struct FileMapping_t
{
c8 *file_name;
u32 ix;
} FileMapping;
const FileMapping g_Shader_File_Map[] = {
{ .file_name = "quad.frag.spv", .ix = QUAD_FRAG_SPIRV_SHADER },
{ .file_name = "quad.vert.spv", .ix = QUAD_VERT_SPIRV_SHADER },
{ .file_name = "gui.frag.spv", .ix = GUI_FRAG_SPIRV_SHADER },
{ .file_name = "gui.vert.spv", .ix = GUI_VERT_SPIRV_SHADER },
};
const FileMapping g_Texture_File_Map[] = {
{ .file_name = "pattermon.png", .ix = PATTERMON_OBESE },
{ .file_name = "patamon.png", .ix = PATTERMON_ORIENTAL },
{ .file_name = "purplemon.png", .ix = PATTERMON_NIGGER },
};
c8 *g_Shader_File_Names[SHADER_ASSET_MAX] = {};
c8 *g_Texture_File_Names[TEXTURE_ASSET_MAX] = {};
b32 ChangeDir(c8 *dir);
u8 **GetFileNamesInDir(Arena *arena, u32 *count);
b8 DirVisible(c8 *dir);
void InitHeader(FileHeader *header);
i32 WriteHeader(FILE *file, FileHeader *header);

View File

@ -95,7 +95,7 @@ i32 EPrintf(const char *fmt, ...);
// ::Platform::Functions::Window::Header:: // ::Platform::Functions::Window::Header::
b32 CreatePlatformWindow(); b32 CreatePlatformWindow(const char *window_name);
WindowSize GetWindowSize(); WindowSize GetWindowSize();
b32 ShouldQuit(); b32 ShouldQuit();
PlatformWindow *GetPlatformWindow(); PlatformWindow *GetPlatformWindow();

View File

@ -70,8 +70,10 @@ void HandleWindowEvent(GameInput *inputs, u32 *i_count, b32 wait_for_event)
if (msg->data.data32[0] == linux_window.close_event) if (msg->data.data32[0] == linux_window.close_event)
global_quit = true; global_quit = true;
#ifndef STG_NO_RENDERER
if (msg->type == XCB_ATOM_NOTICE) if (msg->type == XCB_ATOM_NOTICE)
SetRenderResolution(linux_window.w, linux_window.h); SetRenderResolution(linux_window.w, linux_window.h);
#endif
} break; } break;
case XCB_EXPOSE: case XCB_EXPOSE:
{ {

View File

@ -21,6 +21,10 @@
#include <nmmintrin.h> #include <nmmintrin.h>
#include <immintrin.h> #include <immintrin.h>
#ifndef STG_NO_WINDOW
#endif
#include <sched.h> #include <sched.h>
// syscall defines // syscall defines

View File

@ -116,7 +116,7 @@ isize GetPageSize()
return (isize)sysconf(_SC_PAGESIZE); return (isize)sysconf(_SC_PAGESIZE);
} }
b32 CreatePlatformWindow() b32 CreatePlatformWindow(const char *window_name)
{ {
PlatformWindow *window = &linux_window; PlatformWindow *window = &linux_window;
@ -176,8 +176,8 @@ b32 CreatePlatformWindow()
XCB_ATOM_WM_NAME, XCB_ATOM_WM_NAME,
XCB_ATOM_STRING, XCB_ATOM_STRING,
8, 8,
StrLen(WINDOW_NAME), StrLen(window_name),
WINDOW_NAME window_name
); );
XCB_CHECK_ERROR(window, cookie, error, "Failed to rename window."); XCB_CHECK_ERROR(window, cookie, error, "Failed to rename window.");

View File

@ -128,7 +128,7 @@ i32 _EPrintf(const char *fmt, va_list arg)
return EPrint(&buffer); return EPrint(&buffer);
} }
b32 CreatePlatformWindow() b32 CreatePlatformWindow(const char *window_name)
{ {
b32 success = true; b32 success = true;
@ -148,7 +148,7 @@ b32 CreatePlatformWindow()
HWND window_handle = CreateWindowExA( HWND window_handle = CreateWindowExA(
0, 0,
window_class.lpszClassName, window_class.lpszClassName,
"Video Game", window_name,
WS_OVERLAPPEDWINDOW|WS_VISIBLE, WS_OVERLAPPEDWINDOW|WS_VISIBLE,
CW_USEDEFAULT, CW_USEDEFAULT,
CW_USEDEFAULT, CW_USEDEFAULT,

View File

@ -338,7 +338,7 @@ static DeviceQueues CheckDeviceQueueSupport(VkPhysicalDevice device, VkSurfaceKH
u32 queue_count; u32 queue_count;
vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_count, NULL); vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_count, NULL);
VkQueueFamilyProperties *families = ArenaAlloc(renderer.arena, sizeof(VkQueueFamilyProperties) * queue_count); VkQueueFamilyProperties *families = MakeArray(renderer.arena, VkQueueFamilyProperties, queue_count);
vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_count, families); vkGetPhysicalDeviceQueueFamilyProperties(device, &queue_count, families);
if (queue_count == 1 && if (queue_count == 1 &&
@ -748,13 +748,13 @@ static b32 CreateFrameStructures()
pool_create_info.queueFamilyIndex = renderer.vk.queues.graphics; pool_create_info.queueFamilyIndex = renderer.vk.queues.graphics;
renderer.vk.frame.pools = ArenaAlloc(renderer.perm_arena, sizeof(VkCommandPool) * img_count); renderer.vk.frame.pools = MakeArray(renderer.perm_arena, VkCommandPool, img_count);
renderer.vk.frame.buffers = ArenaAlloc(renderer.perm_arena, sizeof(VkCommandBuffer) * img_count); renderer.vk.frame.buffers = MakeArray(renderer.perm_arena, VkCommandBuffer, img_count);
renderer.vk.frame.swapchain_sems = ArenaAlloc(renderer.perm_arena, sizeof(VkSemaphore) * img_count); renderer.vk.frame.swapchain_sems = MakeArray(renderer.perm_arena, VkSemaphore, img_count);
renderer.vk.frame.render_sems = ArenaAlloc(renderer.perm_arena, sizeof(VkSemaphore) * img_count); renderer.vk.frame.render_sems = MakeArray(renderer.perm_arena, VkSemaphore, img_count);
renderer.vk.frame.render_fences = ArenaAlloc(renderer.perm_arena, sizeof(VkFence) * img_count); renderer.vk.frame.render_fences = MakeArray(renderer.perm_arena, VkFence, img_count);
renderer.vk.frame.buffer_destroy_queues = ArenaAlloc(renderer.perm_arena, sizeof(RenderBuffer *) * FRAME_OVERLAP); renderer.vk.frame.buffer_destroy_queues = MakeArray(renderer.perm_arena, RenderBuffer *, FRAME_OVERLAP);
renderer.vk.frame.buffer_counts = ArenaAlloc(renderer.perm_arena, sizeof(u32) * FRAME_OVERLAP); renderer.vk.frame.buffer_counts = MakeArray(renderer.perm_arena, u32, FRAME_OVERLAP);
for (u32 i = 0; i < FRAME_OVERLAP; i++) for (u32 i = 0; i < FRAME_OVERLAP; i++)
{ {

View File

@ -16,6 +16,8 @@ typedef uint16_t u16;
typedef uint32_t u32; typedef uint32_t u32;
typedef uint64_t u64; typedef uint64_t u64;
typedef char c8;
typedef intptr_t intptr; typedef intptr_t intptr;
typedef uintptr_t uintptr; typedef uintptr_t uintptr;

View File

@ -1,3 +1,5 @@
// ::Util::Strings::Functions::Start::
b32 StrEq(const char *l, const char *r) { b32 StrEq(const char *l, const char *r) {
for (; *l == *r && *l; l++, r++); for (; *l == *r && *l; l++, r++);
return *l == '\0' && *r == '\0'; return *l == '\0' && *r == '\0';
@ -24,6 +26,52 @@ i32 SPrintf(char *buf, rawptr fmt, ...)
return sprf_res; return sprf_res;
} }
Str8 MakeStr8(u8 *str, u32 len)
{
return (Str8){ .len = len, .value = str };
}
Str8 PreSplitStr8(Str8 string, Str8 delimiter)
{
if (string.len == 0 || delimiter.len == 0) return string;
u32 new_len = 0;
u32 matched_chars = 0;
for (u32 i = 0; i < string.len; i++)
{
if (string.value[i] == delimiter.value[matched_chars])
matched_chars += 1;
else
matched_chars = 0;
if (matched_chars == delimiter.len || delimiter.value[matched_chars+1] == '\0')
{
new_len = i;
break;
}
}
return (Str8){ .len = new_len, .value = string.value };
}
Str8 PreSplitNewStr8(Arena *arena, Str8 string, Str8 delimiter)
{
Str8 result = PreSplitStr8(string, delimiter);
if (result.len > 0)
{
result.value = MakeArray(arena, char, result.len+1);
MemCpy(result.value, string.value, result.len);
result.value[result.len] = '\0';
}
return result;
}
// ::Util::Strings::Functions::End::
void MemZero(void *ptr, isize size) void MemZero(void *ptr, isize size)
{ {
if (!size || !ptr) return; if (!size || !ptr) return;
@ -62,10 +110,12 @@ void MemCpy(rawptr dst, rawptr src, usize size)
isize iter_size = size > sizeof(u64) ? 8 : 1; isize iter_size = size > sizeof(u64) ? 8 : 1;
iter_size *= 4; iter_size *= 4;
isize iter_len = size / iter_size; isize iter_len = size / iter_size;
isize rem_len = 0;
u64 *mem_dst = (u64 *)dst; u64 *mem_dst = (u64 *)dst;
u64 *mem_src = (u64 *)src; u64 *mem_src = (u64 *)src;
if (iter_size > sizeof(size)) if (iter_size > sizeof(size))
{ {
while (iter_len > 0) while (iter_len > 0)
@ -78,12 +128,15 @@ void MemCpy(rawptr dst, rawptr src, usize size)
mem_src += 4; mem_src += 4;
iter_len--; iter_len--;
} }
rem_len = size % iter_size;
} }
else
rem_len = size;
u8 *byte_dst = (u8 *)mem_dst; u8 *byte_dst = (u8 *)mem_dst;
u8 *byte_src = (u8 *)mem_src; u8 *byte_src = (u8 *)mem_src;
isize rem_len = size % iter_size;
while (rem_len > 0) while (rem_len > 0)
{ {
byte_dst[0] = byte_src[0]; byte_dst[0] = byte_src[0];
@ -250,3 +303,12 @@ b32 HMCompressData(u8 *data, u32 len, u8 *out)
return success; return success;
} }
// ::Util::Hashing::Functions::Start::
u64 static inline HashFromString(Str8 string)
{
return XXH3_64bits_withSeed(string.value, string.len, HASH_SEED);
}
// ::Util::Hashing::Functions::End::

View File

@ -2,24 +2,42 @@
#include <assert.h> #include <assert.h>
#define Assert(condition, message) do { assert(condition && message); } while(0) typedef struct Arena_t Arena;
#define Assert(condition, message) do { assert((condition) && (message)); } while(0)
// ::Util::Size::Defines::
// generic defines
#define KB(n) n * 1024LL #define KB(n) n * 1024LL
#define MB(n) KB(n) * 1024LL #define MB(n) KB(n) * 1024LL
#define GB(n) MB(n) * 1024LL #define GB(n) MB(n) * 1024LL
#define TB(n) GB(n) * 1024LL #define TB(n) GB(n) * 1024LL
#define DEFAULT_ALIGNMENT (2*sizeof(rawptr)) // ::Util::Memory::Defines::
#define HM_MAX_SYMBOLS 256
#define DEFAULT_ALIGNMENT (2*sizeof(rawptr))
#define Len(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x])))))
#define MakeString(x) #x
#define BitEq(var, bits) (((var) & (bits)) == (bits)) #define BitEq(var, bits) (((var) & (bits)) == (bits))
#define AlignPow2(x, b) (((x) + (b) - 1) & (~((b) - 1))) #define AlignPow2(x, b) (((x) + (b) - 1) & (~((b) - 1)))
#define IsPow2(x) ((x) != 0 && ((x) &((x) - 1)) == 0) #define IsPow2(x) ((x) != 0 && ((x) &((x) - 1)) == 0)
#define PtrAdd(ptr, add) (((char *)ptr) + add) #define PtrAdd(ptr, add) (((char *)ptr) + add)
// ::Util::Array::Defines::
#define MakeArray(arena, type, count) ArenaAlloc(arena, (isize)(sizeof(type)) * (isize)(count))
#define Len(x) ((sizeof(x)/sizeof(0[x])) / ((size_t)(!(sizeof(x) % sizeof(0[x])))))
// ::Util::Compression::Defines::
#define HM_MAX_SYMBOLS 256
// ::Util::Strings::Defines::
#define Str8L(x) (Str8){ .len = Len(x), .value = (u8 *)x }
#define MakeString(x) #x
// ::Util::Hashing::Defines::
#define HASH_SEED 5995
typedef struct HMEnc_t typedef struct HMEnc_t
{ {
u8 length[256]; u8 length[256];
@ -43,16 +61,28 @@ typedef struct BitWriter_t
u8 *end; u8 *end;
} BitWriter; } BitWriter;
// String Functions typedef struct Str8_t
{
u32 len;
u8 *value;
} Str8;
// ::Util::Strings::Functions::Header::
u32 StrLen(const char *str); u32 StrLen(const char *str);
b32 StrEq(const char *l, const char *r); b32 StrEq(const char *l, const char *r);
i32 SPrintf(char *buf, rawptr fmt, ...); i32 SPrintf(char *buf, rawptr fmt, ...);
Str8 MakeStr8(u8 *str, u32 len);
Str8 PreSplitStr8(Str8 string, Str8 delimiter);
Str8 PreSplitNewStr8(Arena *arena, Str8 string, Str8 delimiter);
// ::Util::Memory::Functions::Header::
// Memory Functions
void MemZero(void *ptr, isize size); void MemZero(void *ptr, isize size);
void MemCpy(rawptr dst, rawptr src, usize len); void MemCpy(rawptr dst, rawptr src, usize len);
// Math Functions // ::Util::Math::Functions::Header::
u32 u32Min(u32 l, u32 r); u32 u32Min(u32 l, u32 r);
i32 i32Min(i32 l, i32 r); i32 i32Min(i32 l, i32 r);
u32 u32Max(u32 l, u32 r); u32 u32Max(u32 l, u32 r);
@ -60,7 +90,8 @@ i32 i32Max(i32 l, i32 r);
i32 i32Clamp(i32 v, i32 min, i32 max); i32 i32Clamp(i32 v, i32 min, i32 max);
u32 u32Clamp(u32 v, u32 min, u32 max); u32 u32Clamp(u32 v, u32 min, u32 max);
// Compression // ::Util::Compression::Functions::Header::
b32 HMCompressData(u8 *data, u32 len, u8 *out); b32 HMCompressData(u8 *data, u32 len, u8 *out);
u32 static inline ReadBit(BitWriter *bw); u32 static inline ReadBit(BitWriter *bw);
void static inline RefillBits(BitWriter *bw); void static inline RefillBits(BitWriter *bw);
@ -72,3 +103,7 @@ void static inline HMScan3(HMNode *nodes, u8 symbol1, u8 symbol2, u8 symbol3);
void static inline HMScan4(HMNode *nodes, u8 symbol1, u8 symbol2, u8 symbol3, u8 symbol4); void static inline HMScan4(HMNode *nodes, u8 symbol1, u8 symbol2, u8 symbol3, u8 symbol4);
void static inline HMBuildTable(HMNode *nodes); void static inline HMBuildTable(HMNode *nodes);
void static inline HMQuicksort(HMNode *nodes, u32 low, u32 high); void static inline HMQuicksort(HMNode *nodes, u32 low, u32 high);
// ::Util::Hashing::Functions::Header::
u64 static inline HashFromString(Str8 string);