3333 lines
114 KiB
D
3333 lines
114 KiB
D
/**
|
|
* Computes xxHash hashes of arbitrary data. xxHash hashes are either uint32_t,
|
|
* uint64_t or uint128_t quantities that are like a
|
|
* checksum or CRC, but are more robust and very performant.
|
|
*
|
|
$(SCRIPT inhibitQuickIndex = 1;)
|
|
|
|
$(DIVC quickindex,
|
|
$(BOOKTABLE ,
|
|
$(TR $(TH Category) $(TH Functions)
|
|
)
|
|
$(TR $(TDNW Template API) $(TD $(MYREF XXHTemplate)
|
|
)
|
|
)
|
|
$(TR $(TDNW OOP API) $(TD $(MYREF XXH32Digest))
|
|
)
|
|
$(TR $(TDNW Helpers) $(TD $(MYREF xxh32Of))
|
|
)
|
|
)
|
|
)
|
|
|
|
* This module conforms to the APIs defined in `std.digest`. To understand the
|
|
* differences between the template and the OOP API, see $(MREF std, digest).
|
|
*
|
|
* This module publicly imports $(MREF std, digest) and can be used as a stand-alone
|
|
* module.
|
|
*
|
|
* License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
|
|
*
|
|
* CTFE:
|
|
* Digests do not work in CTFE
|
|
*
|
|
* Authors:
|
|
* Carsten Schlote, Piotr Szturmaj, Kai Nacke, Johannes Pfau $(BR)
|
|
* The routines and algorithms are provided by the xxhash.[ch] source
|
|
* provided at $(I git@github.com:Cyan4973/xxHash.git).
|
|
*
|
|
* References:
|
|
* $(LINK2 https://github.com/Cyan4973/xxHash, GitHub website of project)
|
|
*
|
|
* Source: $(PHOBOSSRC std/digest/xxh.d)
|
|
*
|
|
*/
|
|
|
|
/* xxh.d - A wrapper for the original C implementation */
|
|
//module std.digest.xxh;
|
|
module xxhash3;
|
|
|
|
version (X86)
|
|
version = HaveUnalignedLoads;
|
|
else version (X86_64)
|
|
version = HaveUnalignedLoads;
|
|
|
|
//TODO: Properly detect this requirement.
|
|
//version = CheckACCAlignment;
|
|
|
|
//TODO: Check, if this code is an advantage over XXH provided code
|
|
// The code from core.int128 doesn't inline.
|
|
//version = Have128BitInteger;
|
|
|
|
version (Have128BitInteger)
|
|
{
|
|
import core.int128;
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Template API
|
|
import xxhash3 : XXH_32;
|
|
|
|
//Feeding data
|
|
ubyte[1024] data;
|
|
XXH_32 xxh;
|
|
xxh.start();
|
|
xxh.put(data[]);
|
|
xxh.start(); //Start again
|
|
xxh.put(data[]);
|
|
auto hash = xxh.finish();
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//OOP API
|
|
import xxhash3 : XXH32Digest;
|
|
|
|
auto xxh = new XXH32Digest();
|
|
ubyte[] hash = xxh.digest("abc");
|
|
assert(toHexString(hash) == "32D153FF", "Got " ~ toHexString(hash));
|
|
|
|
//Feeding data
|
|
ubyte[1024] data;
|
|
xxh.put(data[]);
|
|
xxh.reset(); //Start again
|
|
xxh.put(data[]);
|
|
hash = xxh.finish();
|
|
}
|
|
|
|
public import std.digest;
|
|
|
|
/* --- Port of C sources (release 0.8.1) to D language below ---------------- */
|
|
|
|
enum XXH_NO_STREAM = false;
|
|
enum XXH_SIZE_OPT = 0;
|
|
enum XXH_FORCE_ALIGN_CHECK = true;
|
|
enum XXH32_ENDJMP = false;
|
|
|
|
private import core.bitop : rol, bswap;
|
|
private import std.exception : enforce;
|
|
private import object : Exception;
|
|
|
|
/** Thrown on XXH errors. */
|
|
class XXHException : Exception
|
|
{
|
|
import std.exception : basicExceptionCtors;
|
|
///
|
|
mixin basicExceptionCtors;
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
import std.exception : enforce, assertThrown;
|
|
assertThrown(enforce!XXHException(false, "Throw me..."));
|
|
}
|
|
|
|
/* *************************************
|
|
* Misc
|
|
***************************************/
|
|
|
|
alias XXH32_hash_t = uint;
|
|
alias XXH64_hash_t = ulong;
|
|
/** Storage for 128bit hash digest */
|
|
align(16) struct XXH128_hash_t
|
|
{
|
|
XXH64_hash_t low64; /** `value & 0xFFFFFFFFFFFFFFFF` */
|
|
XXH64_hash_t high64; /** `value >> 64` */
|
|
}
|
|
|
|
alias XXH32_canonical_t = ubyte[XXH32_hash_t.sizeof];
|
|
static assert(XXH32_canonical_t.sizeof == 4, "32bit integers should be 4 bytes?");
|
|
alias XXH64_canonical_t = ubyte[XXH64_hash_t.sizeof];
|
|
static assert(XXH64_hash_t.sizeof == 8, "64bit integers should be 8 bytes?");
|
|
alias XXH128_canonical_t = ubyte[XXH128_hash_t.sizeof];
|
|
static assert(XXH128_hash_t.sizeof == 16, "128bit integers should be 16 bytes?");
|
|
|
|
enum XXH_VERSION_MAJOR = 0; /** XXHASH Major version */
|
|
enum XXH_VERSION_MINOR = 8; /** XXHASH Minor version */
|
|
enum XXH_VERSION_RELEASE = 1; /** XXHASH Build/Release version */
|
|
|
|
/** Version number, encoded as two digits each */
|
|
enum XXH_VERSION_NUMBER =
|
|
(XXH_VERSION_MAJOR * 100 * 100 +
|
|
XXH_VERSION_MINOR * 100 +
|
|
XXH_VERSION_RELEASE);
|
|
|
|
/** Get version number */
|
|
uint xxh_versionNumber() @safe pure nothrow @nogc
|
|
{
|
|
return XXH_VERSION_NUMBER;
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
assert(XXH_VERSION_NUMBER == xxh_versionNumber(), "Version mismatch");
|
|
}
|
|
|
|
/** The error code of public API functions */
|
|
enum XXH_errorcode
|
|
{
|
|
XXH_OK = 0, /** OK */
|
|
XXH_ERROR /** Error */
|
|
}
|
|
|
|
/** Structure for XXH32 streaming API.
|
|
*
|
|
* See: XXH64_state_s, XXH3_state_s
|
|
*/
|
|
struct XXH32_state_t
|
|
{
|
|
XXH32_hash_t total_len_32; /** Total length hashed, modulo 2^32 */
|
|
XXH32_hash_t large_len; /** Whether the hash is >= 16 (handles total_len_32 overflow) */
|
|
XXH32_hash_t[4] v; /** Accumulator lanes */
|
|
XXH32_hash_t[4] mem32; /** Internal buffer for partial reads. Treated as unsigned char[16]. */
|
|
XXH32_hash_t memsize; /** Amount of data in mem32 */
|
|
XXH32_hash_t reserved; /** Reserved field. Do not read nor write to it. */
|
|
}
|
|
|
|
/* Structure for XXH64 streaming API.
|
|
*
|
|
* See: XXH32_state_s, XXH3_state_s
|
|
*/
|
|
struct XXH64_state_t
|
|
{
|
|
XXH64_hash_t total_len; /** Total length hashed. This is always 64-bit. */
|
|
XXH64_hash_t[4] v; /** Accumulator lanes */
|
|
XXH64_hash_t[4] mem64; /** Internal buffer for partial reads. Treated as unsigned char[32]. */
|
|
XXH32_hash_t memsize; /** Amount of data in mem64 */
|
|
XXH32_hash_t reserved32; /** Reserved field, needed for padding anyways*/
|
|
XXH64_hash_t reserved64; /** Reserved field. Do not read or write to it. */
|
|
} /* typedef'd to XXH64_state_t */
|
|
|
|
|
|
/* ***************************
|
|
* Memory reads
|
|
*****************************/
|
|
|
|
/** Enum to indicate whether a pointer is aligned.
|
|
*/
|
|
private enum XXH_alignment
|
|
{
|
|
XXH_aligned, /** Aligned */
|
|
XXH_unaligned /** Possibly unaligned */
|
|
}
|
|
|
|
private uint xxh_read32(const void* ptr) @trusted pure nothrow @nogc
|
|
{
|
|
uint val;
|
|
version (HaveUnalignedLoads)
|
|
val = *(cast(uint*) ptr);
|
|
else
|
|
(cast(ubyte*)&val)[0 .. uint.sizeof] = (cast(ubyte*) ptr)[0 .. uint.sizeof];
|
|
return val;
|
|
}
|
|
|
|
private uint xxh_readLE32(const void* ptr) @safe pure nothrow @nogc
|
|
{
|
|
version (LittleEndian)
|
|
return xxh_read32(ptr);
|
|
else
|
|
return bswap(xxh_read32(ptr));
|
|
}
|
|
|
|
private uint xxh_readBE32(const void* ptr) @safe pure nothrow @nogc
|
|
{
|
|
version (LittleEndian)
|
|
return bswap(xxh_read32(ptr));
|
|
else
|
|
return xxh_read32(ptr);
|
|
}
|
|
|
|
private uint xxh_readLE32_align(const void* ptr, XXH_alignment align_) @trusted pure nothrow @nogc
|
|
{
|
|
if (align_ == XXH_alignment.XXH_unaligned)
|
|
{
|
|
return xxh_readLE32(ptr);
|
|
}
|
|
else
|
|
{
|
|
version (LittleEndian)
|
|
return *cast(const uint*) ptr;
|
|
else
|
|
return bswap(*cast(const uint*) ptr);
|
|
}
|
|
}
|
|
|
|
/* *******************************************************************
|
|
* 32-bit hash functions
|
|
*********************************************************************/
|
|
|
|
enum XXH_PRIME32_1 = 0x9E3779B1U; /** 0b10011110001101110111100110110001 */
|
|
enum XXH_PRIME32_2 = 0x85EBCA77U; /** 0b10000101111010111100101001110111 */
|
|
enum XXH_PRIME32_3 = 0xC2B2AE3DU; /** 0b11000010101100101010111000111101 */
|
|
enum XXH_PRIME32_4 = 0x27D4EB2FU; /** 0b00100111110101001110101100101111 */
|
|
enum XXH_PRIME32_5 = 0x165667B1U; /** 0b00010110010101100110011110110001 */
|
|
|
|
/** Normal stripe processing routine.
|
|
*
|
|
* This shuffles the bits so that any bit from @p input impacts several bits in
|
|
* acc.
|
|
*
|
|
* Param: acc = The accumulator lane.
|
|
* Param: input = The stripe of input to mix.
|
|
* Return: The mixed accumulator lane.
|
|
*/
|
|
private uint xxh32_round(uint acc, uint input) @safe pure nothrow @nogc
|
|
{
|
|
acc += input * XXH_PRIME32_2;
|
|
acc = rol(acc, 13);
|
|
acc *= XXH_PRIME32_1;
|
|
return acc;
|
|
}
|
|
|
|
/** Mixes all bits to finalize the hash.
|
|
*
|
|
* The final mix ensures that all input bits have a chance to impact any bit in
|
|
* the output digest, resulting in an unbiased distribution.
|
|
*
|
|
* Param: hash = The hash to avalanche.
|
|
* Return The avalanched hash.
|
|
*/
|
|
private uint xxh32_avalanche(uint hash) @safe pure nothrow @nogc
|
|
{
|
|
hash ^= hash >> 15;
|
|
hash *= XXH_PRIME32_2;
|
|
hash ^= hash >> 13;
|
|
hash *= XXH_PRIME32_3;
|
|
hash ^= hash >> 16;
|
|
return hash;
|
|
}
|
|
|
|
/* Alias wrapper for xxh_readLE32_align() */
|
|
private uint xxh_get32bits(const void* p, XXH_alignment align_) @safe pure nothrow @nogc
|
|
{
|
|
return xxh_readLE32_align(p, align_);
|
|
}
|
|
|
|
/** Processes the last 0-15 bytes of @p ptr.
|
|
*
|
|
* There may be up to 15 bytes remaining to consume from the input.
|
|
* This final stage will digest them to ensure that all input bytes are present
|
|
* in the final mix.
|
|
*
|
|
* Param: hash = The hash to finalize.
|
|
* Param: ptr = The pointer to the remaining input.
|
|
* Param: len = The remaining length, modulo 16.
|
|
* Param: align = Whether @p ptr is aligned.
|
|
* Return: The finalized hash.
|
|
* See: xxh64_finalize().
|
|
*/
|
|
private uint xxh32_finalize(uint hash, const(ubyte)* ptr, size_t len, XXH_alignment align_)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
static void XXH_PROCESS1(ref uint hash, ref const(ubyte)* ptr)
|
|
{
|
|
hash += (*ptr++) * XXH_PRIME32_5;
|
|
hash = rol(hash, 11) * XXH_PRIME32_1;
|
|
}
|
|
|
|
void XXH_PROCESS4(ref uint hash, ref const(ubyte)* ptr)
|
|
{
|
|
hash += xxh_get32bits(ptr, align_) * XXH_PRIME32_3;
|
|
ptr += 4;
|
|
hash = rol(hash, 17) * XXH_PRIME32_4;
|
|
}
|
|
|
|
/* Compact rerolled version; generally faster */
|
|
if (!XXH32_ENDJMP)
|
|
{
|
|
len &= 15;
|
|
while (len >= 4)
|
|
{
|
|
XXH_PROCESS4(hash, ptr);
|
|
len -= 4;
|
|
}
|
|
while (len > 0)
|
|
{
|
|
XXH_PROCESS1(hash, ptr);
|
|
--len;
|
|
}
|
|
return xxh32_avalanche(hash);
|
|
}
|
|
else
|
|
{
|
|
switch (len & 15) /* or switch (bEnd - p) */
|
|
{
|
|
case 12:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 8:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 4:
|
|
XXH_PROCESS4(hash, ptr);
|
|
return xxh32_avalanche(hash);
|
|
|
|
case 13:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 9:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 5:
|
|
XXH_PROCESS4(hash, ptr);
|
|
XXH_PROCESS1(hash, ptr);
|
|
return xxh32_avalanche(hash);
|
|
|
|
case 14:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 10:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 6:
|
|
XXH_PROCESS4(hash, ptr);
|
|
XXH_PROCESS1(hash, ptr);
|
|
XXH_PROCESS1(hash, ptr);
|
|
return xxh32_avalanche(hash);
|
|
|
|
case 15:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 11:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 7:
|
|
XXH_PROCESS4(hash, ptr);
|
|
goto case;
|
|
case 3:
|
|
XXH_PROCESS1(hash, ptr);
|
|
goto case;
|
|
case 2:
|
|
XXH_PROCESS1(hash, ptr);
|
|
goto case;
|
|
case 1:
|
|
XXH_PROCESS1(hash, ptr);
|
|
goto case;
|
|
case 0:
|
|
return xxh32_avalanche(hash);
|
|
default:
|
|
assert(0, "Internal error");
|
|
}
|
|
return hash; /* reaching this point is deemed impossible */
|
|
}
|
|
}
|
|
|
|
/** The implementation for XXH32().
|
|
*
|
|
* Params:
|
|
* input = Directly passed from XXH32().
|
|
* len = Ditto
|
|
* seed = Ditto
|
|
* align_ = Whether input is aligned.
|
|
* Return: The calculated hash.
|
|
*/
|
|
private uint xxh32_endian_align(
|
|
const(ubyte)* input, size_t len, uint seed, XXH_alignment align_)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
uint h32;
|
|
|
|
if (len >= 16)
|
|
{
|
|
const ubyte* bEnd = input + len;
|
|
const ubyte* limit = bEnd - 15;
|
|
uint v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
|
|
uint v2 = seed + XXH_PRIME32_2;
|
|
uint v3 = seed + 0;
|
|
uint v4 = seed - XXH_PRIME32_1;
|
|
|
|
do
|
|
{
|
|
v1 = xxh32_round(v1, xxh_get32bits(input, align_));
|
|
input += 4;
|
|
v2 = xxh32_round(v2, xxh_get32bits(input, align_));
|
|
input += 4;
|
|
v3 = xxh32_round(v3, xxh_get32bits(input, align_));
|
|
input += 4;
|
|
v4 = xxh32_round(v4, xxh_get32bits(input, align_));
|
|
input += 4;
|
|
}
|
|
while (input < limit);
|
|
|
|
h32 = rol(v1, 1) + rol(v2, 7) + rol(v3, 12) + rol(v4, 18);
|
|
}
|
|
else
|
|
{
|
|
h32 = seed + XXH_PRIME32_5;
|
|
}
|
|
|
|
h32 += cast(uint) len;
|
|
|
|
return xxh32_finalize(h32, input, len & 15, align_);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module ! */
|
|
/** Calculate a XXH32 digest on provided data
|
|
*
|
|
* Params:
|
|
* input = Pointer to data
|
|
* len = length of datablock
|
|
* seed = seed value
|
|
* Returns: a XXH32_hash_t
|
|
*/
|
|
private XXH32_hash_t XXH32(const void* input, size_t len, XXH32_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
static if (!XXH_NO_STREAM && XXH_SIZE_OPT >= 2)
|
|
{
|
|
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
|
XXH32_state_t state;
|
|
auto rc = xxh32_reset(&state, seed);
|
|
if (rc == XXH_errorcode.XXH_OK)
|
|
{
|
|
rc = xxh32_update(&state, cast(const(ubyte)*) input, len);
|
|
if (rc == XXH_errorcode.XXH_OK)
|
|
{
|
|
return xxh32_digest(&state);
|
|
}
|
|
}
|
|
return XXH_errorcode.XXH_ERROR;
|
|
}
|
|
else
|
|
{
|
|
if (XXH_FORCE_ALIGN_CHECK)
|
|
{
|
|
if (((cast(size_t) input) & 3) == 0)
|
|
{ /* Input is 4-bytes aligned, leverage the speed benefit */
|
|
return xxh32_endian_align(cast(const(ubyte)*) input, len,
|
|
seed, XXH_alignment.XXH_aligned);
|
|
}
|
|
}
|
|
|
|
return xxh32_endian_align(cast(const(ubyte)*) input, len, seed,
|
|
XXH_alignment.XXH_unaligned);
|
|
}
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
/** Reset state with seed
|
|
*
|
|
* Params:
|
|
* state = Pointer to state structure
|
|
* seed = A seed value
|
|
* Returns: XXH_errorcode.OK or XXH_errorcode.FAIL
|
|
*/
|
|
private XXH_errorcode xxh32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
in (statePtr != null, "statePtr is null")
|
|
{
|
|
*statePtr = XXH32_state_t.init;
|
|
statePtr.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
|
|
statePtr.v[1] = seed + XXH_PRIME32_2;
|
|
statePtr.v[2] = seed + 0;
|
|
statePtr.v[3] = seed - XXH_PRIME32_1;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
/** Update the state with some more data
|
|
*
|
|
* Params:
|
|
* state = Pointer to state structure
|
|
* input = A pointer to data
|
|
* len = length of input data
|
|
* Returns: XXH_errorcode.OK or XXH_errorcode.FAIL
|
|
*/
|
|
private XXH_errorcode xxh32_update(XXH32_state_t* state, const void* input, size_t len)
|
|
@trusted pure nothrow @nogc
|
|
in
|
|
{
|
|
if (input == null) assert(len == 0, "input null ptr only allowed with len == 0");
|
|
}
|
|
do
|
|
{
|
|
if (input == null && len == 0)
|
|
return XXH_errorcode.XXH_OK;
|
|
else if (input == null && len != 0)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
else
|
|
{
|
|
const(ubyte)* p = cast(const(ubyte)*) input;
|
|
const ubyte* bEnd = p + len;
|
|
|
|
state.total_len_32 += cast(XXH32_hash_t) len;
|
|
state.large_len |= cast(XXH32_hash_t)((len >= 16) | (state.total_len_32 >= 16));
|
|
|
|
if (state.memsize + len < 16)
|
|
{
|
|
/* fill in tmp buffer */
|
|
(cast(ubyte*) state.mem32)[state.memsize .. state.memsize + len] = (cast(ubyte*) input)[0 .. len];
|
|
state.memsize += cast(XXH32_hash_t) len;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
if (state.memsize)
|
|
{
|
|
/* some data left from previous update */
|
|
(cast(ubyte*) state.mem32)[state.memsize .. state.memsize + (16 - state.memsize)] =
|
|
(cast(ubyte*) input)[0 .. (16 - state.memsize)];
|
|
{
|
|
const(uint)* p32 = cast(const(uint)*)&state.mem32[0];
|
|
state.v[0] = xxh32_round(state.v[0], xxh_readLE32(p32));
|
|
p32++;
|
|
state.v[1] = xxh32_round(state.v[1], xxh_readLE32(p32));
|
|
p32++;
|
|
state.v[2] = xxh32_round(state.v[2], xxh_readLE32(p32));
|
|
p32++;
|
|
state.v[3] = xxh32_round(state.v[3], xxh_readLE32(p32));
|
|
}
|
|
p += 16 - state.memsize;
|
|
state.memsize = 0;
|
|
}
|
|
|
|
if (p <= bEnd - 16)
|
|
{
|
|
const ubyte* limit = bEnd - 16;
|
|
|
|
do
|
|
{
|
|
state.v[0] = xxh32_round(state.v[0], xxh_readLE32(p));
|
|
p += 4;
|
|
state.v[1] = xxh32_round(state.v[1], xxh_readLE32(p));
|
|
p += 4;
|
|
state.v[2] = xxh32_round(state.v[2], xxh_readLE32(p));
|
|
p += 4;
|
|
state.v[3] = xxh32_round(state.v[3], xxh_readLE32(p));
|
|
p += 4;
|
|
}
|
|
while (p <= limit);
|
|
|
|
}
|
|
|
|
if (p < bEnd)
|
|
{
|
|
(cast(ubyte*) state.mem32)[0 .. cast(size_t)(bEnd - p) ] =
|
|
(cast(ubyte*) p)[0 .. cast(size_t)(bEnd - p) ];
|
|
state.memsize = cast(XXH32_hash_t)(bEnd - p);
|
|
}
|
|
}
|
|
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
/** Finalize state and return the final XXH32 digest
|
|
*
|
|
* Params:
|
|
* state = Pointer to state structure
|
|
* Returns: the final XXH32 digest
|
|
*/
|
|
private XXH32_hash_t xxh32_digest(const XXH32_state_t* state)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
uint h32;
|
|
|
|
if (state.large_len)
|
|
{
|
|
h32 = rol(state.v[0], 1) + rol(state.v[1], 7) +
|
|
rol(state.v[2], 12) + rol(state.v[3], 18);
|
|
}
|
|
else
|
|
{
|
|
h32 = state.v[2] /* == seed */ + XXH_PRIME32_5;
|
|
}
|
|
|
|
h32 += state.total_len_32;
|
|
|
|
return xxh32_finalize(h32, cast(const ubyte*) state.mem32, state.memsize,
|
|
XXH_alignment.XXH_aligned);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
/** Covert the XXH32_hash_t to a byte array of same size
|
|
*
|
|
* Params:
|
|
* dst = Pointer to target storage
|
|
* hash = a XXH32_hash_t value
|
|
* Returns: nothing
|
|
*/
|
|
private void xxh32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
static assert((XXH32_canonical_t).sizeof == (XXH32_hash_t).sizeof,
|
|
"(XXH32_canonical_t).sizeof != (XXH32_hash_t).sizeof");
|
|
version (LittleEndian)
|
|
hash = bswap(hash);
|
|
(cast(ubyte*) dst) [0 .. dst.sizeof] = (cast(ubyte*) &hash) [0 .. dst.sizeof];
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
/** Covert the XXH32_hash_t to a byte array of same size
|
|
*
|
|
* Params:
|
|
* src = Pointer to source storage
|
|
* Returns: the converted value as XXH32_hash_t
|
|
*/
|
|
private XXH32_hash_t xxh32_hashFromCanonical(const XXH32_canonical_t* src) @safe pure nothrow @nogc
|
|
{
|
|
return xxh_readBE32(src);
|
|
}
|
|
|
|
/* ----------------------------------------------------------------------------------------*/
|
|
|
|
/* Helper functions to read 64bit data quantities from memory follow */
|
|
|
|
private ulong xxh_read64(const void* ptr)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
ulong val;
|
|
version (HaveUnalignedLoads)
|
|
val = *(cast(ulong*) ptr);
|
|
else
|
|
(cast(ubyte*)&val)[0 .. ulong.sizeof] = (cast(ubyte*) ptr)[0 .. ulong.sizeof];
|
|
return val;
|
|
}
|
|
|
|
private ulong xxh_readLE64(const void* ptr)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
version (LittleEndian)
|
|
return xxh_read64(ptr);
|
|
else
|
|
return bswap(xxh_read64(ptr));
|
|
}
|
|
|
|
private ulong xxh_readBE64(const void* ptr)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
version (LittleEndian)
|
|
return bswap(xxh_read64(ptr));
|
|
else
|
|
return xxh_read64(ptr);
|
|
}
|
|
|
|
private ulong xxh_readLE64_align(const void* ptr, XXH_alignment align_)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
if (align_ == XXH_alignment.XXH_unaligned)
|
|
{
|
|
return xxh_readLE64(ptr);
|
|
}
|
|
else
|
|
{
|
|
version (LittleEndian)
|
|
return *cast(const ulong*) ptr;
|
|
else
|
|
return bswap(*cast(const ulong*) ptr);
|
|
}
|
|
}
|
|
|
|
enum XXH_PRIME64_1 = 0x9E3779B185EBCA87; /** 0b1001111000110111011110011011000110000101111010111100101010000111 */
|
|
enum XXH_PRIME64_2 = 0xC2B2AE3D27D4EB4F; /** 0b1100001010110010101011100011110100100111110101001110101101001111 */
|
|
enum XXH_PRIME64_3 = 0x165667B19E3779F9; /** 0b0001011001010110011001111011000110011110001101110111100111111001 */
|
|
enum XXH_PRIME64_4 = 0x85EBCA77C2B2AE63; /** 0b1000010111101011110010100111011111000010101100101010111001100011 */
|
|
enum XXH_PRIME64_5 = 0x27D4EB2F165667C5; /** 0b0010011111010100111010110010111100010110010101100110011111000101 */
|
|
|
|
private ulong xxh64_round(ulong acc, ulong input)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
acc += input * XXH_PRIME64_2;
|
|
acc = rol(acc, 31);
|
|
acc *= XXH_PRIME64_1;
|
|
return acc;
|
|
}
|
|
|
|
private ulong xxh64_mergeRound(ulong acc, ulong val)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
val = xxh64_round(0, val);
|
|
acc ^= val;
|
|
acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
|
|
return acc;
|
|
}
|
|
|
|
private ulong xxh64_avalanche(ulong hash)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
hash ^= hash >> 33;
|
|
hash *= XXH_PRIME64_2;
|
|
hash ^= hash >> 29;
|
|
hash *= XXH_PRIME64_3;
|
|
hash ^= hash >> 32;
|
|
return hash;
|
|
}
|
|
|
|
ulong xxh_get64bits(const void* p, XXH_alignment align_)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh_readLE64_align(p, align_);
|
|
}
|
|
|
|
/** Processes the last 0-31 bytes of data at ptr addr.
|
|
*
|
|
* There may be up to 31 bytes remaining to consume from the input.
|
|
* This final stage will digest them to ensure that all input bytes are present
|
|
* in the final mix.
|
|
*
|
|
* Param: hash = The hash to finalize.
|
|
* Param: ptr = The pointer to the remaining input.
|
|
* Param: len = The remaining length, modulo 32.
|
|
* Param: align = Whether @p ptr is aligned.
|
|
* Return: The finalized hash
|
|
* See: xxh32_finalize().
|
|
*/
|
|
private ulong xxh64_finalize(ulong hash, const(ubyte)* ptr, size_t len, XXH_alignment align_)
|
|
@trusted pure nothrow @nogc
|
|
in
|
|
{
|
|
if (ptr == null) assert(len == 0, "input null ptr only allowed with len == 0");
|
|
}
|
|
do
|
|
{
|
|
len &= 31;
|
|
while (len >= 8)
|
|
{
|
|
const ulong k1 = xxh64_round(0, xxh_get64bits(ptr, align_));
|
|
ptr += 8;
|
|
hash ^= k1;
|
|
hash = rol(hash, 27) * XXH_PRIME64_1 + XXH_PRIME64_4;
|
|
len -= 8;
|
|
}
|
|
if (len >= 4)
|
|
{
|
|
hash ^= cast(ulong)(xxh_get32bits(ptr, align_)) * XXH_PRIME64_1;
|
|
ptr += 4;
|
|
hash = rol(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
|
|
len -= 4;
|
|
}
|
|
while (len > 0)
|
|
{
|
|
hash ^= (*ptr++) * XXH_PRIME64_5;
|
|
hash = rol(hash, 11) * XXH_PRIME64_1;
|
|
--len;
|
|
}
|
|
return xxh64_avalanche(hash);
|
|
}
|
|
|
|
/** The implementation for XXH64().
|
|
*
|
|
* Params:
|
|
* input = pointer to input data, directly passed from XXH64()
|
|
* len = length of input data, directly passed from XXH64()
|
|
* seed = Seed value, directly passed from XXH64()
|
|
* align = Whether input pointer is aligned.
|
|
* Return: The calculated hash.
|
|
*/
|
|
private ulong xxh64_endian_align(
|
|
const(ubyte)* input, size_t len,
|
|
ulong seed, XXH_alignment align_)
|
|
@trusted pure nothrow @nogc
|
|
in
|
|
{
|
|
if (input == null) assert(len == 0, "input null ptr only allowed with len == 0");
|
|
}
|
|
do
|
|
{
|
|
ulong h64;
|
|
|
|
if (len >= 32)
|
|
{
|
|
const ubyte* bEnd = input + len;
|
|
const ubyte* limit = bEnd - 31;
|
|
ulong v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
|
|
ulong v2 = seed + XXH_PRIME64_2;
|
|
ulong v3 = seed + 0;
|
|
ulong v4 = seed - XXH_PRIME64_1;
|
|
|
|
do
|
|
{
|
|
v1 = xxh64_round(v1, xxh_get64bits(input, align_));
|
|
input += 8;
|
|
v2 = xxh64_round(v2, xxh_get64bits(input, align_));
|
|
input += 8;
|
|
v3 = xxh64_round(v3, xxh_get64bits(input, align_));
|
|
input += 8;
|
|
v4 = xxh64_round(v4, xxh_get64bits(input, align_));
|
|
input += 8;
|
|
}
|
|
while (input < limit);
|
|
|
|
h64 = rol(v1, 1) + rol(v2, 7) + rol(v3, 12) + rol(v4, 18);
|
|
h64 = xxh64_mergeRound(h64, v1);
|
|
h64 = xxh64_mergeRound(h64, v2);
|
|
h64 = xxh64_mergeRound(h64, v3);
|
|
h64 = xxh64_mergeRound(h64, v4);
|
|
|
|
}
|
|
else
|
|
{
|
|
h64 = seed + XXH_PRIME64_5;
|
|
}
|
|
|
|
h64 += cast(ulong) len;
|
|
|
|
return xxh64_finalize(h64, input, len, align_);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
|
|
private XXH64_hash_t XXH64(const void* input, size_t len, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
static if (!XXH_NO_STREAM && XXH_SIZE_OPT >= 2)
|
|
{
|
|
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
|
|
XXH64_state_t state;
|
|
auto rc = xxh64_reset(&state, seed);
|
|
if (rc == XXH_errorcode.XXH_OK)
|
|
{
|
|
rc = xxh64_update(&state, cast(const(ubyte)*) input, len);
|
|
if (rc == XXH_errorcode.XXH_OK)
|
|
{
|
|
return xxh64_digest(&state);
|
|
}
|
|
}
|
|
return XXH_errorcode.XXH_ERROR;
|
|
}
|
|
else
|
|
{
|
|
if (XXH_FORCE_ALIGN_CHECK)
|
|
{
|
|
if (((cast(size_t) input) & 7) == 0)
|
|
{ /* Input is aligned, let's leverage the speed advantage */
|
|
return xxh64_endian_align(cast(const(ubyte)*) input, len,
|
|
seed, XXH_alignment.XXH_aligned);
|
|
}
|
|
}
|
|
|
|
return xxh64_endian_align(cast(const(ubyte)*) input, len, seed,
|
|
XXH_alignment.XXH_unaligned);
|
|
}
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh64_reset(XXH64_state_t* statePtr, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
assert(statePtr != null, "statePtr == null");
|
|
*statePtr = XXH64_state_t.init;
|
|
statePtr.v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
|
|
statePtr.v[1] = seed + XXH_PRIME64_2;
|
|
statePtr.v[2] = seed + 0;
|
|
statePtr.v[3] = seed - XXH_PRIME64_1;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh64_update(XXH64_state_t* state, const void* input, size_t len)
|
|
@trusted pure nothrow @nogc
|
|
in
|
|
{
|
|
if (input == null) assert(len == 0, "input null ptr only allowed with len == 0");
|
|
}
|
|
do
|
|
{
|
|
if (input == null && len == 0)
|
|
return XXH_errorcode.XXH_OK;
|
|
else if (input == null && len != 0)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
else
|
|
{
|
|
const(ubyte)* p = cast(const(ubyte)*) input;
|
|
const ubyte* bEnd = p + len;
|
|
|
|
state.total_len += len;
|
|
|
|
if (state.memsize + len < 32)
|
|
{
|
|
/* fill in tmp buffer */
|
|
(cast(ubyte*) state.mem64) [state.memsize .. state.memsize + len] =
|
|
(cast(ubyte*) input) [0 .. len];
|
|
state.memsize += cast(uint) len;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
if (state.memsize)
|
|
{
|
|
/* tmp buffer is full */
|
|
(cast(ubyte*) state.mem64) [state.memsize .. state.memsize + (32 - state.memsize)] =
|
|
(cast(ubyte*) input) [0 .. (32 - state.memsize)];
|
|
state.v[0] = xxh64_round(state.v[0], xxh_readLE64(&state.mem64[0]));
|
|
state.v[1] = xxh64_round(state.v[1], xxh_readLE64(&state.mem64[1]));
|
|
state.v[2] = xxh64_round(state.v[2], xxh_readLE64(&state.mem64[2]));
|
|
state.v[3] = xxh64_round(state.v[3], xxh_readLE64(&state.mem64[3]));
|
|
p += 32 - state.memsize;
|
|
state.memsize = 0;
|
|
}
|
|
|
|
if (p + 32 <= bEnd)
|
|
{
|
|
const ubyte* limit = bEnd - 32;
|
|
|
|
do
|
|
{
|
|
state.v[0] = xxh64_round(state.v[0], xxh_readLE64(p));
|
|
p += 8;
|
|
state.v[1] = xxh64_round(state.v[1], xxh_readLE64(p));
|
|
p += 8;
|
|
state.v[2] = xxh64_round(state.v[2], xxh_readLE64(p));
|
|
p += 8;
|
|
state.v[3] = xxh64_round(state.v[3], xxh_readLE64(p));
|
|
p += 8;
|
|
}
|
|
while (p <= limit);
|
|
|
|
}
|
|
|
|
if (p < bEnd)
|
|
{
|
|
(cast(void*) &state.mem64[0]) [0 .. cast(size_t) (bEnd - p)] =
|
|
(cast(void*) p) [0 .. cast(size_t) (bEnd - p)];
|
|
state.memsize = cast(XXH32_hash_t)(bEnd - p);
|
|
}
|
|
}
|
|
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH64_hash_t xxh64_digest(const XXH64_state_t* state)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
ulong h64;
|
|
|
|
if (state.total_len >= 32)
|
|
{
|
|
h64 = rol(state.v[0], 1) + rol(state.v[1],
|
|
7) + rol(state.v[2], 12) + rol(state.v[3], 18);
|
|
h64 = xxh64_mergeRound(h64, state.v[0]);
|
|
h64 = xxh64_mergeRound(h64, state.v[1]);
|
|
h64 = xxh64_mergeRound(h64, state.v[2]);
|
|
h64 = xxh64_mergeRound(h64, state.v[3]);
|
|
}
|
|
else
|
|
{
|
|
h64 = state.v[2] /*seed*/ + XXH_PRIME64_5;
|
|
}
|
|
|
|
h64 += cast(ulong) state.total_len;
|
|
|
|
return xxh64_finalize(h64, cast(const ubyte*) state.mem64,
|
|
cast(size_t) state.total_len, XXH_alignment.XXH_aligned);
|
|
}
|
|
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private void xxh64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
static assert((XXH64_canonical_t).sizeof == (XXH64_hash_t).sizeof,
|
|
"(XXH64_canonical_t).sizeof != (XXH64_hash_t).sizeof");
|
|
version (LittleEndian)
|
|
hash = bswap(hash);
|
|
(cast(ubyte*) dst) [0 .. dst.sizeof] = (cast(ubyte*) &hash) [0 .. dst.sizeof];
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH64_hash_t xxh64_hashFromCanonical(const XXH64_canonical_t* src)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh_readBE64(src);
|
|
}
|
|
|
|
/* *********************************************************************
|
|
* XXH3
|
|
* New generation hash designed for speed on small keys and vectorization
|
|
************************************************************************ */
|
|
|
|
enum XXH3_SECRET_SIZE_MIN = 136; /// The bare minimum size for a custom secret.
|
|
enum XXH3_SECRET_DEFAULT_SIZE = 192; /* minimum XXH3_SECRET_SIZE_MIN */
|
|
enum XXH_SECRET_DEFAULT_SIZE = 192; /* minimum XXH3_SECRET_SIZE_MIN */
|
|
enum XXH3_INTERNALBUFFER_SIZE = 256; ///The size of the internal XXH3 buffer.
|
|
|
|
/* Structure for XXH3 streaming API.
|
|
*
|
|
* Note: ** This structure has a strict alignment requirement of 64 bytes!! **
|
|
* Do not allocate this with `malloc()` or `new`, it will not be sufficiently aligned.
|
|
*
|
|
* Do never access the members of this struct directly.
|
|
*
|
|
* See: XXH3_INITSTATE() for stack initialization.
|
|
* See: XXH32_state_s, XXH64_state_s
|
|
*/
|
|
private align(64) struct XXH3_state_t
|
|
{
|
|
align(64) XXH64_hash_t[8] acc;
|
|
/** The 8 accumulators. See XXH32_state_s::v and XXH64_state_s::v */
|
|
align(64) ubyte[XXH3_SECRET_DEFAULT_SIZE] customSecret;
|
|
/** Used to store a custom secret generated from a seed. */
|
|
align(64) ubyte[XXH3_INTERNALBUFFER_SIZE] buffer;
|
|
/** The internal buffer. See: XXH32_state_s::mem32 */
|
|
XXH32_hash_t bufferedSize;
|
|
/** The amount of memory in buffer, See: XXH32_state_s::memsize */
|
|
XXH32_hash_t useSeed;
|
|
/** Reserved field. Needed for padding on 64-bit. */
|
|
size_t nbStripesSoFar;
|
|
/** Number or stripes processed. */
|
|
XXH64_hash_t totalLen;
|
|
/** Total length hashed. 64-bit even on 32-bit targets. */
|
|
size_t nbStripesPerBlock;
|
|
/** Number of stripes per block. */
|
|
size_t secretLimit;
|
|
/** Size of customSecret or extSecret */
|
|
XXH64_hash_t seed;
|
|
/** Seed for _withSeed variants. Must be zero otherwise, See: XXH3_INITSTATE() */
|
|
XXH64_hash_t reserved64;
|
|
/** Reserved field. */
|
|
const(ubyte)* extSecret;
|
|
/** Reference to an external secret for the _withSecret variants, null
|
|
* for other variants. */
|
|
/* note: there may be some padding at the end due to alignment on 64 bytes */
|
|
} /* typedef'd to XXH3_state_t */
|
|
|
|
static assert(XXH_SECRET_DEFAULT_SIZE >= XXH3_SECRET_SIZE_MIN, "default keyset is not large enough");
|
|
|
|
/** Pseudorandom secret taken directly from FARSH. */
|
|
private align(64) immutable ubyte[XXH3_SECRET_DEFAULT_SIZE] xxh3_kSecret = [
|
|
0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c,
|
|
0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb,
|
|
0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e,
|
|
0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
|
|
0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6,
|
|
0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb,
|
|
0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97,
|
|
0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
|
|
0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7,
|
|
0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31,
|
|
0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83,
|
|
0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
|
|
0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26,
|
|
0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc,
|
|
0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f,
|
|
0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
|
|
];
|
|
|
|
/* This performs a 32x32 -> 64 bit multiplikation */
|
|
private ulong xxh_mult32to64(uint x, uint y) @safe pure nothrow @nogc
|
|
{
|
|
return (cast(ulong)(x) * cast(ulong)(y));
|
|
}
|
|
|
|
/** Calculates a 64 to 128-bit long multiply.
|
|
*
|
|
* Param: lhs , rhs The 64-bit integers to be multiplied
|
|
* Return: The 128-bit result represented in an XXH128_hash_t structure.
|
|
*/
|
|
private XXH128_hash_t xxh_mult64to128(ulong lhs, ulong rhs) @safe pure nothrow @nogc
|
|
{
|
|
version (Have128BitInteger)
|
|
{
|
|
Cent cent_lhs; cent_lhs.lo = lhs;
|
|
Cent cent_rhs; cent_rhs.lo = rhs;
|
|
const Cent product = mul(cent_lhs, cent_rhs);
|
|
XXH128_hash_t r128;
|
|
r128.low64 = product.lo;
|
|
r128.high64 = product.hi;
|
|
}
|
|
else
|
|
{
|
|
/* First calculate all of the cross products. */
|
|
const ulong lo_lo = xxh_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
|
|
const ulong hi_lo = xxh_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
|
|
const ulong lo_hi = xxh_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
|
|
const ulong hi_hi = xxh_mult32to64(lhs >> 32, rhs >> 32);
|
|
|
|
/* Now add the products together. These will never overflow. */
|
|
const ulong cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
|
|
const ulong upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
|
|
const ulong lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
|
|
|
|
XXH128_hash_t r128;
|
|
r128.low64 = lower;
|
|
r128.high64 = upper;
|
|
}
|
|
return r128;
|
|
}
|
|
|
|
/** Calculates a 64-bit to 128-bit multiply, then XOR folds it.
|
|
*
|
|
* The reason for the separate function is to prevent passing too many structs
|
|
* around by value. This will hopefully inline the multiply, but we don't force it.
|
|
*
|
|
* Param: lhs , rhs The 64-bit integers to multiply
|
|
* Return: The low 64 bits of the product XOR'd by the high 64 bits.
|
|
* See: xxh_mult64to128()
|
|
*/
|
|
private ulong xxh3_mul128_fold64(ulong lhs, ulong rhs) @safe pure nothrow @nogc
|
|
{
|
|
XXH128_hash_t product = xxh_mult64to128(lhs, rhs);
|
|
return product.low64 ^ product.high64;
|
|
}
|
|
|
|
/* Seems to produce slightly better code on GCC for some reason. */
|
|
static private ulong xxh_xorshift64(ulong v64, int shift) @safe pure nothrow @nogc
|
|
in(0 <= shift && shift < 64, "shift out of range")
|
|
{
|
|
return v64 ^ (v64 >> shift);
|
|
}
|
|
|
|
/*
|
|
* This is a fast avalanche stage,
|
|
* suitable when input bits are already partially mixed
|
|
*/
|
|
static private XXH64_hash_t xxh3_avalanche(ulong h64) @safe pure nothrow @nogc
|
|
{
|
|
h64 = xxh_xorshift64(h64, 37);
|
|
h64 *= 0x165667919E3779F9;
|
|
h64 = xxh_xorshift64(h64, 32);
|
|
return h64;
|
|
}
|
|
|
|
/*
|
|
* This is a stronger avalanche,
|
|
* inspired by Pelle Evensen's rrmxmx
|
|
* preferable when input has not been previously mixed
|
|
*/
|
|
static private XXH64_hash_t xxh3_rrmxmx(ulong h64, ulong len) @safe pure nothrow @nogc
|
|
{
|
|
/* this mix is inspired by Pelle Evensen's rrmxmx */
|
|
h64 ^= rol(h64, 49) ^ rol(h64, 24);
|
|
h64 *= 0x9FB21C651E98DF25;
|
|
h64 ^= (h64 >> 35) + len;
|
|
h64 *= 0x9FB21C651E98DF25;
|
|
return xxh_xorshift64(h64, 28);
|
|
}
|
|
|
|
/* ==========================================
|
|
* Short keys
|
|
* ==========================================
|
|
* One of the shortcomings of XXH32 and XXH64 was that their performance was
|
|
* sub-optimal on short lengths. It used an iterative algorithm which strongly
|
|
* favored lengths that were a multiple of 4 or 8.
|
|
*
|
|
* Instead of iterating over individual inputs, we use a set of single shot
|
|
* functions which piece together a range of lengths and operate in constant time.
|
|
*
|
|
* Additionally, the number of multiplies has been significantly reduced. This
|
|
* reduces latency, especially when emulating 64-bit multiplies on 32-bit.
|
|
*
|
|
* Depending on the platform, this may or may not be faster than XXH32, but it
|
|
* is almost guaranteed to be faster than XXH64.
|
|
*/
|
|
|
|
/*
|
|
* At very short lengths, there isn't enough input to fully hide secrets, or use
|
|
* the entire secret.
|
|
*
|
|
* There is also only a limited amount of mixing we can do before significantly
|
|
* impacting performance.
|
|
*
|
|
* Therefore, we use different sections of the secret and always mix two secret
|
|
* samples with an XOR. This should have no effect on performance on the
|
|
* seedless or withSeed variants because everything _should_ be constant folded
|
|
* by modern compilers.
|
|
*
|
|
* The XOR mixing hides individual parts of the secret and increases entropy.
|
|
*
|
|
* This adds an extra layer of strength for custom secrets.
|
|
*/
|
|
private XXH64_hash_t xxh3_len_1to3_64b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(input != null, "input == null")
|
|
in(1 <= len && len <= 3, "len out of range")
|
|
in(secret != null, "secret == null")
|
|
{
|
|
/*
|
|
* len = 1: combined = { input[0], 0x01, input[0], input[0] }
|
|
* len = 2: combined = { input[1], 0x02, input[0], input[1] }
|
|
* len = 3: combined = { input[2], 0x03, input[0], input[1] }
|
|
*/
|
|
{
|
|
const ubyte c1 = input[0];
|
|
const ubyte c2 = input[len >> 1];
|
|
const ubyte c3 = input[len - 1];
|
|
const uint combined = (cast(uint) c1 << 16) | (
|
|
cast(uint) c2 << 24) | (cast(uint) c3 << 0) | (cast(uint) len << 8);
|
|
const ulong bitflip = (xxh_readLE32(secret) ^ xxh_readLE32(secret + 4)) + seed;
|
|
const ulong keyed = cast(ulong) combined ^ bitflip;
|
|
return xxh64_avalanche(keyed);
|
|
}
|
|
}
|
|
|
|
private XXH64_hash_t xxh3_len_4to8_64b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(input != null, "input == null")
|
|
in(secret != null, "secret == null")
|
|
in(4 <= len && len <= 8, "len out of range")
|
|
{
|
|
seed ^= cast(ulong) bswap(cast(uint) seed) << 32;
|
|
{
|
|
const uint input1 = xxh_readLE32(input);
|
|
const uint input2 = xxh_readLE32(input + len - 4);
|
|
const ulong bitflip = (xxh_readLE64(secret + 8) ^ xxh_readLE64(secret + 16)) - seed;
|
|
const ulong input64 = input2 + ((cast(ulong) input1) << 32);
|
|
const ulong keyed = input64 ^ bitflip;
|
|
return xxh3_rrmxmx(keyed, len);
|
|
}
|
|
}
|
|
|
|
private XXH64_hash_t xxh3_len_9to16_64b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(input != null, "input == null")
|
|
in(secret != null, "secret == null")
|
|
in(9 <= len && len <= 16, "len out of range")
|
|
{
|
|
{
|
|
const ulong bitflip1 = (xxh_readLE64(secret + 24) ^ xxh_readLE64(secret + 32)) + seed;
|
|
const ulong bitflip2 = (xxh_readLE64(secret + 40) ^ xxh_readLE64(secret + 48)) - seed;
|
|
const ulong input_lo = xxh_readLE64(input) ^ bitflip1;
|
|
const ulong input_hi = xxh_readLE64(input + len - 8) ^ bitflip2;
|
|
const ulong acc = len + bswap(input_lo) + input_hi + xxh3_mul128_fold64(input_lo,
|
|
input_hi);
|
|
return xxh3_avalanche(acc);
|
|
}
|
|
}
|
|
|
|
private bool xxh_likely(bool exp)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return exp;
|
|
}
|
|
|
|
private bool xxh_unlikely(bool exp)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return exp;
|
|
}
|
|
|
|
private XXH64_hash_t xxh3_len_0to16_64b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(len <= 16, "len > 16")
|
|
{
|
|
{
|
|
if (xxh_likely(len > 8))
|
|
return xxh3_len_9to16_64b(input, len, secret, seed);
|
|
if (xxh_likely(len >= 4))
|
|
return xxh3_len_4to8_64b(input, len, secret, seed);
|
|
if (len)
|
|
return xxh3_len_1to3_64b(input, len, secret, seed);
|
|
return xxh64_avalanche(seed ^ (xxh_readLE64(secret + 56) ^ xxh_readLE64(secret + 64)));
|
|
}
|
|
}
|
|
|
|
/*
|
|
* DISCLAIMER: There are known *seed-dependent* multicollisions here due to
|
|
* multiplication by zero, affecting hashes of lengths 17 to 240.
|
|
*
|
|
* However, they are very unlikely.
|
|
*
|
|
* Keep this in mind when using the unseeded xxh3_64bits() variant: As with all
|
|
* unseeded non-cryptographic hashes, it does not attempt to defend itself
|
|
* against specially crafted inputs, only random inputs.
|
|
*
|
|
* Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
|
|
* cancelling out the secret is taken an arbitrary number of times (addressed
|
|
* in xxh3_accumulate_512), this collision is very unlikely with random inputs
|
|
* and/or proper seeding:
|
|
*
|
|
* This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
|
|
* function that is only called up to 16 times per hash with up to 240 bytes of
|
|
* input.
|
|
*
|
|
* This is not too bad for a non-cryptographic hash function, especially with
|
|
* only 64 bit outputs.
|
|
*
|
|
* The 128-bit variant (which trades some speed for strength) is NOT affected
|
|
* by this, although it is always a good idea to use a proper seed if you care
|
|
* about strength.
|
|
*/
|
|
private ulong xxh3_mix16B(const(ubyte)* input, const(ubyte)* secret, ulong seed64)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
{
|
|
const ulong input_lo = xxh_readLE64(input);
|
|
const ulong input_hi = xxh_readLE64(input + 8);
|
|
return xxh3_mul128_fold64(
|
|
input_lo ^ (xxh_readLE64(secret) + seed64),
|
|
input_hi ^ (xxh_readLE64(secret + 8) - seed64));
|
|
}
|
|
}
|
|
|
|
/* For mid range keys, XXH3 uses a Mum-hash variant. */
|
|
private XXH64_hash_t xxh3_len_17to128_64b(
|
|
const(ubyte)* input, size_t len, const(ubyte)* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN")
|
|
in(16 < len && len <= 128, "len out of range")
|
|
{
|
|
ulong acc = len * XXH_PRIME64_1;
|
|
if (len > 32)
|
|
{
|
|
if (len > 64)
|
|
{
|
|
if (len > 96)
|
|
{
|
|
acc += xxh3_mix16B(input + 48, secret + 96, seed);
|
|
acc += xxh3_mix16B(input + len - 64, secret + 112, seed);
|
|
}
|
|
acc += xxh3_mix16B(input + 32, secret + 64, seed);
|
|
acc += xxh3_mix16B(input + len - 48, secret + 80, seed);
|
|
}
|
|
acc += xxh3_mix16B(input + 16, secret + 32, seed);
|
|
acc += xxh3_mix16B(input + len - 32, secret + 48, seed);
|
|
}
|
|
acc += xxh3_mix16B(input + 0, secret + 0, seed);
|
|
acc += xxh3_mix16B(input + len - 16, secret + 16, seed);
|
|
|
|
return xxh3_avalanche(acc);
|
|
}
|
|
|
|
enum XXH3_MIDSIZE_MAX = 240;
|
|
enum XXH3_MIDSIZE_STARTOFFSET = 3;
|
|
enum XXH3_MIDSIZE_LASTOFFSET = 17;
|
|
|
|
private XXH64_hash_t xxh3_len_129to240_64b(
|
|
const(ubyte)* input, size_t len, const(ubyte)* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN")
|
|
in { const int nbRounds = cast(int) len / 16; assert(nbRounds >= 8, "nbRounds < 8"); }
|
|
in(128 < len && len <= XXH3_MIDSIZE_MAX, "128 >= len || len > XXH3_MIDSIZE_MAX")
|
|
{
|
|
ulong acc = len * XXH_PRIME64_1;
|
|
const int nbRounds = cast(int) len / 16;
|
|
int i;
|
|
for (i = 0; i < 8; i++)
|
|
{
|
|
acc += xxh3_mix16B(input + (16 * i), secret + (16 * i), seed);
|
|
}
|
|
acc = xxh3_avalanche(acc);
|
|
for (i = 8; i < nbRounds; i++)
|
|
{
|
|
acc += xxh3_mix16B(input + (16 * i),
|
|
secret + (16 * (i - 8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
|
|
}
|
|
/* last bytes */
|
|
acc += xxh3_mix16B(input + len - 16,
|
|
secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
|
|
return xxh3_avalanche(acc);
|
|
}
|
|
|
|
/* ======= Long Keys ======= */
|
|
|
|
enum XXH_STRIPE_LEN = 64;
|
|
enum XXH_SECRET_CONSUME_RATE = 8; /* nb of secret bytes consumed at each accumulation */
|
|
enum XXH_ACC_NB = (XXH_STRIPE_LEN / (ulong).sizeof);
|
|
|
|
private void xxh_writeLE64(void* dst, ulong v64)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
version (LittleEndian) {}
|
|
else
|
|
v64 = bswap(v64);
|
|
(cast(ubyte*) dst) [0 .. v64.sizeof] = (cast(ubyte*) &v64) [0 .. v64.sizeof];
|
|
}
|
|
|
|
/* scalar variants - universal */
|
|
|
|
enum XXH_ACC_ALIGN = 8;
|
|
|
|
/* Scalar round for xxh3_accumulate_512_scalar(). */
|
|
private void xxh3_scalarRound(void* acc, const(void)* input, const(void)* secret, size_t lane)
|
|
@trusted pure nothrow @nogc
|
|
in(lane < XXH_ACC_NB, "lane >= XXH_ACC_NB")
|
|
{
|
|
version (CheckACCAlignment)
|
|
assert((cast(size_t) acc & (XXH_ACC_ALIGN - 1)) == 0, "(cast(size_t) acc & (XXH_ACC_ALIGN - 1)) != 0");
|
|
ulong* xacc = cast(ulong*) acc;
|
|
ubyte* xinput = cast(ubyte*) input;
|
|
ubyte* xsecret = cast(ubyte*) secret;
|
|
{
|
|
const ulong data_val = xxh_readLE64(xinput + lane * 8);
|
|
const ulong data_key = data_val ^ xxh_readLE64(xsecret + lane * 8);
|
|
xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
|
|
xacc[lane] += xxh_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32);
|
|
}
|
|
}
|
|
|
|
/* Processes a 64 byte block of data using the scalar path. */
|
|
private void xxh3_accumulate_512_scalar(void* acc, const(void)* input, const(void)* secret)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
size_t i;
|
|
for (i = 0; i < XXH_ACC_NB; i++)
|
|
{
|
|
xxh3_scalarRound(acc, input, secret, i);
|
|
}
|
|
}
|
|
|
|
/* Scalar scramble step for xxh3_scrambleAcc_scalar().
|
|
*
|
|
* This is extracted to its own function because the NEON path uses a combination
|
|
* of NEON and scalar.
|
|
*/
|
|
private void xxh3_scalarScrambleRound(void* acc, const(void)* secret, size_t lane)
|
|
@trusted pure nothrow @nogc
|
|
in(lane < XXH_ACC_NB, "lane >= XXH_ACC_NB")
|
|
{
|
|
version (CheckACCAlignment)
|
|
assert(((cast(size_t) acc) & (XXH_ACC_ALIGN - 1)) == 0, "((cast(size_t) acc) & (XXH_ACC_ALIGN - 1)) != 0");
|
|
ulong* xacc = cast(ulong*) acc; /* presumed aligned */
|
|
const ubyte* xsecret = cast(const ubyte*) secret; /* no alignment restriction */
|
|
{
|
|
const ulong key64 = xxh_readLE64(xsecret + lane * 8);
|
|
ulong acc64 = xacc[lane];
|
|
acc64 = xxh_xorshift64(acc64, 47);
|
|
acc64 ^= key64;
|
|
acc64 *= XXH_PRIME32_1;
|
|
xacc[lane] = acc64;
|
|
}
|
|
}
|
|
|
|
/* Scrambles the accumulators after a large chunk has been read */
|
|
private void xxh3_scrambleAcc_scalar(void* acc, const(void)* secret)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
size_t i;
|
|
for (i = 0; i < XXH_ACC_NB; i++)
|
|
{
|
|
xxh3_scalarScrambleRound(acc, secret, i);
|
|
}
|
|
}
|
|
|
|
private void xxh3_initCustomSecret_scalar(void* customSecret, ulong seed64)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
/*
|
|
* We need a separate pointer for the hack below,
|
|
* which requires a non-const pointer.
|
|
* Any decent compiler will optimize this out otherwise.
|
|
*/
|
|
const ubyte* kSecretPtr = cast(ubyte*) xxh3_kSecret;
|
|
static assert((XXH_SECRET_DEFAULT_SIZE & 15) == 0, "(XXH_SECRET_DEFAULT_SIZE & 15) != 0");
|
|
|
|
/*
|
|
* Note: in debug mode, this overrides the asm optimization
|
|
* and Clang will emit MOVK chains again.
|
|
*/
|
|
//assert(kSecretPtr == xxh3_kSecret);
|
|
|
|
{
|
|
const int nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
|
|
int i;
|
|
for (i = 0; i < nbRounds; i++)
|
|
{
|
|
/*
|
|
* The asm hack causes Clang to assume that kSecretPtr aliases with
|
|
* customSecret, and on aarch64, this prevented LDP from merging two
|
|
* loads together for free. Putting the loads together before the stores
|
|
* properly generates LDP.
|
|
*/
|
|
const ulong lo = xxh_readLE64(kSecretPtr + 16 * i) + seed64;
|
|
const ulong hi = xxh_readLE64(kSecretPtr + 16 * i + 8) - seed64;
|
|
xxh_writeLE64(cast(ubyte*) customSecret + 16 * i, lo);
|
|
xxh_writeLE64(cast(ubyte*) customSecret + 16 * i + 8, hi);
|
|
}
|
|
}
|
|
}
|
|
|
|
alias XXH3_f_accumulate_512 = void function(void*, const(void)*, const(void)*) @safe pure nothrow @nogc;
|
|
alias XXH3_f_scrambleAcc = void function(void*, const void*) @safe pure nothrow @nogc;
|
|
alias XXH3_f_initCustomSecret = void function(void*, ulong) @safe pure nothrow @nogc;
|
|
|
|
immutable XXH3_f_accumulate_512 xxh3_accumulate_512 = &xxh3_accumulate_512_scalar;
|
|
immutable XXH3_f_scrambleAcc xxh3_scrambleAcc = &xxh3_scrambleAcc_scalar;
|
|
immutable XXH3_f_initCustomSecret xxh3_initCustomSecret = &xxh3_initCustomSecret_scalar;
|
|
|
|
enum XXH_PREFETCH_DIST = 384;
|
|
/* TODO: Determine how to implement prefetching in D! Disabled for now */
|
|
private void XXH_PREFETCH(const ubyte* ptr) @safe pure nothrow @nogc
|
|
{
|
|
// cast(void)(ptr); /* DISABLED prefetch and do nothing here */
|
|
|
|
// In C it is done with the following code lines:
|
|
// if XXH_SIZE_OPT >= 1
|
|
// define XXH_PREFETCH(ptr) (void)(ptr)
|
|
// elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
|
|
// include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
|
|
// define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
|
|
// elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
|
|
// define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw == read */, 3 /* locality */)
|
|
// else
|
|
// define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
|
|
// endif
|
|
}
|
|
|
|
/*
|
|
* xxh3_accumulate()
|
|
* Loops over xxh3_accumulate_512().
|
|
* Assumption: nbStripes will not overflow the secret size
|
|
*/
|
|
private void xxh3_accumulate(
|
|
ulong* acc, const ubyte* input,
|
|
const ubyte* secret, size_t nbStripes, XXH3_f_accumulate_512 f_acc512)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
size_t n;
|
|
for (n = 0; n < nbStripes; n++)
|
|
{
|
|
const ubyte* in_ = input + n * XXH_STRIPE_LEN;
|
|
XXH_PREFETCH(in_ + XXH_PREFETCH_DIST);
|
|
f_acc512(acc, in_, secret + n * XXH_SECRET_CONSUME_RATE);
|
|
}
|
|
}
|
|
|
|
private void xxh3_hashLong_internal_loop(
|
|
ulong* acc, const ubyte* input, size_t len, const ubyte* secret,
|
|
size_t secretSize, XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble)
|
|
@trusted pure nothrow @nogc
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN")
|
|
in(len > XXH_STRIPE_LEN, "len <= XXH_STRIPE_LEN")
|
|
{
|
|
const size_t nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
|
|
const size_t block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
|
|
const size_t nb_blocks = (len - 1) / block_len;
|
|
|
|
size_t n;
|
|
|
|
for (n = 0; n < nb_blocks; n++)
|
|
{
|
|
xxh3_accumulate(acc, input + n * block_len, secret, nbStripesPerBlock, f_acc512);
|
|
f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
|
|
}
|
|
|
|
/* last partial block */
|
|
{
|
|
const size_t nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
|
|
assert(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE),
|
|
"nbStripes > (secretSize / XXH_SECRET_CONSUME_RATE)");
|
|
xxh3_accumulate(acc, input + nb_blocks * block_len, secret, nbStripes, f_acc512);
|
|
|
|
/* last stripe */
|
|
{
|
|
const ubyte* p = input + len - XXH_STRIPE_LEN;
|
|
f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
|
|
}
|
|
}
|
|
}
|
|
|
|
enum XXH_SECRET_LASTACC_START = 7; /* not aligned on 8, last secret is different from acc & scrambler */
|
|
|
|
private ulong xxh3_mix2Accs(const(ulong)* acc, const(ubyte)* secret)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
return xxh3_mul128_fold64(acc[0] ^ xxh_readLE64(secret), acc[1] ^ xxh_readLE64(secret + 8));
|
|
}
|
|
|
|
private XXH64_hash_t xxh3_mergeAccs(const(ulong)* acc, const(ubyte)* secret, ulong start)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
ulong result64 = start;
|
|
size_t i = 0;
|
|
|
|
for (i = 0; i < 4; i++)
|
|
{
|
|
result64 += xxh3_mix2Accs(acc + 2 * i, secret + 16 * i);
|
|
}
|
|
|
|
return xxh3_avalanche(result64);
|
|
}
|
|
|
|
static immutable XXH3_INIT_ACC = [
|
|
XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3,
|
|
XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1
|
|
];
|
|
|
|
private XXH64_hash_t xxh3_hashLong_64b_internal(
|
|
const(void)* input, size_t len, const(void)* secret, size_t secretSize,
|
|
XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
align(XXH_ACC_ALIGN) ulong[XXH_ACC_NB] acc = XXH3_INIT_ACC; /* NOTE: This doesn't work in D, fails on 32bit NetBSD */
|
|
|
|
xxh3_hashLong_internal_loop(&acc[0], cast(const(ubyte)*) input, len,
|
|
cast(const(ubyte)*) secret, secretSize, f_acc512, f_scramble);
|
|
|
|
/* converge into final hash */
|
|
static assert(acc.sizeof == 64, "acc.sizeof != 64");
|
|
/* do not align on 8, so that the secret is different from the accumulator */
|
|
assert(secretSize >= acc.sizeof + XXH_SECRET_MERGEACCS_START,
|
|
"secretSize < acc.sizeof + XXH_SECRET_MERGEACCS_START");
|
|
return xxh3_mergeAccs(&acc[0], cast(const(ubyte)*) secret + XXH_SECRET_MERGEACCS_START,
|
|
cast(ulong) len * XXH_PRIME64_1);
|
|
}
|
|
|
|
enum XXH_SECRET_MERGEACCS_START = 11;
|
|
|
|
/*
|
|
* It's important for performance to transmit secret's size (when it's static)
|
|
* so that the compiler can properly optimize the vectorized loop.
|
|
* This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
|
|
*/
|
|
private XXH64_hash_t xxh3_hashLong_64b_withSecret(
|
|
const(void)* input, size_t len, XXH64_hash_t seed64, const(ubyte)* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_64b_internal(input, len, secret, secretLen,
|
|
xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
/*
|
|
* It's preferable for performance that XXH3_hashLong is not inlined,
|
|
* as it results in a smaller function for small data, easier to the instruction cache.
|
|
* Note that inside this no_inline function, we do inline the internal loop,
|
|
* and provide a statically defined secret size to allow optimization of vector loop.
|
|
*/
|
|
private XXH64_hash_t xxh3_hashLong_64b_default(
|
|
const(void)* input, size_t len, XXH64_hash_t seed64, const(ubyte)* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_64b_internal(input, len, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
enum XXH_SEC_ALIGN = 8;
|
|
|
|
/*
|
|
* xxh3_hashLong_64b_withSeed():
|
|
* Generate a custom key based on alteration of default xxh3_kSecret with the seed,
|
|
* and then use this key for long mode hashing.
|
|
*
|
|
* This operation is decently fast but nonetheless costs a little bit of time.
|
|
* Try to avoid it whenever possible (typically when seed == 0).
|
|
*
|
|
* It's important for performance that XXH3_hashLong is not inlined. Not sure
|
|
* why (uop cache maybe?), but the difference is large and easily measurable.
|
|
*/
|
|
private XXH64_hash_t xxh3_hashLong_64b_withSeed_internal(
|
|
const(void)* input, size_t len, XXH64_hash_t seed,
|
|
XXH3_f_accumulate_512 f_acc512,
|
|
XXH3_f_scrambleAcc f_scramble,
|
|
XXH3_f_initCustomSecret f_initSec)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
//#if XXH_SIZE_OPT <= 0
|
|
if (seed == 0)
|
|
return xxh3_hashLong_64b_internal(input, len, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, f_acc512, f_scramble);
|
|
//#endif
|
|
else
|
|
{
|
|
align(XXH_SEC_ALIGN) ubyte[XXH_SECRET_DEFAULT_SIZE] secret;
|
|
f_initSec(&secret[0], seed);
|
|
return xxh3_hashLong_64b_internal(input, len, &secret[0],
|
|
(secret).sizeof, f_acc512, f_scramble);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* It's important for performance that XXH3_hashLong is not inlined.
|
|
*/
|
|
private XXH64_hash_t xxh3_hashLong_64b_withSeed(const(void)* input, size_t len,
|
|
XXH64_hash_t seed, const(ubyte)* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_64b_withSeed_internal(input, len, seed,
|
|
xxh3_accumulate_512, xxh3_scrambleAcc, xxh3_initCustomSecret);
|
|
}
|
|
|
|
alias XXH3_hashLong64_f = XXH64_hash_t function(
|
|
const(void)*, size_t, XXH64_hash_t, const(ubyte)*, size_t)
|
|
@safe pure nothrow @nogc;
|
|
|
|
private XXH64_hash_t xxh3_64bits_internal(const(void)* input, size_t len,
|
|
XXH64_hash_t seed64, const(void)* secret, size_t secretLen, XXH3_hashLong64_f f_hashLong)
|
|
@safe pure nothrow @nogc
|
|
in(secretLen >= XXH3_SECRET_SIZE_MIN, "secretLen < XXH3_SECRET_SIZE_MIN")
|
|
{
|
|
/*
|
|
* If an action is to be taken if `secretLen` condition is not respected,
|
|
* it should be done here.
|
|
* For now, it's a contract pre-condition.
|
|
* Adding a check and a branch here would cost performance at every hash.
|
|
* Also, note that function signature doesn't offer room to return an error.
|
|
*/
|
|
if (len <= 16)
|
|
return xxh3_len_0to16_64b(cast(const(ubyte)*) input, len,
|
|
cast(const(ubyte)*) secret, seed64);
|
|
if (len <= 128)
|
|
return xxh3_len_17to128_64b(cast(const(ubyte)*) input, len,
|
|
cast(const(ubyte)*) secret, secretLen, seed64);
|
|
if (len <= XXH3_MIDSIZE_MAX)
|
|
return xxh3_len_129to240_64b(cast(const(ubyte)*) input, len,
|
|
cast(const(ubyte)*) secret, secretLen, seed64);
|
|
return f_hashLong(input, len, seed64, cast(const(ubyte)*) secret, secretLen);
|
|
}
|
|
|
|
/* === Public entry point === */
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH64_hash_t xxh3_64bits(const(void)* input, size_t length)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_internal(input, length, 0, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, &xxh3_hashLong_64b_default);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH64_hash_t xxh3_64bits_withSecret(
|
|
const(void)* input, size_t length, const(void)* secret, size_t secretSize)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_internal(input, length, 0, secret, secretSize,
|
|
&xxh3_hashLong_64b_withSecret);
|
|
}
|
|
|
|
XXH64_hash_t xxh3_64bits_withSeed(const(void)* input, size_t length, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_internal(input, length, seed, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, &xxh3_hashLong_64b_withSeed);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH64_hash_t xxh3_64bits_withSecretandSeed(
|
|
const(void)* input, size_t length, const(void)* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (length <= XXH3_MIDSIZE_MAX)
|
|
return xxh3_64bits_internal(input, length, seed, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, null);
|
|
return xxh3_hashLong_64b_withSecret(input, length, seed,
|
|
cast(const(ubyte)*) secret, secretSize);
|
|
}
|
|
|
|
/* === XXH3 streaming === */
|
|
|
|
private void XXH3_INITSTATE(XXH3_state_t* XXH3_state_ptr)
|
|
@safe nothrow @nogc
|
|
{
|
|
(XXH3_state_ptr).seed = 0;
|
|
}
|
|
|
|
private void xxh3_reset_internal(
|
|
scope XXH3_state_t* statePtr, XXH64_hash_t seed, const void* secret, size_t secretSize)
|
|
@trusted pure nothrow @nogc
|
|
in
|
|
{
|
|
const size_t initStart = XXH3_state_t.bufferedSize.offsetof;
|
|
assert(XXH3_state_t.nbStripesPerBlock.offsetof > initStart,
|
|
"(XXH3_state_t.nbStripesPerBlock.offsetof <= initStart");
|
|
}
|
|
in(statePtr != null, "statePtr == null")
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN")
|
|
{
|
|
/* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
|
|
*statePtr = XXH3_state_t.init;
|
|
statePtr.acc[0] = XXH_PRIME32_3;
|
|
statePtr.acc[1] = XXH_PRIME64_1;
|
|
statePtr.acc[2] = XXH_PRIME64_2;
|
|
statePtr.acc[3] = XXH_PRIME64_3;
|
|
statePtr.acc[4] = XXH_PRIME64_4;
|
|
statePtr.acc[5] = XXH_PRIME32_2;
|
|
statePtr.acc[6] = XXH_PRIME64_5;
|
|
statePtr.acc[7] = XXH_PRIME32_1;
|
|
statePtr.seed = seed;
|
|
statePtr.useSeed = (seed != 0);
|
|
statePtr.extSecret = cast(const(ubyte)*) secret;
|
|
statePtr.secretLimit = secretSize - XXH_STRIPE_LEN;
|
|
statePtr.nbStripesPerBlock = statePtr.secretLimit / XXH_SECRET_CONSUME_RATE;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH_errorcode xxh3_64bits_reset(scope XXH3_state_t* statePtr)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (statePtr == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
xxh3_reset_internal(statePtr, 0, &xxh3_kSecret[0], XXH_SECRET_DEFAULT_SIZE);
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_64bits_reset_withSecret(
|
|
XXH3_state_t* statePtr, const void* secret, size_t secretSize)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (statePtr == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
xxh3_reset_internal(statePtr, 0, secret, secretSize);
|
|
if (secret == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
if (secretSize < XXH3_SECRET_SIZE_MIN)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (statePtr == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
if (seed == 0)
|
|
return xxh3_64bits_reset(statePtr);
|
|
if ((seed != statePtr.seed) || (statePtr.extSecret != null))
|
|
xxh3_initCustomSecret(&statePtr.customSecret[0], seed);
|
|
xxh3_reset_internal(statePtr, seed, null, XXH_SECRET_DEFAULT_SIZE);
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_64bits_reset_withSecretandSeed(
|
|
XXH3_state_t* statePtr, const(void)* secret, size_t secretSize, XXH64_hash_t seed64)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (statePtr == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
if (secret == null)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
if (secretSize < XXH3_SECRET_SIZE_MIN)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
xxh3_reset_internal(statePtr, seed64, secret, secretSize);
|
|
statePtr.useSeed = 1; /* always, even if seed64 == 0 */
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* Note : when xxh3_consumeStripes() is invoked,
|
|
* there must be a guarantee that at least one more byte must be consumed from input
|
|
* so that the function can blindly consume all stripes using the "normal" secret segment */
|
|
private void xxh3_consumeStripes(
|
|
ulong* acc, size_t* nbStripesSoFarPtr, size_t nbStripesPerBlock, const ubyte* input, size_t nbStripes,
|
|
const ubyte* secret, size_t secretLimit,
|
|
XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble)
|
|
@trusted pure nothrow @nogc
|
|
in(nbStripes <= nbStripesPerBlock, "nbStripes > nbStripesPerBlock") /* can handle max 1 scramble per invocation */
|
|
in(*nbStripesSoFarPtr < nbStripesPerBlock, "*nbStripesSoFarPtr >= nbStripesPerBlock")
|
|
{
|
|
if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes)
|
|
{
|
|
/* need a scrambling operation */
|
|
const size_t nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr;
|
|
const size_t nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock;
|
|
xxh3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE,
|
|
nbStripesToEndofBlock, f_acc512);
|
|
f_scramble(acc, secret + secretLimit);
|
|
xxh3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN,
|
|
secret, nbStripesAfterBlock, f_acc512);
|
|
*nbStripesSoFarPtr = nbStripesAfterBlock;
|
|
}
|
|
else
|
|
{
|
|
xxh3_accumulate(acc, input,
|
|
secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512);
|
|
*nbStripesSoFarPtr += nbStripes;
|
|
}
|
|
}
|
|
|
|
enum XXH3_STREAM_USE_STACK = 1;
|
|
/*
|
|
* Both xxh3_64bits_update and xxh3_128bits_update use this routine.
|
|
*/
|
|
private XXH_errorcode xxh3_update(
|
|
scope XXH3_state_t* state, scope const(ubyte)* input, size_t len,
|
|
XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble)
|
|
@trusted pure nothrow @nogc
|
|
in(state != null, "state == null")
|
|
in
|
|
{
|
|
if (input == null) assert(len == 0, "input null ptr only allowed with len == 0");
|
|
}
|
|
do
|
|
{
|
|
if (input == null && len == 0)
|
|
return XXH_errorcode.XXH_OK;
|
|
else if (input == null && len != 0)
|
|
return XXH_errorcode.XXH_ERROR;
|
|
else
|
|
{
|
|
const ubyte* bEnd = input + len;
|
|
const(ubyte)* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0];
|
|
static if (XXH3_STREAM_USE_STACK >= 1)
|
|
{
|
|
/* For some reason, gcc and MSVC seem to suffer greatly
|
|
* when operating accumulators directly into state.
|
|
* Operating into stack space seems to enable proper optimization.
|
|
* clang, on the other hand, doesn't seem to need this trick */
|
|
align(XXH_ACC_ALIGN) ulong[8] acc;
|
|
(cast(ubyte*) &acc[0]) [0 .. acc.sizeof] = (cast(ubyte*) &state.acc[0]) [0 .. acc.sizeof];
|
|
}
|
|
else
|
|
{
|
|
ulong* acc = state.acc;
|
|
}
|
|
state.totalLen += len;
|
|
assert(state.bufferedSize <= XXH3_INTERNALBUFFER_SIZE, "state.bufferedSize > XXH3_INTERNALBUFFER_SIZE");
|
|
|
|
/* small input : just fill in tmp buffer */
|
|
if (state.bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE)
|
|
{
|
|
(cast(ubyte*) &state.buffer[0]) [state.bufferedSize .. state.bufferedSize + len] =
|
|
(cast(ubyte*) input) [0 .. len];
|
|
state.bufferedSize += cast(XXH32_hash_t) len;
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* total input is now > XXH3_INTERNALBUFFER_SIZE */
|
|
enum XXH3_INTERNALBUFFER_STRIPES = (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN);
|
|
static assert(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0, "XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN != 0"); /* clean multiple */
|
|
|
|
/*
|
|
* Internal buffer is partially filled (always, except at beginning)
|
|
* Complete it, then consume it.
|
|
*/
|
|
if (state.bufferedSize)
|
|
{
|
|
const size_t loadSize = XXH3_INTERNALBUFFER_SIZE - state.bufferedSize;
|
|
(cast(ubyte*)&state.buffer[0]) [state.bufferedSize .. state.bufferedSize + loadSize] =
|
|
(cast(ubyte*) input) [0 .. loadSize];
|
|
input += loadSize;
|
|
xxh3_consumeStripes(&acc[0], &state.nbStripesSoFar, state.nbStripesPerBlock,
|
|
&state.buffer[0], XXH3_INTERNALBUFFER_STRIPES, secret,
|
|
state.secretLimit, f_acc512, f_scramble);
|
|
state.bufferedSize = 0;
|
|
}
|
|
assert(input < bEnd, "input >= bEnd");
|
|
|
|
/* large input to consume : ingest per full block */
|
|
if (cast(size_t)(bEnd - input) > state.nbStripesPerBlock * XXH_STRIPE_LEN)
|
|
{
|
|
size_t nbStripes = cast(size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
|
|
assert(state.nbStripesPerBlock >= state.nbStripesSoFar, "state.nbStripesPerBlock < state.nbStripesSoFar");
|
|
/* join to current block's end */
|
|
{
|
|
const size_t nbStripesToEnd = state.nbStripesPerBlock - state.nbStripesSoFar;
|
|
assert(nbStripesToEnd <= nbStripes, "nbStripesToEnd > nbStripes");
|
|
xxh3_accumulate(&acc[0], input,
|
|
secret + state.nbStripesSoFar * XXH_SECRET_CONSUME_RATE,
|
|
nbStripesToEnd, f_acc512);
|
|
f_scramble(&acc[0], secret + state.secretLimit);
|
|
state.nbStripesSoFar = 0;
|
|
input += nbStripesToEnd * XXH_STRIPE_LEN;
|
|
nbStripes -= nbStripesToEnd;
|
|
}
|
|
/* consume per entire blocks */
|
|
while (nbStripes >= state.nbStripesPerBlock)
|
|
{
|
|
xxh3_accumulate(&acc[0], input, secret, state.nbStripesPerBlock, f_acc512);
|
|
f_scramble(&acc[0], secret + state.secretLimit);
|
|
input += state.nbStripesPerBlock * XXH_STRIPE_LEN;
|
|
nbStripes -= state.nbStripesPerBlock;
|
|
}
|
|
/* consume last partial block */
|
|
xxh3_accumulate(&acc[0], input, secret, nbStripes, f_acc512);
|
|
input += nbStripes * XXH_STRIPE_LEN;
|
|
assert(input < bEnd, "input exceeds buffer, no bytes left"); /* at least some bytes left */
|
|
state.nbStripesSoFar = nbStripes;
|
|
/* buffer predecessor of last partial stripe */
|
|
(cast(ubyte*) &state.buffer[0])
|
|
[state.buffer.sizeof - XXH_STRIPE_LEN .. state.buffer.sizeof - XXH_STRIPE_LEN + XXH_STRIPE_LEN] =
|
|
(cast(ubyte*) input - XXH_STRIPE_LEN) [0 .. XXH_STRIPE_LEN];
|
|
assert(bEnd - input <= XXH_STRIPE_LEN, "input exceed strip length");
|
|
}
|
|
else
|
|
{
|
|
/* content to consume <= block size */
|
|
/* Consume input by a multiple of internal buffer size */
|
|
if (bEnd - input > XXH3_INTERNALBUFFER_SIZE)
|
|
{
|
|
const ubyte* limit = bEnd - XXH3_INTERNALBUFFER_SIZE;
|
|
do
|
|
{
|
|
xxh3_consumeStripes(&acc[0], &state.nbStripesSoFar, state.nbStripesPerBlock, input,
|
|
XXH3_INTERNALBUFFER_STRIPES, secret,
|
|
state.secretLimit, f_acc512, f_scramble);
|
|
input += XXH3_INTERNALBUFFER_SIZE;
|
|
}
|
|
while (input < limit);
|
|
/* buffer predecessor of last partial stripe */
|
|
(cast(ubyte*) &state.buffer[0])
|
|
[state.buffer.sizeof - XXH_STRIPE_LEN .. state.buffer.sizeof - XXH_STRIPE_LEN + XXH_STRIPE_LEN] =
|
|
(cast(ubyte*) input - XXH_STRIPE_LEN) [0 .. XXH_STRIPE_LEN];
|
|
}
|
|
}
|
|
|
|
/* Some remaining input (always) : buffer it */
|
|
assert(input < bEnd, "input exceeds buffer");
|
|
assert(bEnd - input <= XXH3_INTERNALBUFFER_SIZE, "input outside buffer");
|
|
assert(state.bufferedSize == 0, "bufferedSize != 0");
|
|
(cast(ubyte*) &state.buffer[0]) [0 .. cast(size_t)(bEnd - input)] =
|
|
(cast(ubyte*) input) [0 .. cast(size_t)(bEnd - input)];
|
|
state.bufferedSize = cast(XXH32_hash_t)(bEnd - input);
|
|
static if (XXH3_STREAM_USE_STACK >= 1)
|
|
{
|
|
/* save stack accumulators into state */
|
|
(cast(ubyte*) &state.acc[0]) [0 .. acc.sizeof] = (cast(ubyte*) &acc[0]) [0 .. acc.sizeof];
|
|
}
|
|
}
|
|
|
|
return XXH_errorcode.XXH_OK;
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_64bits_update(scope XXH3_state_t* state, scope const(void)* input, size_t len)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_update(state, cast(const(ubyte)*) input, len,
|
|
xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
private void xxh3_digest_long(XXH64_hash_t* acc, const XXH3_state_t* state, const ubyte* secret)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
/*
|
|
* Digest on a local copy. This way, the state remains unaltered, and it can
|
|
* continue ingesting more input afterwards.
|
|
*/
|
|
(cast(ubyte*) &acc[0]) [0 .. state.acc.sizeof] = (cast(ubyte*) &state.acc[0]) [0 .. state.acc.sizeof];
|
|
if (state.bufferedSize >= XXH_STRIPE_LEN)
|
|
{
|
|
const size_t nbStripes = (state.bufferedSize - 1) / XXH_STRIPE_LEN;
|
|
size_t nbStripesSoFar = state.nbStripesSoFar;
|
|
xxh3_consumeStripes(acc, &nbStripesSoFar, state.nbStripesPerBlock, &state.buffer[0],
|
|
nbStripes, secret, state.secretLimit, xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
/* last stripe */
|
|
xxh3_accumulate_512(acc, &state.buffer[0] + state.bufferedSize - XXH_STRIPE_LEN,
|
|
secret + state.secretLimit - XXH_SECRET_LASTACC_START);
|
|
}
|
|
else
|
|
{ /* bufferedSize < XXH_STRIPE_LEN */
|
|
ubyte[XXH_STRIPE_LEN] lastStripe;
|
|
const size_t catchupSize = XXH_STRIPE_LEN - state.bufferedSize;
|
|
assert(state.bufferedSize > 0, "bufferedSize <= 0"); /* there is always some input buffered */
|
|
(cast(ubyte*) &lastStripe[0]) [0 .. catchupSize] =
|
|
(cast(ubyte*) &state.buffer[0]) [state.buffer.sizeof - catchupSize .. state.buffer.sizeof];
|
|
(cast(ubyte*) &lastStripe[0]) [catchupSize .. catchupSize + state.bufferedSize] =
|
|
(cast(ubyte*) &state.buffer[0]) [0 .. state.buffer.sizeof];
|
|
xxh3_accumulate_512(&acc[0], &lastStripe[0], &secret[0] + state.secretLimit - XXH_SECRET_LASTACC_START);
|
|
}
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH64_hash_t xxh3_64bits_digest(const XXH3_state_t* state)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
const ubyte* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0];
|
|
if (state.totalLen > XXH3_MIDSIZE_MAX)
|
|
{
|
|
align(XXH_ACC_ALIGN) XXH64_hash_t[XXH_ACC_NB] acc;
|
|
xxh3_digest_long(&acc[0], state, secret);
|
|
return xxh3_mergeAccs(&acc[0], secret + XXH_SECRET_MERGEACCS_START,
|
|
cast(ulong) state.totalLen * XXH_PRIME64_1);
|
|
}
|
|
/* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
|
|
if (state.useSeed)
|
|
return xxh3_64bits_withSeed(&state.buffer[0], cast(size_t) state.totalLen, state.seed);
|
|
return xxh3_64bits_withSecret(&state.buffer[0],
|
|
cast(size_t)(state.totalLen), secret, state.secretLimit + XXH_STRIPE_LEN);
|
|
}
|
|
|
|
/* ==========================================
|
|
* XXH3 128 bits (a.k.a XXH128)
|
|
* ==========================================
|
|
* XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
|
|
* even without counting the significantly larger output size.
|
|
*
|
|
* For example, extra steps are taken to avoid the seed-dependent collisions
|
|
* in 17-240 byte inputs (See xxh3_mix16B and xxh128_mix32B).
|
|
*
|
|
* This strength naturally comes at the cost of some speed, especially on short
|
|
* lengths. Note that longer hashes are about as fast as the 64-bit version
|
|
* due to it using only a slight modification of the 64-bit loop.
|
|
*
|
|
* XXH128 is also more oriented towards 64-bit machines. It is still extremely
|
|
* fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
|
|
*/
|
|
private XXH128_hash_t xxh3_len_1to3_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
/* A doubled version of 1to3_64b with different constants. */
|
|
assert(input != null, "input is null");
|
|
assert(1 <= len && len <= 3, "len is out of range");
|
|
assert(secret != null, "secret is null");
|
|
/*
|
|
* len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
|
|
* len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
|
|
* len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
|
|
*/
|
|
{
|
|
const ubyte c1 = input[0];
|
|
const ubyte c2 = input[len >> 1];
|
|
const ubyte c3 = input[len - 1];
|
|
const uint combinedl = (cast(uint) c1 << 16) | (
|
|
cast(uint) c2 << 24) | (cast(uint) c3 << 0) | (cast(uint) len << 8);
|
|
const uint combinedh = rol(bswap(combinedl), 13);
|
|
const ulong bitflipl = (xxh_readLE32(secret) ^ xxh_readLE32(secret + 4)) + seed;
|
|
const ulong bitfliph = (xxh_readLE32(secret + 8) ^ xxh_readLE32(secret + 12)) - seed;
|
|
const ulong keyed_lo = cast(ulong) combinedl ^ bitflipl;
|
|
const ulong keyed_hi = cast(ulong) combinedh ^ bitfliph;
|
|
XXH128_hash_t h128;
|
|
h128.low64 = xxh64_avalanche(keyed_lo);
|
|
h128.high64 = xxh64_avalanche(keyed_hi);
|
|
return h128;
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_len_4to8_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
assert(input != null, "input is null");
|
|
assert(secret != null, "secret is null");
|
|
assert(4 <= len && len <= 8, "len is out of range");
|
|
seed ^= cast(ulong) bswap(cast(uint) seed) << 32;
|
|
{
|
|
const uint input_lo = xxh_readLE32(input);
|
|
const uint input_hi = xxh_readLE32(input + len - 4);
|
|
const ulong input_64 = input_lo + (cast(ulong) input_hi << 32);
|
|
const ulong bitflip = (xxh_readLE64(secret + 16) ^ xxh_readLE64(secret + 24)) + seed;
|
|
const ulong keyed = input_64 ^ bitflip;
|
|
|
|
/* Shift len to the left to ensure it is even, this avoids even multiplies. */
|
|
XXH128_hash_t m128 = xxh_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
|
|
|
|
m128.high64 += (m128.low64 << 1);
|
|
m128.low64 ^= (m128.high64 >> 3);
|
|
|
|
m128.low64 = xxh_xorshift64(m128.low64, 35);
|
|
m128.low64 *= 0x9FB21C651E98DF25;
|
|
m128.low64 = xxh_xorshift64(m128.low64, 28);
|
|
m128.high64 = xxh3_avalanche(m128.high64);
|
|
return m128;
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_len_9to16_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
assert(input != null, "input is null");
|
|
assert(secret != null, "secret is null");
|
|
assert(9 <= len && len <= 16, "len out of range");
|
|
{
|
|
const ulong bitflipl = (xxh_readLE64(secret + 32) ^ xxh_readLE64(secret + 40)) - seed;
|
|
const ulong bitfliph = (xxh_readLE64(secret + 48) ^ xxh_readLE64(secret + 56)) + seed;
|
|
const ulong input_lo = xxh_readLE64(input);
|
|
ulong input_hi = xxh_readLE64(input + len - 8);
|
|
XXH128_hash_t m128 = xxh_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
|
|
/*
|
|
* Put len in the middle of m128 to ensure that the length gets mixed to
|
|
* both the low and high bits in the 128x64 multiply below.
|
|
*/
|
|
m128.low64 += cast(ulong)(len - 1) << 54;
|
|
input_hi ^= bitfliph;
|
|
/*
|
|
* Add the high 32 bits of input_hi to the high 32 bits of m128, then
|
|
* add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
|
|
* the high 64 bits of m128.
|
|
*
|
|
* The best approach to this operation is different on 32-bit and 64-bit.
|
|
*/
|
|
if ((void*).sizeof < (ulong).sizeof)
|
|
{ /* 32-bit */
|
|
/*
|
|
* 32-bit optimized version, which is more readable.
|
|
*
|
|
* On 32-bit, it removes an ADC and delays a dependency between the two
|
|
* halves of m128.high64, but it generates an extra mask on 64-bit.
|
|
*/
|
|
m128.high64 += (input_hi & 0xFFFFFFFF00000000) + xxh_mult32to64(cast(uint) input_hi,
|
|
XXH_PRIME32_2);
|
|
}
|
|
else
|
|
{
|
|
/*
|
|
* 64-bit optimized (albeit more confusing) version.
|
|
*
|
|
* Uses some properties of addition and multiplication to remove the mask:
|
|
*
|
|
* Let:
|
|
* a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
|
|
* b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
|
|
* c = XXH_PRIME32_2
|
|
*
|
|
* a + (b * c)
|
|
* Inverse Property: x + y - x == y
|
|
* a + (b * (1 + c - 1))
|
|
* Distributive Property: x * (y + z) == (x * y) + (x * z)
|
|
* a + (b * 1) + (b * (c - 1))
|
|
* Identity Property: x * 1 == x
|
|
* a + b + (b * (c - 1))
|
|
*
|
|
* Substitute a, b, and c:
|
|
* input_hi.hi + input_hi.lo + ((ulong)input_hi.lo * (XXH_PRIME32_2 - 1))
|
|
*
|
|
* Since input_hi.hi + input_hi.lo == input_hi, we get this:
|
|
* input_hi + ((ulong)input_hi.lo * (XXH_PRIME32_2 - 1))
|
|
*/
|
|
m128.high64 += input_hi + xxh_mult32to64(cast(uint) input_hi, XXH_PRIME32_2 - 1);
|
|
}
|
|
/* m128 ^= bswap(m128 >> 64); */
|
|
m128.low64 ^= bswap(m128.high64);
|
|
|
|
{ /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
|
|
XXH128_hash_t h128 = xxh_mult64to128(m128.low64, XXH_PRIME64_2);
|
|
h128.high64 += m128.high64 * XXH_PRIME64_2;
|
|
|
|
h128.low64 = xxh3_avalanche(h128.low64);
|
|
h128.high64 = xxh3_avalanche(h128.high64);
|
|
return h128;
|
|
}
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_len_0to16_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
assert(len <= 16, "len > 16");
|
|
{
|
|
if (len > 8)
|
|
return xxh3_len_9to16_128b(input, len, secret, seed);
|
|
if (len >= 4)
|
|
return xxh3_len_4to8_128b(input, len, secret, seed);
|
|
if (len)
|
|
return xxh3_len_1to3_128b(input, len, secret, seed);
|
|
{
|
|
XXH128_hash_t h128;
|
|
const ulong bitflipl = xxh_readLE64(secret + 64) ^ xxh_readLE64(secret + 72);
|
|
const ulong bitfliph = xxh_readLE64(secret + 80) ^ xxh_readLE64(secret + 88);
|
|
h128.low64 = xxh64_avalanche(seed ^ bitflipl);
|
|
h128.high64 = xxh64_avalanche(seed ^ bitfliph);
|
|
return h128;
|
|
}
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh128_mix32B(
|
|
XXH128_hash_t acc, const ubyte* input_1, const ubyte* input_2, const ubyte* secret, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
acc.low64 += xxh3_mix16B(input_1, secret + 0, seed);
|
|
acc.low64 ^= xxh_readLE64(input_2) + xxh_readLE64(input_2 + 8);
|
|
acc.high64 += xxh3_mix16B(input_2, secret + 16, seed);
|
|
acc.high64 ^= xxh_readLE64(input_1) + xxh_readLE64(input_1 + 8);
|
|
return acc;
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_len_17to128_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSie < XXH3_SECRET_SIZE_MIN")
|
|
in(16 < len && len <= 128, "len out of range")
|
|
{
|
|
XXH128_hash_t acc;
|
|
acc.low64 = len * XXH_PRIME64_1;
|
|
acc.high64 = 0;
|
|
|
|
static if (XXH_SIZE_OPT >= 1)
|
|
{
|
|
/* Smaller, but slightly slower. */
|
|
size_t i = (len - 1) / 32;
|
|
do
|
|
{
|
|
acc = xxh128_mix32B(acc, input + 16 * i,
|
|
input + len - 16 * (i + 1), secret + 32 * i, seed);
|
|
}
|
|
while (i-- != 0);
|
|
}
|
|
else
|
|
{
|
|
if (len > 32)
|
|
{
|
|
if (len > 64)
|
|
{
|
|
if (len > 96)
|
|
{
|
|
acc = xxh128_mix32B(acc, input + 48, input + len - 64, secret + 96, seed);
|
|
}
|
|
acc = xxh128_mix32B(acc, input + 32, input + len - 48, secret + 64, seed);
|
|
}
|
|
acc = xxh128_mix32B(acc, input + 16, input + len - 32, secret + 32, seed);
|
|
}
|
|
acc = xxh128_mix32B(acc, input, input + len - 16, secret, seed);
|
|
}
|
|
{
|
|
XXH128_hash_t h128;
|
|
h128.low64 = acc.low64 + acc.high64;
|
|
h128.high64 = (acc.low64 * XXH_PRIME64_1) + (
|
|
acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2);
|
|
h128.low64 = xxh3_avalanche(h128.low64);
|
|
h128.high64 = cast(XXH64_hash_t) 0 - xxh3_avalanche(h128.high64);
|
|
return h128;
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_len_129to240_128b(
|
|
const ubyte* input, size_t len, const ubyte* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@trusted pure nothrow @nogc
|
|
in(secretSize >= XXH3_SECRET_SIZE_MIN, "secretSize < XXH3_SECRET_SIZE_MIN")
|
|
in(128 < len && len <= XXH3_MIDSIZE_MAX, "len > 128 or len > XXH3_MIDSIZE_MAX")
|
|
{
|
|
XXH128_hash_t acc;
|
|
const int nbRounds = cast(int) len / 32;
|
|
int i;
|
|
acc.low64 = len * XXH_PRIME64_1;
|
|
acc.high64 = 0;
|
|
for (i = 0; i < 4; i++)
|
|
{
|
|
acc = xxh128_mix32B(acc, input + (32 * i), input + (32 * i) + 16, secret + (32 * i),
|
|
seed);
|
|
}
|
|
acc.low64 = xxh3_avalanche(acc.low64);
|
|
acc.high64 = xxh3_avalanche(acc.high64);
|
|
assert(nbRounds >= 4, "nbRounds < 4");
|
|
for (i = 4; i < nbRounds; i++)
|
|
{
|
|
acc = xxh128_mix32B(acc, input + (32 * i), input + (32 * i) + 16,
|
|
secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), seed);
|
|
}
|
|
/* last bytes */
|
|
acc = xxh128_mix32B(acc, input + len - 16, input + len - 32,
|
|
secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, 0 - seed);
|
|
|
|
{
|
|
XXH128_hash_t h128;
|
|
h128.low64 = acc.low64 + acc.high64;
|
|
h128.high64 = (acc.low64 * XXH_PRIME64_1) + (
|
|
acc.high64 * XXH_PRIME64_4) + ((len - seed) * XXH_PRIME64_2);
|
|
h128.low64 = xxh3_avalanche(h128.low64);
|
|
h128.high64 = cast(XXH64_hash_t) 0 - xxh3_avalanche(h128.high64);
|
|
return h128;
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_hashLong_128b_internal(
|
|
const void* input, size_t len, const ubyte* secret, size_t secretSize,
|
|
XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
align(XXH_ACC_ALIGN) ulong[XXH_ACC_NB] acc = XXH3_INIT_ACC;
|
|
|
|
xxh3_hashLong_internal_loop(&acc[0], cast(const ubyte*) input, len,
|
|
secret, secretSize, f_acc512, f_scramble);
|
|
|
|
/* converge into final hash */
|
|
static assert(acc.sizeof == 64, "acc isn't 64 bytes long");
|
|
assert(secretSize >= acc.sizeof + XXH_SECRET_MERGEACCS_START, "secretSze < allowed limit.");
|
|
{
|
|
XXH128_hash_t h128;
|
|
h128.low64 = xxh3_mergeAccs(&acc[0],
|
|
secret + XXH_SECRET_MERGEACCS_START, cast(ulong) len * XXH_PRIME64_1);
|
|
h128.high64 = xxh3_mergeAccs(&acc[0], secret + secretSize - (acc)
|
|
.sizeof - XXH_SECRET_MERGEACCS_START, ~(cast(ulong) len * XXH_PRIME64_2));
|
|
return h128;
|
|
}
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_hashLong_128b_default(
|
|
const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_128b_internal(input, len, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
/*
|
|
* It's important for performance to pass @p secretLen (when it's static)
|
|
* to the compiler, so that it can properly optimize the vectorized loop.
|
|
*/
|
|
private XXH128_hash_t xxh3_hashLong_128b_withSecret(
|
|
const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_128b_internal(input, len, cast(const ubyte*) secret,
|
|
secretLen, xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
private XXH128_hash_t xxh3_hashLong_128b_withSeed_internal(
|
|
const void* input, size_t len, XXH64_hash_t seed64,
|
|
XXH3_f_accumulate_512 f_acc512, XXH3_f_scrambleAcc f_scramble,
|
|
XXH3_f_initCustomSecret f_initSec)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
if (seed64 == 0)
|
|
return xxh3_hashLong_128b_internal(input, len, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, f_acc512, f_scramble);
|
|
{
|
|
align(XXH_SEC_ALIGN) ubyte[XXH_SECRET_DEFAULT_SIZE] secret;
|
|
f_initSec(&secret[0], seed64);
|
|
return xxh3_hashLong_128b_internal(input, len,
|
|
cast(const ubyte*)&secret[0], (secret).sizeof, f_acc512, f_scramble);
|
|
}
|
|
}
|
|
/*
|
|
* It's important for performance that XXH3_hashLong is not inlined.
|
|
*/
|
|
private XXH128_hash_t xxh3_hashLong_128b_withSeed(
|
|
const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_hashLong_128b_withSeed_internal(input, len, seed64,
|
|
xxh3_accumulate_512, xxh3_scrambleAcc, xxh3_initCustomSecret);
|
|
}
|
|
|
|
alias XXH3_hashLong128_f = XXH128_hash_t function(const void*, size_t,
|
|
XXH64_hash_t, const void*, size_t) @safe pure nothrow @nogc;
|
|
|
|
private XXH128_hash_t xxh3_128bits_internal(
|
|
const void* input, size_t len, XXH64_hash_t seed64, const void* secret, size_t secretLen,
|
|
XXH3_hashLong128_f f_hl128)
|
|
@safe pure nothrow @nogc
|
|
in(secretLen >= XXH3_SECRET_SIZE_MIN, "Secret length is < XXH3_SECRET_SIZE_MIN")
|
|
{
|
|
/*
|
|
* If an action is to be taken if `secret` conditions are not respected,
|
|
* it should be done here.
|
|
* For now, it's a contract pre-condition.
|
|
* Adding a check and a branch here would cost performance at every hash.
|
|
*/
|
|
if (len <= 16)
|
|
return xxh3_len_0to16_128b(cast(const ubyte*) input, len,
|
|
cast(const ubyte*) secret, seed64);
|
|
if (len <= 128)
|
|
return xxh3_len_17to128_128b(cast(const ubyte*) input, len,
|
|
cast(const ubyte*) secret, secretLen, seed64);
|
|
if (len <= XXH3_MIDSIZE_MAX)
|
|
return xxh3_len_129to240_128b(cast(const ubyte*) input, len,
|
|
cast(const ubyte*) secret, secretLen, seed64);
|
|
return f_hl128(input, len, seed64, secret, secretLen);
|
|
}
|
|
|
|
/* === Public XXH128 API === */
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH128_hash_t xxh3_128bits(const void* input, size_t len)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_128bits_internal(input, len, 0, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, &xxh3_hashLong_128b_default);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH128_hash_t xxh3_128bits_withSecret(
|
|
const void* input, size_t len, const void* secret, size_t secretSize)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_128bits_internal(input, len, 0, cast(const ubyte*) secret,
|
|
secretSize, &xxh3_hashLong_128b_withSecret);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH128_hash_t xxh3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_128bits_internal(input, len, seed, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, &xxh3_hashLong_128b_withSeed);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH128_hash_t xxh3_128bits_withSecretandSeed(
|
|
const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
if (len <= XXH3_MIDSIZE_MAX)
|
|
return xxh3_128bits_internal(input, len, seed, &xxh3_kSecret[0],
|
|
(xxh3_kSecret).sizeof, null);
|
|
return xxh3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH128_hash_t XXH128(const void* input, size_t len, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_128bits_withSeed(input, len, seed);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_128bits_reset(scope XXH3_state_t* statePtr)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_reset(statePtr);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */
|
|
private XXH_errorcode xxh3_128bits_reset_withSecret(
|
|
XXH3_state_t* statePtr, const void* secret, size_t secretSize)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_reset_withSecret(statePtr, secret, secretSize);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH_errorcode xxh3_128bits_reset_withSeed(
|
|
XXH3_state_t* statePtr, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_reset_withSeed(statePtr, seed);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH_errorcode xxh3_128bits_reset_withSecretandSeed(
|
|
XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH_errorcode xxh3_128bits_update(
|
|
scope XXH3_state_t* state, scope const void* input, size_t len)
|
|
@safe pure nothrow @nogc
|
|
{
|
|
return xxh3_update(state, cast(const ubyte*) input, len,
|
|
xxh3_accumulate_512, xxh3_scrambleAcc);
|
|
}
|
|
|
|
/* XXH PUBLIC API - hidden in D module */ private
|
|
XXH128_hash_t xxh3_128bits_digest(const XXH3_state_t* state)
|
|
@trusted pure nothrow @nogc
|
|
{
|
|
const ubyte* secret = (state.extSecret == null) ? &state.customSecret[0] : &state.extSecret[0];
|
|
if (state.totalLen > XXH3_MIDSIZE_MAX)
|
|
{
|
|
align(XXH_ACC_ALIGN) XXH64_hash_t[XXH_ACC_NB] acc;
|
|
xxh3_digest_long(&acc[0], state, secret);
|
|
assert(state.secretLimit + XXH_STRIPE_LEN >= acc.sizeof + XXH_SECRET_MERGEACCS_START, "Internal error");
|
|
{
|
|
XXH128_hash_t h128;
|
|
h128.low64 = xxh3_mergeAccs(&acc[0], secret + XXH_SECRET_MERGEACCS_START,
|
|
cast(ulong) state.totalLen * XXH_PRIME64_1);
|
|
h128.high64 = xxh3_mergeAccs(&acc[0], secret + state.secretLimit + XXH_STRIPE_LEN - (acc)
|
|
.sizeof - XXH_SECRET_MERGEACCS_START,
|
|
~(cast(ulong) state.totalLen * XXH_PRIME64_2));
|
|
return h128;
|
|
}
|
|
}
|
|
/* len <= XXH3_MIDSIZE_MAX : short code */
|
|
if (state.seed)
|
|
return xxh3_128bits_withSeed(&state.buffer[0], cast(size_t) state.totalLen, state.seed);
|
|
return xxh3_128bits_withSecret(&state.buffer[0],
|
|
cast(size_t)(state.totalLen), secret, state.secretLimit + XXH_STRIPE_LEN);
|
|
}
|
|
|
|
/* ----------------------------------------------------------------------------------------*/
|
|
|
|
import core.bitop;
|
|
|
|
public import std.digest;
|
|
|
|
/*
|
|
* Helper methods for encoding the buffer.
|
|
* Can be removed if the optimizer can inline the methods from std.bitmanip.
|
|
*/
|
|
version (LittleEndian)
|
|
{
|
|
private alias nativeToBigEndian = bswap;
|
|
private alias bigEndianToNative = bswap;
|
|
}
|
|
else
|
|
pragma(inline, true) private pure @nogc nothrow @safe
|
|
{
|
|
uint nativeToBigEndian(uint val)
|
|
{
|
|
return val;
|
|
}
|
|
|
|
ulong nativeToBigEndian(ulong val)
|
|
{
|
|
return val;
|
|
}
|
|
|
|
alias bigEndianToNative = nativeToBigEndian;
|
|
}
|
|
|
|
/**
|
|
* Template API XXHTemplate implementation. Uses parameters to configure for number of bits and XXH variant (classic or XXH3)
|
|
* See `std.digest` for differences between template and OOP API.
|
|
*/
|
|
struct XXHTemplate(HASH, STATE, bool useXXH3)
|
|
{
|
|
private:
|
|
HASH hash;
|
|
STATE state;
|
|
HASH seed = HASH.init;
|
|
|
|
public:
|
|
enum digestSize = HASH.sizeof * 8;
|
|
|
|
/**
|
|
* Use this to feed the digest with data.
|
|
* Also implements the $(REF isOutputRange, std,range,primitives)
|
|
* interface for `ubyte` and `const(ubyte)[]`.
|
|
*
|
|
* Example:
|
|
* ----
|
|
* XXHTemplate!(hashtype,statetype,useXXH3) dig;
|
|
* dig.put(cast(ubyte) 0); //single ubyte
|
|
* dig.put(cast(ubyte) 0, cast(ubyte) 0); //variadic
|
|
* ubyte[10] buf;
|
|
* dig.put(buf); //buffer
|
|
* ----
|
|
*/
|
|
void put(scope const(ubyte)[] data...) @safe nothrow @nogc
|
|
{
|
|
XXH_errorcode ec = XXH_errorcode.XXH_OK;
|
|
if (data.length > 0) // digest will only change, when there is data!
|
|
{
|
|
static if (digestSize == 32)
|
|
ec = xxh32_update(&state, &data[0], data.length);
|
|
else static if (digestSize == 64 && !useXXH3)
|
|
ec = xxh64_update(&state, &data[0], data.length);
|
|
else static if (digestSize == 64 && useXXH3)
|
|
ec = xxh3_64bits_update(&state, &data[0], data.length);
|
|
else static if (digestSize == 128)
|
|
ec = xxh3_128bits_update(&state, &data[0], data.length);
|
|
else
|
|
assert(false, "Unknown XXH bitdeep or variant");
|
|
}
|
|
assert(ec == XXH_errorcode.XXH_OK, "Update failed");
|
|
}
|
|
|
|
/**
|
|
* Used to (re)initialize the XXHTemplate digest.
|
|
*
|
|
* Example:
|
|
* --------
|
|
* XXHTemplate!(hashtype,statetype,useXXH3) digest;
|
|
* digest.start();
|
|
* digest.put(0);
|
|
* --------
|
|
*/
|
|
void start() @safe nothrow @nogc
|
|
{
|
|
this = typeof(this).init;
|
|
XXH_errorcode ec;
|
|
static if (digestSize == 32)
|
|
{
|
|
assert(state.alignof == uint.alignof, "Wrong alignment for state structure");
|
|
ec = xxh32_reset(&state, seed);
|
|
}
|
|
else static if (digestSize == 64 && !useXXH3)
|
|
{
|
|
assert(state.alignof == ulong.alignof, "Wrong alignment for state structure");
|
|
ec = xxh64_reset(&state, seed);
|
|
}
|
|
else static if (digestSize == 64 && useXXH3)
|
|
{
|
|
assert(state.alignof == 64, "Wrong alignment for state structure");
|
|
ec = xxh3_64bits_reset(&state);
|
|
}
|
|
else static if (digestSize == 128)
|
|
{
|
|
assert(state.alignof == 64, "Wrong alignment for state structure");
|
|
ec = xxh3_128bits_reset(&state);
|
|
}
|
|
else
|
|
assert(false, "Unknown XXH bitdeep or variant");
|
|
//assert(ec == XXH_errorcode.XXH_OK, "reset failed");
|
|
}
|
|
|
|
/**
|
|
* Returns the finished XXH hash. This also calls $(LREF start) to
|
|
* reset the internal state.
|
|
*/
|
|
ubyte[digestSize / 8] finish() @trusted nothrow @nogc
|
|
{
|
|
static if (digestSize == 32)
|
|
{
|
|
hash = xxh32_digest(&state);
|
|
const auto rc = nativeToBigEndian(hash);
|
|
}
|
|
else static if (digestSize == 64 && !useXXH3)
|
|
{
|
|
hash = xxh64_digest(&state);
|
|
const auto rc = nativeToBigEndian(hash);
|
|
}
|
|
else static if (digestSize == 64 && useXXH3)
|
|
{
|
|
hash = xxh3_64bits_digest(&state);
|
|
const auto rc = nativeToBigEndian(hash);
|
|
}
|
|
else static if (digestSize == 128)
|
|
{
|
|
hash = xxh3_128bits_digest(&state);
|
|
HASH rc;
|
|
// Note: low64 and high64 are intentionally exchanged!
|
|
rc.low64 = nativeToBigEndian(hash.high64);
|
|
rc.high64 = nativeToBigEndian(hash.low64);
|
|
}
|
|
|
|
return (cast(ubyte*)&rc)[0 .. rc.sizeof];
|
|
}
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
// Simple example using the XXH_64 digest
|
|
XXHTemplate!(XXH64_hash_t, XXH64_state_t, false) hash1;
|
|
hash1.start();
|
|
hash1.put(cast(ubyte) 0);
|
|
auto result = hash1.finish();
|
|
}
|
|
|
|
alias XXH_32 = XXHTemplate!(XXH32_hash_t, XXH32_state_t, false); /// XXH_32 for XXH, 32bit, hash is ubyte[4]
|
|
alias XXH_64 = XXHTemplate!(XXH64_hash_t, XXH64_state_t, false); /// XXH_64 for XXH, 64bit, hash is ubyte[8]
|
|
alias XXH3_64 = XXHTemplate!(XXH64_hash_t, XXH3_state_t, true); /// XXH3_64 for XXH3, 64bits, hash is ubyte[8]
|
|
alias XXH3_128 = XXHTemplate!(XXH128_hash_t, XXH3_state_t, true); /// XXH3_128 for XXH3, 128bits, hash is ubyte[16]
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example
|
|
XXH_32 hash1;
|
|
hash1.start();
|
|
hash1.put(cast(ubyte) 0);
|
|
auto result = hash1.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example
|
|
XXH_64 hash1;
|
|
hash1.start();
|
|
hash1.put(cast(ubyte) 0);
|
|
auto result = hash1.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example
|
|
XXH3_64 hash1;
|
|
hash1.start();
|
|
hash1.put(cast(ubyte) 0);
|
|
auto result = hash1.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example
|
|
XXH3_128 hash1;
|
|
hash1.start();
|
|
hash1.put(cast(ubyte) 0);
|
|
auto result = hash1.finish();
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using xxh32Of helper function
|
|
auto hash = xxh32Of("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "32D153FF");
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using xxh32Of helper function
|
|
auto hash = xxh64Of("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "44BC2CF5AD770999"); // XXH64
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using xxh32Of helper function
|
|
auto hash = xxh3_64Of("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "78AF5F94892F3950"); // XXH3/64
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using xxh32Of helper function
|
|
auto hash = xxh128Of("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "06B05AB6733A618578AF5F94892F3950");
|
|
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Using the basic API
|
|
XXH_32 hash;
|
|
hash.start();
|
|
ubyte[1024] data;
|
|
//Initialize data here...
|
|
hash.put(data);
|
|
ubyte[4] result = hash.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Using the basic API
|
|
XXH_64 hash;
|
|
hash.start();
|
|
ubyte[1024] data;
|
|
//Initialize data here...
|
|
hash.put(data);
|
|
ubyte[8] result = hash.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Using the basic API
|
|
XXH3_64 hash;
|
|
hash.start();
|
|
ubyte[1024] data;
|
|
//Initialize data here...
|
|
hash.put(data);
|
|
ubyte[8] result = hash.finish();
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Using the basic API
|
|
XXH3_128 hash;
|
|
hash.start();
|
|
ubyte[1024] data;
|
|
//Initialize data here...
|
|
hash.put(data);
|
|
ubyte[16] result = hash.finish();
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Let's use the template features:
|
|
void doSomething(T)(ref T hash)
|
|
if (isDigest!T)
|
|
{
|
|
hash.put(cast(ubyte) 0);
|
|
}
|
|
|
|
XXH_32 xxh;
|
|
xxh.start();
|
|
doSomething(xxh);
|
|
auto hash = xxh.finish;
|
|
assert(toHexString(hash) == "CF65B03E", "Got " ~ toHexString(hash));
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Let's use the template features:
|
|
void doSomething(T)(ref T hash)
|
|
if (isDigest!T)
|
|
{
|
|
hash.put(cast(ubyte) 0);
|
|
}
|
|
|
|
XXH_64 xxh;
|
|
xxh.start();
|
|
doSomething(xxh);
|
|
auto hash = xxh.finish;
|
|
assert(toHexString(hash) == "E934A84ADB052768", "Got " ~ toHexString(hash));
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Let's use the template features:
|
|
void doSomething(T)(ref T hash)
|
|
if (isDigest!T)
|
|
{
|
|
hash.put(cast(ubyte) 0);
|
|
}
|
|
|
|
XXH3_64 xxh;
|
|
xxh.start();
|
|
doSomething(xxh);
|
|
auto hash = xxh.finish;
|
|
assert(toHexString(hash) == "C44BDFF4074EECDB", "Got " ~ toHexString(hash));
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Let's use the template features:
|
|
void doSomething(T)(ref T hash)
|
|
if (isDigest!T)
|
|
{
|
|
hash.put(cast(ubyte) 0);
|
|
}
|
|
|
|
XXH3_128 xxh;
|
|
xxh.start();
|
|
doSomething(xxh);
|
|
auto hash = xxh.finish;
|
|
assert(toHexString(hash) == "A6CD5E9392000F6AC44BDFF4074EECDB", "Got " ~ toHexString(hash));
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
assert(isDigest!XXH_32);
|
|
assert(isDigest!XXH_64);
|
|
assert(isDigest!XXH3_64);
|
|
assert(isDigest!XXH3_128);
|
|
}
|
|
|
|
@system unittest
|
|
{
|
|
import std.range;
|
|
import std.conv : hexString;
|
|
|
|
ubyte[4] digest32;
|
|
ubyte[8] digest64;
|
|
ubyte[16] digest128;
|
|
|
|
XXH_32 xxh;
|
|
xxh.put(cast(ubyte[]) "abcdef");
|
|
xxh.start();
|
|
xxh.put(cast(ubyte[]) "");
|
|
assert(xxh.finish() == cast(ubyte[]) hexString!"02cc5d05");
|
|
|
|
digest32 = xxh32Of("");
|
|
assert(digest32 == cast(ubyte[]) hexString!"02cc5d05");
|
|
digest64 = xxh64Of("");
|
|
assert(digest64 == cast(ubyte[]) hexString!"EF46DB3751D8E999", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("");
|
|
assert(digest64 == cast(ubyte[]) hexString!"2D06800538D394C2", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("");
|
|
assert(digest128 == cast(ubyte[]) hexString!"99AA06D3014798D86001C324468D497F",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of("a");
|
|
assert(digest32 == cast(ubyte[]) hexString!"550d7456");
|
|
digest64 = xxh64Of("a");
|
|
assert(digest64 == cast(ubyte[]) hexString!"D24EC4F1A98C6E5B", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("a");
|
|
assert(digest64 == cast(ubyte[]) hexString!"E6C632B61E964E1F", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("a");
|
|
assert(digest128 == cast(ubyte[]) hexString!"A96FAF705AF16834E6C632B61E964E1F",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of("abc");
|
|
assert(digest32 == cast(ubyte[]) hexString!"32D153FF");
|
|
digest64 = xxh64Of("abc");
|
|
assert(digest64 == cast(ubyte[]) hexString!"44BC2CF5AD770999");
|
|
digest64 = xxh3_64Of("abc");
|
|
assert(digest64 == cast(ubyte[]) hexString!"78AF5F94892F3950");
|
|
digest128 = xxh128Of("abc");
|
|
assert(digest128 == cast(ubyte[]) hexString!"06B05AB6733A618578AF5F94892F3950");
|
|
|
|
digest32 = xxh32Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
|
|
assert(digest32 == cast(ubyte[]) hexString!"89ea60c3");
|
|
digest64 = xxh64Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
|
|
assert(digest64 == cast(ubyte[]) hexString!"F06103773E8585DF", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
|
|
assert(digest64 == cast(ubyte[]) hexString!"5BBCBBABCDCC3D3F", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq");
|
|
assert(digest128 == cast(ubyte[]) hexString!"3D62D22A5169B016C0D894FD4828A1A7",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of("message digest");
|
|
assert(digest32 == cast(ubyte[]) hexString!"7c948494");
|
|
digest64 = xxh64Of("message digest");
|
|
assert(digest64 == cast(ubyte[]) hexString!"066ED728FCEEB3BE", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("message digest");
|
|
assert(digest64 == cast(ubyte[]) hexString!"160D8E9329BE94F9", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("message digest");
|
|
assert(digest128 == cast(ubyte[]) hexString!"34AB715D95E3B6490ABFABECB8E3A424",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of("abcdefghijklmnopqrstuvwxyz");
|
|
assert(digest32 == cast(ubyte[]) hexString!"63a14d5f");
|
|
digest64 = xxh64Of("abcdefghijklmnopqrstuvwxyz");
|
|
assert(digest64 == cast(ubyte[]) hexString!"CFE1F278FA89835C", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("abcdefghijklmnopqrstuvwxyz");
|
|
assert(digest64 == cast(ubyte[]) hexString!"810F9CA067FBB90C", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("abcdefghijklmnopqrstuvwxyz");
|
|
assert(digest128 == cast(ubyte[]) hexString!"DB7CA44E84843D67EBE162220154E1E6",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
|
|
assert(digest32 == cast(ubyte[]) hexString!"9c285e64");
|
|
digest64 = xxh64Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
|
|
assert(digest64 == cast(ubyte[]) hexString!"AAA46907D3047814", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
|
|
assert(digest64 == cast(ubyte[]) hexString!"643542BB51639CB2", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789");
|
|
assert(digest128 == cast(ubyte[]) hexString!"5BCB80B619500686A3C0560BD47A4FFB",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
digest32 = xxh32Of(
|
|
"1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890");
|
|
assert(digest32 == cast(ubyte[]) hexString!"9c05f475");
|
|
digest64 = xxh64Of(
|
|
"1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890");
|
|
assert(digest64 == cast(ubyte[]) hexString!"E04A477F19EE145D", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of(
|
|
"1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890");
|
|
assert(digest64 == cast(ubyte[]) hexString!"7F58AA2520C681F9", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of(
|
|
"1234567890123456789012345678901234567890" ~ "1234567890123456789012345678901234567890");
|
|
assert(digest128 == cast(ubyte[]) hexString!"08DD22C3DDC34CE640CB8D6AC672DCB8",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
enum ubyte[16] input = cast(ubyte[16]) hexString!"c3fcd3d76192e4007dfb496cca67e13b";
|
|
assert(toHexString(input) == "C3FCD3D76192E4007DFB496CCA67E13B");
|
|
|
|
ubyte[] onemilliona = new ubyte[1_000_000];
|
|
onemilliona[] = 'a';
|
|
digest32 = xxh32Of(onemilliona);
|
|
assert(digest32 == cast(ubyte[]) hexString!"E1155920", "Got " ~ toHexString(digest32));
|
|
digest64 = xxh64Of(onemilliona);
|
|
assert(digest64 == cast(ubyte[]) hexString!"DC483AAA9B4FDC40", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of(onemilliona);
|
|
assert(digest64 == cast(ubyte[]) hexString!"B1FD6FAE5285C4EB", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of(onemilliona);
|
|
assert(digest128 == cast(ubyte[]) hexString!"A545DF8E384A9579B1FD6FAE5285C4EB",
|
|
"Got " ~ toHexString(digest128));
|
|
|
|
auto oneMillionRange = repeat!ubyte(cast(ubyte) 'a', 1_000_000);
|
|
digest32 = xxh32Of(oneMillionRange);
|
|
assert(digest32 == cast(ubyte[]) hexString!"E1155920", "Got " ~ toHexString(digest32));
|
|
digest64 = xxh64Of(oneMillionRange);
|
|
assert(digest64 == cast(ubyte[]) hexString!"DC483AAA9B4FDC40", "Got " ~ toHexString(digest64));
|
|
digest64 = xxh3_64Of(oneMillionRange);
|
|
assert(digest64 == cast(ubyte[]) hexString!"B1FD6FAE5285C4EB", "Got " ~ toHexString(digest64));
|
|
digest128 = xxh128Of(oneMillionRange);
|
|
assert(digest128 == cast(ubyte[]) hexString!"A545DF8E384A9579B1FD6FAE5285C4EB",
|
|
"Got " ~ toHexString(digest128));
|
|
}
|
|
|
|
/**
|
|
* This is a convenience alias for $(REF digest, std,digest) using the
|
|
* XXH implementation.
|
|
*/
|
|
//simple alias doesn't work here, hope this gets inlined...
|
|
auto xxh32Of(T...)(T data)
|
|
{
|
|
return digest!(XXH_32, T)(data);
|
|
}
|
|
/// Ditto
|
|
auto xxh64Of(T...)(T data)
|
|
{
|
|
return digest!(XXH_64, T)(data);
|
|
}
|
|
/// Ditto
|
|
auto xxh3_64Of(T...)(T data)
|
|
{
|
|
return digest!(XXH3_64, T)(data);
|
|
}
|
|
/// Ditto
|
|
auto xxh128Of(T...)(T data)
|
|
{
|
|
return digest!(XXH3_128, T)(data);
|
|
}
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
auto hash = xxh32Of("abc");
|
|
assert(hash == digest!XXH_32("abc"));
|
|
auto hash1 = xxh64Of("abc");
|
|
assert(hash1 == digest!XXH_64("abc"));
|
|
auto hash2 = xxh3_64Of("abc");
|
|
assert(hash2 == digest!XXH3_64("abc"));
|
|
auto hash3 = xxh128Of("abc");
|
|
assert(hash3 == digest!XXH3_128("abc"));
|
|
}
|
|
|
|
/**
|
|
* OOP API XXH implementation.
|
|
* See `std.digest` for differences between template and OOP API.
|
|
*
|
|
* This is an alias for $(D $(REF WrapperDigest, std,digest)!XXH_32), see
|
|
* there for more information.
|
|
*/
|
|
alias XXH32Digest = WrapperDigest!XXH_32;
|
|
alias XXH64Digest = WrapperDigest!XXH_64; ///ditto
|
|
alias XXH3_64Digest = WrapperDigest!XXH3_64; ///ditto
|
|
alias XXH3_128Digest = WrapperDigest!XXH3_128; ///ditto
|
|
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using Digest.digest helper function
|
|
auto xxh = new XXH32Digest();
|
|
ubyte[] hash = xxh.digest("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "32D153FF");
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using Digest.digest helper function
|
|
auto xxh = new XXH64Digest();
|
|
ubyte[] hash = xxh.digest("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "44BC2CF5AD770999");
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using Digest.digest helper function
|
|
auto xxh = new XXH3_64Digest();
|
|
ubyte[] hash = xxh.digest("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "78AF5F94892F3950");
|
|
}
|
|
///
|
|
@safe unittest
|
|
{
|
|
//Simple example, hashing a string using Digest.digest helper function
|
|
auto xxh = new XXH3_128Digest();
|
|
ubyte[] hash = xxh.digest("abc");
|
|
//Let's get a hash string
|
|
assert(toHexString(hash) == "06B05AB6733A618578AF5F94892F3950");
|
|
}
|
|
|
|
///
|
|
@system unittest
|
|
{
|
|
//Let's use the OOP features:
|
|
void test(Digest dig)
|
|
{
|
|
dig.put(cast(ubyte) 0);
|
|
}
|
|
|
|
auto xxh = new XXH32Digest();
|
|
test(xxh);
|
|
|
|
//Let's use a custom buffer:
|
|
ubyte[16] buf;
|
|
ubyte[] result = xxh.finish(buf[]);
|
|
assert(toHexString(result) == "CF65B03E", "Got " ~ toHexString(result));
|
|
}
|
|
///
|
|
@system unittest
|
|
{
|
|
//Let's use the OOP features:
|
|
void test(Digest dig)
|
|
{
|
|
dig.put(cast(ubyte) 0);
|
|
}
|
|
|
|
auto xxh = new XXH64Digest();
|
|
test(xxh);
|
|
|
|
//Let's use a custom buffer:
|
|
ubyte[16] buf;
|
|
ubyte[] result = xxh.finish(buf[]);
|
|
assert(toHexString(result) == "E934A84ADB052768", "Got " ~ toHexString(result));
|
|
}
|
|
///
|
|
@system unittest
|
|
{
|
|
//Let's use the OOP features:
|
|
void test(Digest dig)
|
|
{
|
|
dig.put(cast(ubyte) 0);
|
|
}
|
|
|
|
auto xxh = new XXH3_64Digest();
|
|
test(xxh);
|
|
|
|
//Let's use a custom buffer:
|
|
ubyte[16] buf;
|
|
ubyte[] result = xxh.finish(buf[]);
|
|
assert(toHexString(result) == "C44BDFF4074EECDB", "Got " ~ toHexString(result));
|
|
}
|
|
///
|
|
@system unittest
|
|
{
|
|
//Let's use the OOP features:
|
|
void test(Digest dig)
|
|
{
|
|
dig.put(cast(ubyte) 0);
|
|
}
|
|
|
|
auto xxh = new XXH3_128Digest();
|
|
test(xxh);
|
|
|
|
//Let's use a custom buffer:
|
|
ubyte[16] buf;
|
|
ubyte[] result = xxh.finish(buf[]);
|
|
assert(toHexString(result) == "A6CD5E9392000F6AC44BDFF4074EECDB", "Got " ~ toHexString(result));
|
|
}
|
|
|
|
@system unittest
|
|
{
|
|
import std.conv : hexString;
|
|
|
|
auto xxh = new XXH32Digest();
|
|
auto xxh64 = new XXH64Digest();
|
|
auto xxh3_64 = new XXH3_64Digest();
|
|
auto xxh128 = new XXH3_128Digest();
|
|
|
|
xxh.put(cast(ubyte[]) "abcdef");
|
|
xxh.reset();
|
|
xxh.put(cast(ubyte[]) "");
|
|
assert(xxh.finish() == cast(ubyte[]) hexString!"02cc5d05");
|
|
|
|
xxh.put(cast(ubyte[]) "abcdefghijklmnopqrstuvwxyz");
|
|
ubyte[20] result;
|
|
auto result2 = xxh.finish(result[]);
|
|
assert(result[0 .. 4] == result2
|
|
&& result2 == cast(ubyte[]) hexString!"63a14d5f", "Got " ~ toHexString(result));
|
|
|
|
debug
|
|
{
|
|
import std.exception;
|
|
|
|
assertThrown!Error(xxh.finish(result[0 .. 3]));
|
|
}
|
|
|
|
assert(xxh.length == 4);
|
|
assert(xxh64.length == 8);
|
|
assert(xxh3_64.length == 8);
|
|
assert(xxh128.length == 16);
|
|
|
|
assert(xxh.digest("") == cast(ubyte[]) hexString!"02cc5d05");
|
|
assert(xxh64.digest("") == cast(ubyte[]) hexString!"EF46DB3751D8E999");
|
|
assert(xxh3_64.digest("") == cast(ubyte[]) hexString!"2D06800538D394C2");
|
|
assert(xxh128.digest("") == cast(ubyte[]) hexString!"99AA06D3014798D86001C324468D497F");
|
|
|
|
assert(xxh.digest("a") == cast(ubyte[]) hexString!"550d7456");
|
|
assert(xxh64.digest("a") == cast(ubyte[]) hexString!"D24EC4F1A98C6E5B");
|
|
assert(xxh3_64.digest("a") == cast(ubyte[]) hexString!"E6C632B61E964E1F");
|
|
assert(xxh128.digest("a") == cast(ubyte[]) hexString!"A96FAF705AF16834E6C632B61E964E1F");
|
|
|
|
assert(xxh.digest("abc") == cast(ubyte[]) hexString!"32D153FF");
|
|
assert(xxh64.digest("abc") == cast(ubyte[]) hexString!"44BC2CF5AD770999");
|
|
assert(xxh3_64.digest("abc") == cast(ubyte[]) hexString!"78AF5F94892F3950");
|
|
assert(xxh128.digest("abc") == cast(ubyte[]) hexString!"06B05AB6733A618578AF5F94892F3950");
|
|
|
|
assert(xxh.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast(
|
|
ubyte[]) hexString!"89ea60c3");
|
|
assert(xxh64.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast(
|
|
ubyte[]) hexString!"F06103773E8585DF");
|
|
assert(xxh3_64.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast(
|
|
ubyte[]) hexString!"5BBCBBABCDCC3D3F");
|
|
assert(xxh128.digest("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == cast(
|
|
ubyte[]) hexString!"3D62D22A5169B016C0D894FD4828A1A7");
|
|
|
|
assert(xxh.digest("message digest") == cast(ubyte[]) hexString!"7c948494");
|
|
assert(xxh64.digest("message digest") == cast(ubyte[]) hexString!"066ED728FCEEB3BE");
|
|
assert(xxh3_64.digest("message digest") == cast(ubyte[]) hexString!"160D8E9329BE94F9");
|
|
assert(xxh128.digest("message digest") == cast(
|
|
ubyte[]) hexString!"34AB715D95E3B6490ABFABECB8E3A424");
|
|
|
|
assert(xxh.digest("abcdefghijklmnopqrstuvwxyz") == cast(ubyte[]) hexString!"63a14d5f");
|
|
assert(xxh64.digest("abcdefghijklmnopqrstuvwxyz") == cast(ubyte[]) hexString!"CFE1F278FA89835C");
|
|
assert(xxh3_64.digest("abcdefghijklmnopqrstuvwxyz") == cast(
|
|
ubyte[]) hexString!"810F9CA067FBB90C");
|
|
assert(xxh128.digest("abcdefghijklmnopqrstuvwxyz") == cast(
|
|
ubyte[]) hexString!"DB7CA44E84843D67EBE162220154E1E6");
|
|
|
|
assert(xxh.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast(
|
|
ubyte[]) hexString!"9c285e64");
|
|
assert(xxh64.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast(
|
|
ubyte[]) hexString!"AAA46907D3047814");
|
|
assert(xxh3_64.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast(
|
|
ubyte[]) hexString!"643542BB51639CB2");
|
|
assert(xxh128.digest("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") == cast(
|
|
ubyte[]) hexString!"5BCB80B619500686A3C0560BD47A4FFB");
|
|
|
|
assert(xxh.digest("1234567890123456789012345678901234567890",
|
|
"1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"9c05f475");
|
|
assert(xxh64.digest("1234567890123456789012345678901234567890",
|
|
"1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"E04A477F19EE145D");
|
|
assert(xxh3_64.digest("1234567890123456789012345678901234567890",
|
|
"1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"7F58AA2520C681F9");
|
|
assert(xxh128.digest("1234567890123456789012345678901234567890",
|
|
"1234567890123456789012345678901234567890") == cast(ubyte[]) hexString!"08DD22C3DDC34CE640CB8D6AC672DCB8");
|
|
}
|