[WIP] Add FastLZMA2 compression algorithm

builtins/flzma2/CMakeLists.txt

@@ -0,0 +1,41 @@
+# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers.
+# All rights reserved.
+#
+# For the licensing terms see $ROOTSYS/LICENSE.
+# For the list of contributors see $ROOTSYS/README/CREDITS.
+
+project(FLMA2 C)
+
+file(GLOB FLMA2_HEADERS *.h)
+file(GLOB FLMA2_SOURCES *.c)
+
+unset(FLMA2_FOUND CACHE)
+unset(FLMA2_FOUND PARENT_SCOPE)
+set(FLMA2_FOUND TRUE CACHE BOOL "" FORCE)
+
+file(STRINGS "fast-lzma2.h" FLMA2_H REGEX "^#define FL2_VERSION_[A-Z]+[ ]+[0-9]+.*$")
+string(REGEX REPLACE ".+FLMA2_VERSION_MAJOR[ ]+([0-9]+).*$"   "\\1" FL2_VERSION_MAJOR "${FLMA2_H}")
+string(REGEX REPLACE ".+FLMA2_VERSION_MINOR[ ]+([0-9]+).*$"   "\\1" FL2_VERSION_MINOR "${FLMA2_H}")
+string(REGEX REPLACE ".+FLMA2_VERSION_RELEASE[ ]+([0-9]+).*$" "\\1" FL2_VERSION_PATCH "${FLMA2_H}")
+set(FLMA2_VERSION_STRING "${FLMA2_VERSION_MAJOR}.${FLMA2_VERSION_MINOR}.${FLMA2_VERSION_PATCH}")
+unset(FLMA2_H)
+
+set(FLMA2_VERSION ${FLMA2_VERSION_STRING} CACHE INTERNAL "" FORCE)
+set(FLMA2_VERSION_STRING ${FLMA2_VERSION_STRING} CACHE INTERNAL "" FORCE)
+
+set(FLMA2_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "" FORCE)
+set(FLMA2_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "" FORCE)
+
+add_library(fast-lzma2 STATIC ${FLMA2_HEADERS} ${FLMA2_SOURCES})
+if(NOT MSVC)
+  target_compile_options(fast-lzma2 PRIVATE -fPIC)
+endif()
+target_link_libraries(fast-lzma2 PRIVATE xxHash::xxHash)
+
+add_library(FLMA2::FLMA2 ALIAS fast-lzma2)
+
+set(FLMA2_LIBRARY $<TARGET_FILE:FLMA2> CACHE INTERNAL "")
+set(FLMA2_LIBRARIES FLMA2::FLMA2 CACHE INTERNAL "")
+set_property(GLOBAL APPEND PROPERTY ROOT_BUILTIN_TARGETS FLMA2::FLMA2)
+
+ROOT_INSTALL_HEADERS()

builtins/flzma2/atomic.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2018 Conor McCarthy
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ *
+ */
+
+#ifndef FL2_ATOMIC_H
+#define FL2_ATOMIC_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* atomic add */
+
+#if !defined(FL2_SINGLETHREAD) && defined(_WIN32)
+
+#ifdef WINVER
+#undef WINVER
+#endif
+#define WINVER       0x0600
+
+#ifdef _WIN32_WINNT
+#undef _WIN32_WINNT
+#endif
+#define _WIN32_WINNT 0x0600
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+
+#include <windows.h>
+
+
+typedef LONG volatile FL2_atomic;
+#define ATOMIC_INITIAL_VALUE -1
+#define FL2_atomic_increment(n) InterlockedIncrement(&n)
+#define FL2_atomic_add(n, a) InterlockedAdd(&n, a)
+#define FL2_nonAtomic_increment(n) (++n)
+
+#elif !defined(FL2_SINGLETHREAD) && defined(__GNUC__)
+
+typedef long FL2_atomic;
+#define ATOMIC_INITIAL_VALUE 0
+#define FL2_atomic_increment(n) __sync_fetch_and_add(&n, 1)
+#define FL2_atomic_add(n, a) __sync_fetch_and_add(&n, a)
+#define FL2_nonAtomic_increment(n) (n++)
+
+#elif !defined(FL2_SINGLETHREAD) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) /* C11 */
+
+#include <stdatomic.h>
+
+typedef _Atomic long FL2_atomic;
+#define ATOMIC_INITIAL_VALUE 0
+#define FL2_atomic_increment(n) atomic_fetch_add(&n, 1)
+#define FL2_atomic_add(n, a) atomic_fetch_add(&n, a)
+#define FL2_nonAtomic_increment(n) (n++)
+
+#else  /* No atomics */
+
+#	ifndef FL2_SINGLETHREAD
+#		error  No atomic operations available. Change compiler config or define FL2_SINGLETHREAD for the entire build.
+#	endif
+
+typedef long FL2_atomic;
+#define ATOMIC_INITIAL_VALUE 0
+#define FL2_atomic_increment(n) (n++)
+#define FL2_atomic_add(n, a) (n += (a))
+#define FL2_nonAtomic_increment(n) (n++)
+
+#endif /* FL2_SINGLETHREAD */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* FL2_ATOMIC_H */

builtins/flzma2/compiler.h

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ * Modified for FL2 by Conor McCarthy
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef FL2_COMPILER_H
+#define FL2_COMPILER_H
+
+/*-*******************************************************
+*  Compiler specifics
+*********************************************************/
+/* force inlining */
+
+#if !defined(FL2_NO_INLINE)
+#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#  define INLINE_KEYWORD inline
+#else
+#  define INLINE_KEYWORD
+#endif
+
+#if defined(__GNUC__)
+#  define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#  define FORCE_INLINE_ATTR __forceinline
+#else
+#  define FORCE_INLINE_ATTR
+#endif
+
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
+/**
+ * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
+ * parameters. They must be inlined for the compiler to eliminate the constant
+ * branches.
+ */
+#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
+/**
+ * HINT_INLINE is used to help the compiler generate better code. It is *not*
+ * used for "templates", so it can be tweaked based on the compilers
+ * performance.
+ *
+ * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
+ * always_inline attribute.
+ *
+ * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
+ * attribute.
+ */
+#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
+#  define HINT_INLINE static INLINE_KEYWORD
+#else
+#  define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
+#endif
+
+/* force no inlining */
+#ifdef _MSC_VER
+#  define FORCE_NOINLINE __declspec(noinline)
+#else
+#  ifdef __GNUC__
+#    define FORCE_NOINLINE __attribute__((__noinline__))
+#  else
+#    define FORCE_NOINLINE
+#  endif
+#endif
+
+/* target attribute */
+#ifndef __has_attribute
+  #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
+#endif
+#if defined(__GNUC__)
+#  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+#  define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+  #if ((defined(__clang__) && __has_attribute(__target__)) \
+      || (defined(__GNUC__) \
+          && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
+      && (defined(__x86_64__) || defined(_M_X86)) \
+      && !defined(__BMI2__)
+  #  define DYNAMIC_BMI2 1
+  #else
+  #  define DYNAMIC_BMI2 0
+  #endif
+#endif
+
+/* prefetch
+ * can be disabled, by declaring NO_PREFETCH build macro */
+#if defined(NO_PREFETCH)
+#  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
+#  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
+#else
+#  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
+#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#  else
+#    define PREFETCH_L1(ptr) (void)(ptr)  /* disabled */
+#    define PREFETCH_L2(ptr) (void)(ptr)  /* disabled */
+#  endif
+#endif  /* NO_PREFETCH */
+
+#define CACHELINE_SIZE 64
+
+#define PREFETCH_AREA(p, s)  {            \
+    const char* const _ptr = (const char*)(p);  \
+    size_t const _size = (size_t)(s);     \
+    size_t _pos;                          \
+    for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) {  \
+        PREFETCH_L2(_ptr + _pos);         \
+    }                                     \
+}
+
+/* disable warnings */
+#ifdef _MSC_VER    /* Visual Studio */
+#  include <intrin.h>                    /* For Visual 2005 */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
+#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#endif
+
+#endif /* FL2_COMPILER_H */

builtins/flzma2/count.h

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COUNT_H_
+#define ZSTD_COUNT_H_
+
+#include "mem.h"
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-*************************************
+*  Match length counter
+***************************************/
+static unsigned ZSTD_NbCommonBytes(register size_t val)
+{
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanForward64(&r, (U64)val);
+            return (unsigned)(r >> 3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 4)
+            return (__builtin_ctzll((U64)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
+                0, 3, 1, 3, 1, 4, 2, 7,
+                0, 2, 3, 6, 1, 5, 3, 5,
+                1, 3, 4, 4, 2, 5, 6, 7,
+                7, 0, 1, 2, 3, 3, 4, 6,
+                2, 6, 5, 5, 3, 4, 5, 6,
+                7, 1, 2, 4, 6, 4, 4, 5,
+                7, 2, 6, 5, 7, 6, 7, 7 };
+            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#       endif
+        }
+        else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanForward(&r, (U32)val);
+            return (unsigned)(r >> 3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctz((U32)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
+                3, 2, 2, 1, 3, 2, 0, 1,
+                3, 3, 1, 2, 2, 2, 2, 0,
+                3, 1, 2, 0, 1, 0, 1, 1 };
+            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#       endif
+        }
+    }
+    else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanReverse64(&r, val);
+            return (unsigned)(r >> 3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 4)
+            return (__builtin_clzll(val) >> 3);
+#       else
+            unsigned r;
+            const unsigned n32 = sizeof(size_t) * 4;   /* calculate this way due to compiler complaining in 32-bits mode */
+            if (!(val >> n32)) { r = 4; }
+            else { r = 0; val >>= n32; }
+            if (!(val >> 16)) { r += 2; val >>= 8; }
+            else { val >>= 24; }
+            r += (!val);
+            return r;
+#       endif
+        }
+        else { /* 32 bits */
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanReverse(&r, (unsigned long)val);
+            return (unsigned)(r >> 3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clz((U32)val) >> 3);
+#       else
+            unsigned r;
+            if (!(val >> 16)) { r = 2; val >>= 8; }
+            else { r = 0; val >>= 24; }
+            r += (!val);
+            return r;
+#       endif
+        }
+    }
+}
+
+
+static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
+{
+    const BYTE* const pStart = pIn;
+    const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
+
+    if (pIn < pInLoopLimit) {
+        { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+        if (diff) return ZSTD_NbCommonBytes(diff); }
+        pIn += sizeof(size_t); pMatch += sizeof(size_t);
+        while (pIn < pInLoopLimit) {
+            size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+            if (!diff) { pIn += sizeof(size_t); pMatch += sizeof(size_t); continue; }
+            pIn += ZSTD_NbCommonBytes(diff);
+            return (size_t)(pIn - pStart);
+        }
+    }
+    if (MEM_64bits() && (pIn<(pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn += 4; pMatch += 4; }
+    if ((pIn<(pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn += 2; pMatch += 2; }
+    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+    return (size_t)(pIn - pStart);
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_COUNT_H_ */