dccdc51788Fix the compile flags so that zlib can run on CPUs that do not have SSSE3/SSE4.2/etc. Do not compile zlib with flags that indicate that those features are available, and instead enable them selectively for functions that use them. There are probably better way to do this, e.g. through gyp file modifications as suggested in the issue. However, this patch should do just fine until that happens. Fixes: https://github.com/nodejs/node/issues/32553 PR-URL: https://github.com/nodejs/node/pull/32627 Reviewed-By: Gireesh Punathil <gpunathi@in.ibm.com> Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl> Reviewed-By: Tobias Nießen <tniessen@tnie.de> Reviewed-By: James M Snell <jasnell@gmail.com>
247 lines
7 KiB
C
247 lines
7 KiB
C
/* crc32_simd.c
|
|
*
|
|
* Copyright 2017 The Chromium Authors. All rights reserved.
|
|
* Use of this source code is governed by a BSD-style license that can be
|
|
* found in the Chromium source repository LICENSE file.
|
|
*/
|
|
|
|
#include "crc32_simd.h"
|
|
|
|
#if defined(CRC32_SIMD_SSE42_PCLMUL)
|
|
#ifndef __GNUC__
|
|
#define __attribute__()
|
|
#endif
|
|
|
|
/*
|
|
* crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer
|
|
* length must be at least 64, and a multiple of 16. Based on:
|
|
*
|
|
* "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
|
|
* V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0
|
|
*/
|
|
|
|
#include <emmintrin.h>
|
|
#include <smmintrin.h>
|
|
#include <wmmintrin.h>
|
|
|
|
__attribute__((target("sse4.2,pclmul")))
|
|
uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */
|
|
const unsigned char *buf,
|
|
z_size_t len,
|
|
uint32_t crc)
|
|
{
|
|
/*
|
|
* Definitions of the bit-reflected domain constants k1,k2,k3, etc and
|
|
* the CRC32+Barrett polynomials given at the end of the paper.
|
|
*/
|
|
static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 };
|
|
static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e };
|
|
static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 };
|
|
static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 };
|
|
|
|
__m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
|
|
|
|
/*
|
|
* There's at least one block of 64.
|
|
*/
|
|
x1 = _mm_loadu_si128((__m128i *)(buf + 0x00));
|
|
x2 = _mm_loadu_si128((__m128i *)(buf + 0x10));
|
|
x3 = _mm_loadu_si128((__m128i *)(buf + 0x20));
|
|
x4 = _mm_loadu_si128((__m128i *)(buf + 0x30));
|
|
|
|
x1 = _mm_xor_si128(x1, _mm_cvtsi32_si128(crc));
|
|
|
|
x0 = _mm_load_si128((__m128i *)k1k2);
|
|
|
|
buf += 64;
|
|
len -= 64;
|
|
|
|
/*
|
|
* Parallel fold blocks of 64, if any.
|
|
*/
|
|
while (len >= 64)
|
|
{
|
|
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x6 = _mm_clmulepi64_si128(x2, x0, 0x00);
|
|
x7 = _mm_clmulepi64_si128(x3, x0, 0x00);
|
|
x8 = _mm_clmulepi64_si128(x4, x0, 0x00);
|
|
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
|
x2 = _mm_clmulepi64_si128(x2, x0, 0x11);
|
|
x3 = _mm_clmulepi64_si128(x3, x0, 0x11);
|
|
x4 = _mm_clmulepi64_si128(x4, x0, 0x11);
|
|
|
|
y5 = _mm_loadu_si128((__m128i *)(buf + 0x00));
|
|
y6 = _mm_loadu_si128((__m128i *)(buf + 0x10));
|
|
y7 = _mm_loadu_si128((__m128i *)(buf + 0x20));
|
|
y8 = _mm_loadu_si128((__m128i *)(buf + 0x30));
|
|
|
|
x1 = _mm_xor_si128(x1, x5);
|
|
x2 = _mm_xor_si128(x2, x6);
|
|
x3 = _mm_xor_si128(x3, x7);
|
|
x4 = _mm_xor_si128(x4, x8);
|
|
|
|
x1 = _mm_xor_si128(x1, y5);
|
|
x2 = _mm_xor_si128(x2, y6);
|
|
x3 = _mm_xor_si128(x3, y7);
|
|
x4 = _mm_xor_si128(x4, y8);
|
|
|
|
buf += 64;
|
|
len -= 64;
|
|
}
|
|
|
|
/*
|
|
* Fold into 128-bits.
|
|
*/
|
|
x0 = _mm_load_si128((__m128i *)k3k4);
|
|
|
|
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
|
x1 = _mm_xor_si128(x1, x2);
|
|
x1 = _mm_xor_si128(x1, x5);
|
|
|
|
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
|
x1 = _mm_xor_si128(x1, x3);
|
|
x1 = _mm_xor_si128(x1, x5);
|
|
|
|
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
|
x1 = _mm_xor_si128(x1, x4);
|
|
x1 = _mm_xor_si128(x1, x5);
|
|
|
|
/*
|
|
* Single fold blocks of 16, if any.
|
|
*/
|
|
while (len >= 16)
|
|
{
|
|
x2 = _mm_loadu_si128((__m128i *)buf);
|
|
|
|
x5 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x11);
|
|
x1 = _mm_xor_si128(x1, x2);
|
|
x1 = _mm_xor_si128(x1, x5);
|
|
|
|
buf += 16;
|
|
len -= 16;
|
|
}
|
|
|
|
/*
|
|
* Fold 128-bits to 64-bits.
|
|
*/
|
|
x2 = _mm_clmulepi64_si128(x1, x0, 0x10);
|
|
x3 = _mm_setr_epi32(~0, 0, ~0, 0);
|
|
x1 = _mm_srli_si128(x1, 8);
|
|
x1 = _mm_xor_si128(x1, x2);
|
|
|
|
x0 = _mm_loadl_epi64((__m128i*)k5k0);
|
|
|
|
x2 = _mm_srli_si128(x1, 4);
|
|
x1 = _mm_and_si128(x1, x3);
|
|
x1 = _mm_clmulepi64_si128(x1, x0, 0x00);
|
|
x1 = _mm_xor_si128(x1, x2);
|
|
|
|
/*
|
|
* Barret reduce to 32-bits.
|
|
*/
|
|
x0 = _mm_load_si128((__m128i*)poly);
|
|
|
|
x2 = _mm_and_si128(x1, x3);
|
|
x2 = _mm_clmulepi64_si128(x2, x0, 0x10);
|
|
x2 = _mm_and_si128(x2, x3);
|
|
x2 = _mm_clmulepi64_si128(x2, x0, 0x00);
|
|
x1 = _mm_xor_si128(x1, x2);
|
|
|
|
/*
|
|
* Return the crc32.
|
|
*/
|
|
return _mm_extract_epi32(x1, 1);
|
|
}
|
|
|
|
#elif defined(CRC32_ARMV8_CRC32)
|
|
|
|
/* CRC32 checksums using ARMv8-a crypto instructions.
|
|
*
|
|
* TODO: implement a version using the PMULL instruction.
|
|
*/
|
|
|
|
#if defined(__clang__)
|
|
/* CRC32 intrinsics are #ifdef'ed out of arm_acle.h unless we build with an
|
|
* armv8 target, which is incompatible with ThinLTO optimizations on Android.
|
|
* (Namely, mixing and matching different module-level targets makes ThinLTO
|
|
* warn, and Android defaults to armv7-a. This restriction does not apply to
|
|
* function-level `target`s, however.)
|
|
*
|
|
* Since we only need four crc intrinsics, and since clang's implementation of
|
|
* those are just wrappers around compiler builtins, it's simplest to #define
|
|
* those builtins directly. If this #define list grows too much (or we depend on
|
|
* an intrinsic that isn't a trivial wrapper), we may have to find a better way
|
|
* to go about this.
|
|
*
|
|
* NOTE: clang currently complains that "'+soft-float-abi' is not a recognized
|
|
* feature for this target (ignoring feature)." This appears to be a harmless
|
|
* bug in clang.
|
|
*/
|
|
#define __crc32b __builtin_arm_crc32b
|
|
#define __crc32d __builtin_arm_crc32d
|
|
#define __crc32w __builtin_arm_crc32w
|
|
#define __crc32cw __builtin_arm_crc32cw
|
|
|
|
#if defined(__aarch64__)
|
|
#define TARGET_ARMV8_WITH_CRC __attribute__((target("crc")))
|
|
#else // !defined(__aarch64__)
|
|
#define TARGET_ARMV8_WITH_CRC __attribute__((target("armv8-a,crc")))
|
|
#endif // defined(__aarch64__)
|
|
|
|
#elif defined(__GNUC__)
|
|
/* For GCC, we are setting CRC extensions at module level, so ThinLTO is not
|
|
* allowed. We can just include arm_acle.h.
|
|
*/
|
|
#include <arm_acle.h>
|
|
#define TARGET_ARMV8_WITH_CRC
|
|
#else // !defined(__GNUC__) && !defined(_aarch64__)
|
|
#error ARM CRC32 SIMD extensions only supported for Clang and GCC
|
|
#endif
|
|
|
|
TARGET_ARMV8_WITH_CRC
|
|
uint32_t ZLIB_INTERNAL armv8_crc32_little(unsigned long crc,
|
|
const unsigned char *buf,
|
|
z_size_t len)
|
|
{
|
|
uint32_t c = (uint32_t) ~crc;
|
|
|
|
while (len && ((uintptr_t)buf & 7)) {
|
|
c = __crc32b(c, *buf++);
|
|
--len;
|
|
}
|
|
|
|
const uint64_t *buf8 = (const uint64_t *)buf;
|
|
|
|
while (len >= 64) {
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
c = __crc32d(c, *buf8++);
|
|
len -= 64;
|
|
}
|
|
|
|
while (len >= 8) {
|
|
c = __crc32d(c, *buf8++);
|
|
len -= 8;
|
|
}
|
|
|
|
buf = (const unsigned char *)buf8;
|
|
|
|
while (len--) {
|
|
c = __crc32b(c, *buf++);
|
|
}
|
|
|
|
return ~c;
|
|
}
|
|
|
|
#endif
|