vectors.h

Go to the documentation of this file.

/*
 Copyright (C) 2010 X. Andrade
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2, or (at your option)
 any later version.
 
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 02110-1301, USA.
 
*/
 
#include <config.h>
 
#ifndef VECTORS_H
#define VECTORS_H
 
#ifdef __AVX__
// Check for intel avx
 
#if defined(__AVX512F__) || defined(__AVX512PF__) || \
    defined(__AVX512BW__) || defined(__AVX512ER__) || \
    defined(__AVX512CD__) || defined(__AVX512DQ__) || \
    defined(__AVX512VL__)
// Use AVX512
 
#include <immintrin.h>
#define VEC_SIZE 8
#define VEC_TYPE __m512d
#define VEC_LD(addr) _mm512_load_pd(addr)
#define VEC_LDU(addr) _mm512_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm512_stream_pd(addr, vec)
#define VEC_STA(addr, vec) _mm512_store_pd(addr, vec)
#define VEC_STU(addr, vec) _mm512_storeu_pd(addr, vec)
#define VEC_FMA(aa, bb, cc) _mm512_fmadd_pd(aa, bb, cc)
#define VEC_SCAL(aa) _mm512_set1_pd(aa)
#define VEC_ADD(aa, bb) _mm512_add_pd(aa, bb)
#define VEC_SUB(aa, bb) _mm512_sub_pd(aa, bb)
#define VEC_ZERO _mm512_setzero_pd()
#include <emmintrin.h>
#define FENCE _mm_mfence()
 
/* Use a depth of 16 to avoid register spilling. On skylake CPUs, there are 32
 * AVX512 registers, so we should only use a maximum of 16 for the points because
 * we need an additional one for the weights.
 */
#define DEPTH 16
#define VECTORIZATION_LEVEL "AVX512"
 
#elif defined(__AVX2__)
// Use AVX2
 
#include <immintrin.h>
#if defined(__FMA4__) || defined(__FMA__)
#include <x86intrin.h>
#endif
#define VEC_SIZE 4
#define VEC_TYPE __m256d
#define VEC_LD(addr) _mm256_load_pd(addr)
#define VEC_LDU(addr) _mm256_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm256_stream_pd(addr, vec)
#define VEC_STA(addr, vec) _mm256_store_pd(addr, vec)
#define VEC_STU(addr, vec) _mm256_storeu_pd(addr, vec)
#ifdef __FMA4__
#define VEC_FMA(aa, bb, cc) _mm256_macc_pd(aa, bb, cc)
#elif defined(__FMA__)
#define VEC_FMA(aa, bb, cc) _mm256_fmadd_pd(aa, bb, cc)
#else
#define VEC_FMA(aa, bb, cc) _mm256_add_pd(cc, _mm256_mul_pd(aa, bb))
#endif
#define VEC_SCAL(aa) _mm256_set1_pd(aa)
#define VEC_ADD(aa, bb) _mm256_add_pd(aa, bb)
#define VEC_SUB(aa, bb) _mm256_sub_pd(aa, bb)
#define VEC_ZERO _mm256_setzero_pd()
#include <emmintrin.h>
#define FENCE _mm_mfence()
 
#define DEPTH 16
#define VECTORIZATION_LEVEL "AVX2"
 
#else
// Default to AVX
 
#include <emmintrin.h>
#if defined(__FMA4__) || defined(__FMA__)
#include <x86intrin.h>
#endif
#define VEC_SIZE 2
#define VEC_TYPE __m128d
#define VEC_LD(addr) _mm_load_pd(addr)
#define VEC_LDU(addr) _mm_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm_stream_pd(addr, vec)
#define VEC_STA(addr, vec) _mm_store_pd(addr, vec)
#define VEC_STU(addr, vec) _mm_storeu_pd(addr, vec)
#ifdef __FMA4__
#define VEC_FMA(aa, bb, cc) _mm_macc_pd(aa, bb, cc)
#elif defined(__FMA__)
#define VEC_FMA(aa, bb, cc) _mm_fmadd_pd(aa, bb, cc)
#else
#define VEC_FMA(aa, bb, cc) _mm_add_pd(cc, _mm_mul_pd(aa, bb))
#endif
#define VEC_SCAL(aa) _mm_set1_pd(aa)
#define VEC_ADD(aa, bb) _mm_add_pd(aa, bb)
#define VEC_SUB(aa, bb) _mm_sub_pd(aa, bb)
#define VEC_ZERO _mm_setzero_pd()
#define FENCE _mm_mfence()
 
#define DEPTH 16
#define VECTORIZATION_LEVEL "AVX"
 
#endif
 
#elif defined(__ARM_NEON) && defined(__aarch64__)
// Use NEON
 
#include <arm_neon.h>
#define VEC_SIZE 2
#define VEC_TYPE float64x2_t
#define VEC_LD(addr) vld1q_f64(addr)
#define VEC_LDU(addr) VEC_LD(addr)
#define VEC_ST(addr, vec) vst1q_f64(addr, vec)
#define VEC_STA(addr, vec) vst1q_f64(addr, vec)
#define VEC_STU(addr, vec) VEC_ST(addr, vec)
#define VEC_FMA(aa, bb, cc) vfmaq_f64(cc, aa, bb)
#define VEC_SCAL(aa) vdupq_n_f64(aa)
#define VEC_ADD(aa, bb) vaddq_f64(aa, bb)
#define VEC_SUB(aa, bb) vsubq_f64(aa, bb)
#define VEC_ZERO vdupq_n_f64(0.0)
 
#define DEPTH 16
#define VECTORIZATION_LEVEL "NEON"
 
#else
// Not explicitly optimized
 
#define VEC_SIZE 1
#define VEC_TYPE double
#define VEC_LD(addr) (addr)[0]
#define VEC_LDU(addr) VEC_LD(addr)
#define VEC_ST(addr, vec) (addr)[0] = vec
#define VEC_STA(addr, vec) (addr)[0] = vec
#define VEC_STU(addr, vec) VEC_ST(addr, vec)
#define VEC_FMA(aa, bb, cc) ((aa) * (bb) + (cc))
#define VEC_SCAL(aa) aa
#define VEC_ADD(aa, bb) ((aa)+(bb))
#define VEC_SUB(aa, bb) ((aa)-(bb))
#define VEC_ZERO 0.0
 
#define DEPTH 8
#define VECTORIZATION_LEVEL "default"
#endif
 
#define max1(x) (((x) > 0) ? (x) : 1)
 
#endif