vectors.h

Go to the documentation of this file.

/*
 Copyright (C) 2010 X. Andrade
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2, or (at your option)
 any later version.
 
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 02110-1301, USA.
 
*/
 
#include <config.h>
 
#ifndef VECTORS_H
#define VECTORS_H
 
#ifdef __AVX__
// Check for intel avx
 
#if defined(__AVX512F__) || defined(__AVX512PF__) || \
    defined(__AVX512BW__) || defined(__AVX512ER__) || \
    defined(__AVX512CD__) || defined(__AVX512DQ__) || \
    defined(__AVX512VL__)
// Use AVX512
 
#include <immintrin.h>
#define VEC_SIZE 8
#define VEC_TYPE __m512d
#define VEC_LD(addr) _mm512_load_pd(addr)
#define VEC_LDU(addr) _mm512_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm512_stream_pd(addr, vec)
#define VEC_STU(addr, vec) _mm512_storeu_pd(addr, vec)
#define VEC_FMA(aa, bb, cc) _mm512_fmadd_pd(aa, bb, cc)
#define VEC_SCAL(aa) _mm512_set1_pd(aa)
#define VEC_ZERO _mm512_setzero_pd()
#include <emmintrin.h>
#define FENCE _mm_mfence()
 
/* Use a depth of 16 to avoid register spilling. On skylake CPUs, there are 32
 * AVX512 registers, so we should only use a maximum of 16 for the points because
 * we need an additional one for the weights.
 */
#define DEPTH 16
 
#elif defined(__AVX2__)
// Use AVX2
 
#include <immintrin.h>
#if defined(__FMA4__) || defined(__FMA__)
#include <x86intrin.h>
#endif
#define VEC_SIZE 4
#define VEC_TYPE __m256d
#define VEC_LD(addr) _mm256_load_pd(addr)
#define VEC_LDU(addr) _mm256_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm256_stream_pd(addr, vec)
#define VEC_STU(addr, vec) _mm256_storeu_pd(addr, vec)
#ifdef __FMA4__
#define VEC_FMA(aa, bb, cc) _mm256_macc_pd(aa, bb, cc)
#elif defined(__FMA__)
#define VEC_FMA(aa, bb, cc) _mm256_fmadd_pd(aa, bb, cc)
#else
#define VEC_FMA(aa, bb, cc) _mm256_add_pd(cc, _mm256_mul_pd(aa, bb))
#endif
#define VEC_SCAL(aa) _mm256_set1_pd(aa)
#define VEC_ZERO _mm256_setzero_pd()
#include <emmintrin.h>
#define FENCE _mm_mfence()
 
#define DEPTH 16
 
#else
// Default to AVX
 
#include <emmintrin.h>
#if defined(__FMA4__) || defined(__FMA__)
#include <x86intrin.h>
#endif
#define VEC_SIZE 2
#define VEC_TYPE __m128d
#define VEC_LD(addr) _mm_load_pd(addr)
#define VEC_LDU(addr) _mm_loadu_pd(addr)
#define VEC_ST(addr, vec) _mm_stream_pd(addr, vec)
#define VEC_STU(addr, vec) _mm_storeu_pd(addr, vec)
#ifdef __FMA4__
#define VEC_FMA(aa, bb, cc) _mm_macc_pd(aa, bb, cc)
#elif defined(__FMA__)
#define VEC_FMA(aa, bb, cc) _mm_fmadd_pd(aa, bb, cc)
#else
#define VEC_FMA(aa, bb, cc) _mm_add_pd(cc, _mm_mul_pd(aa, bb))
#endif
#define VEC_SCAL(aa) _mm_set1_pd(aa)
#define VEC_ZERO _mm_setzero_pd()
#define FENCE _mm_mfence()
 
#define DEPTH 16
#endif
 
#elif defined(__bg__)
// Check for ibm blue_gene
 
#ifdef __bgq__
// Check for blue_gene_q
 
#define VEC_SIZE 4
#define VEC_TYPE vector4double
#define VEC_LD(addr) vec_ld(0, (double *)(addr))
#define VEC_LDU(addr)                                                          \
  ((vector4double){(addr)[0], (addr)[1], (addr)[2], (addr)[3]})
#define VEC_ST(addr, vec) vec_st(vec, 0, (double *)(addr))
#define VEC_STU(addr, vec)                                                     \
  (addr)[0] = vec_extract(vec, 0);                                             \
  (addr)[1] = vec_extract(vec, 1);                                             \
  (addr)[2] = vec_extract(vec, 2);                                             \
  (addr)[3] = vec_extract(vec, 3)
#define VEC_FMA(aa, bb, cc) vec_madd(aa, bb, cc)
#define VEC_SCAL(aa) ((vector4double){aa, aa, aa, aa})
#define VEC_SCAL_LD(addr) vec_lds(0, (double *)(addr))
#define VEC_ZERO ((vector4double){0.0, 0.0, 0.0, 0.0})
 
#define DEPTH 16
 
#else
// Otherwise use default blue_gene
 
#define VEC_SIZE 2
#define VEC_TYPE double _Complex
#define VEC_LD(addr) __lfpd(addr)
#define VEC_LDU(addr) __cmplx((addr)[0], (addr)[1])
#define VEC_ST(addr, vec) __stfpd(addr, vec)
#define VEC_STU(addr, vec)                                                     \
  (addr)[0] = __creal(vec);                                                    \
  (addr)[1] = __cimag(vec)
#define VEC_FMA(aa, bb, cc) __fpmadd(cc, aa, bb)
#define VEC_SCAL(aa) __cmplx(aa, aa)
#define VEC_ZERO __cmplx(0.0, 0.0)
 
#define DEPTH 16
#endif
 
#else
// Not explicitly optimized
 
#define VEC_SIZE 1
#define VEC_TYPE double
#define VEC_LD(addr) (addr)[0]
#define VEC_LDU(addr) VEC_LD(addr)
#define VEC_ST(addr, vec) (addr)[0] = vec
#define VEC_STU(addr, vec) VEC_ST(addr, vec)
#define VEC_FMA(aa, bb, cc) aa *bb + cc
#define VEC_SCAL(aa) aa
#define VEC_ZERO 0.0
 
#define DEPTH 8
#endif
 
#define max1(x) (((x) > 0) ? (x) : 1)
 
#endif