26 use,
intrinsic :: iso_c_binding
27 use,
intrinsic :: ieee_arithmetic
34 use,
intrinsic :: iso_fortran_env
44#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
83 integer,
public,
parameter :: &
88 integer,
public,
parameter :: &
96 integer,
parameter :: &
105 integer,
public ::
type
106 integer,
public :: library
113 type(MPI_Comm) :: comm
114 integer :: rs_n_global(3)
115 integer :: fs_n_global(3)
118 integer :: rs_istart(1:3)
119 integer :: fs_istart(1:3)
122 integer,
public :: stride_rs(1:3)
123 integer,
public :: stride_fs(1:3)
132 real(real64),
contiguous,
pointer,
public :: drs_data(:,:,:,:)
133 complex(real64),
contiguous,
pointer,
public :: zrs_data(:,:,:,:)
134 complex(real64),
contiguous,
pointer,
public :: fs_data(:,:,:,:)
137 real(real32),
contiguous,
pointer,
public :: srs_data(:,:,:,:)
138 complex(real32),
contiguous,
pointer,
public :: crs_data(:,:,:,:)
139 complex(real32),
contiguous,
pointer,
public :: cfs_data(:,:,:,:)
140 logical,
public :: single = .false.
141 type(c_ptr) :: cuda_plan_fw
142 type(c_ptr) :: cuda_plan_bw
144 type(finufft_t),
public :: finufft
146 type(pnfft_t),
public :: pnfft
148 logical,
public :: aligned_memory
195 integer,
parameter :: &
210 integer :: ii, fft_default
211#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
308 message(1) =
'FFTLibrary = nfft requires FINUFFT but that library was not linked.'
314#if ! defined(HAVE_CUDA)
315 call messages_write(
'You have selected the Accelerated FFT, but Octopus was compiled', new_line = .
true.)
320 call messages_write(
'You have selected the accelerated FFT, but acceleration is disabled.')
325#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
326 if (omp_get_max_threads() > 1)
then
331 iret = fftw_init_threads()
336 call fftw_plan_with_nthreads(omp_get_max_threads())
366#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
367 call fftw_cleanup_threads()
380 subroutine fft_init(this, nn, dim, type, library, optimize, optimize_parity, comm, mpi_grp, use_aligned, &
381 howmany, nthreads, single)
382 type(
fft_t),
intent(inout) :: this
383 integer,
intent(inout) :: nn(3)
384 integer,
intent(in) :: dim
385 integer,
intent(in) ::
type
386 integer,
intent(in) :: library
388 integer,
intent(in) :: optimize_parity(3)
390 type(mpi_comm),
optional,
intent(out) :: comm
391 type(
mpi_grp_t),
optional,
intent(in) :: mpi_grp
392 logical,
optional,
intent(in) :: use_aligned
393 integer,
optional,
intent(in) :: howmany
394 integer,
optional,
intent(in) :: nthreads
395 logical,
optional,
intent(in) :: single
397 integer :: ii, jj, fft_dim, idir, column_size, row_size, n3
398 integer :: n_1, n_2, n_3, nn_temp(3)
401 integer(int64) :: number_points, alloc_size
413 if (
present(mpi_grp)) mpi_grp_ = mpi_grp
417 assert(this%howmany > 0)
420 if (this%single)
then
421 this%aligned_memory = .
true.
427 if (nn(ii) <= 1)
exit
428 fft_dim = fft_dim + 1
431 if (fft_dim == 0)
then
432 message(1) =
"Internal error in fft_init: apparently, a 1x1x1 FFT is required."
446 nn_temp(1:fft_dim) = nn(1:fft_dim)
448 select case (library_)
451 if(any(optimize_parity(1:fft_dim) > 1))
then
452 message(1) =
"Internal error in fft_init: optimize_parity must be negative, 0, or 1."
457 nn_temp(ii) =
fft_size(nn(ii), (/2, 3, 5, 7/), optimize_parity(ii))
462 assert(this%howmany == 1)
472 assert(this%howmany == 1)
476 if (int(nn(ii)/2)*2 /= nn(ii)) nn(ii) = nn(ii) + 1
479 if (fft_dim < 3)
then
485 if (fft_dim < 3 .and. library_ ==
fftlib_pfft)
then
490 if (any(optimize_parity(1:fft_dim) > 1))
then
491 message(1) =
"Internal error in fft_init: optimize_parity must be negative, 0, or 1."
504 do ii = fft_max, 1, -1
506 if (all(nn(1:dim) ==
fft_array(ii)%rs_n_global(1:dim)) .and.
type ==
fft_array(ii)%type &
509 .and. this%howmany ==
fft_array(ii)%howmany &
510 .and. (this%single .eqv.
fft_array(ii)%single) &
511 .and. (this%aligned_memory .eqv.
fft_array(ii)%aligned_memory))
then
517 if (
present(comm)) comm =
fft_array(ii)%comm
527 message(1) =
"Not enough slots for FFTs."
528 message(2) =
"Please increase FFT_MAX in fft.F90 and recompile."
538 fft_array(jj)%rs_n_global(1:dim) = nn(1:dim)
548 fft_array(jj)%aligned_memory = this%aligned_memory
551 select case (library_)
558 ierror = pfft_create_procmesh_2d(mpi_grp_%comm%MPI_VAL, column_size, row_size,
fft_array(jj)%comm%MPI_VAL)
560 if (ierror /= 0)
then
561 message(1) =
"The number of rows and columns in PFFT processor grid is not equal to "
562 message(2) =
"the number of processor in the MPI communicator."
563 message(3) =
"Please check it."
577 if (
present(comm)) comm =
fft_array(jj)%comm
580 select case (library_)
588 if (this%single)
then
592 else if (this%aligned_memory)
then
599 assert(this%howmany == 1)
614 n3 = ceiling(real(2*alloc_size)/real(n_1*n_2))
615 safe_allocate(
fft_array(jj)%drs_data(1:n_1, 1:n_2, 1:n3, 1:1))
625 n3 = ceiling(real(alloc_size)/real(n_3*n_1))
626 safe_allocate(
fft_array(jj)%fs_data(1:n_3, 1:n_1, 1:n3, 1:1))
654 select case (library_)
658 if (this%single)
then
677 fft_dim,
fft_array(jj)%rs_n_global, nthreads=nthreads)
726 call cuda_fft_plan_many(
fft_array(jj)%cuda_plan_fw, this%howmany, &
729 call cuda_fft_plan_many(
fft_array(jj)%cuda_plan_bw, this%howmany, &
733 call cuda_fft_plan_many(
fft_array(jj)%cuda_plan_fw, this%howmany, &
736 call cuda_fft_plan_many(
fft_array(jj)%cuda_plan_bw, this%howmany, &
757 number_points = number_points *
fft_array(jj)%rs_n_global(idir)
766 if (any(nn(1:fft_dim) /= nn_temp(1:fft_dim)))
then
768 call messages_write(
' Inefficient FFT grid. A better grid would be: ')
776 select case (library_)
778 write(
message(1),
'(a)')
"Info: FFT library = PFFT"
779 write(
message(2),
'(a)')
"Info: PFFT processor grid"
780 write(
message(3),
'(a, i9)')
" No. of processors = ", mpi_grp_%size
781 write(
message(4),
'(a, i9)')
" No. of columns in the proc. grid = ", column_size
782 write(
message(5),
'(a, i9)')
" No. of rows in the proc. grid = ", row_size
783 write(
message(6),
'(a, i9)')
" The size of integer is = ", c_intptr_t
808 type(
fft_t),
intent(inout) :: this
812 real(real64),
contiguous,
intent(in) :: xx(:,:)
813 integer,
contiguous,
optional,
intent(in) :: nn(:)
815 logical,
optional,
intent(in) :: dft_nodes
821 assert(
size(xx,2) == 3)
836 real(real64),
allocatable :: freq(:,:)
838 safe_allocate(freq(1:maxval(nn(1:3)), 1:3))
842 freq(ii, idir) = real(
pad_feq(ii, nn(idir), .
true.), real64)
846 freq(1:nn(1),1), freq(1:nn(2),2), freq(1:nn(3),3), period = real(nn(1:3), real64))
847 safe_deallocate_a(freq)
851 xx(1:nn(1),1), xx(1:nn(2),2), xx(1:nn(3),3))
873 type(
fft_t),
intent(inout) :: this
881 message(1) =
"Trying to deallocate FFT that has not been allocated."
890 call fftwf_destroy_plan(
fft_array(ii)%planf)
891 call fftwf_destroy_plan(
fft_array(ii)%planb)
896 call fftw_destroy_plan(
fft_array(ii)%planb)
898 if (this%aligned_memory)
then
906 call pfft_destroy_plan(
fft_array(ii)%planf)
907 call pfft_destroy_plan(
fft_array(ii)%planb)
909 safe_deallocate_p(
fft_array(ii)%drs_data)
910 safe_deallocate_p(
fft_array(ii)%zrs_data)
916 call cuda_fft_destroy(
fft_array(ii)%cuda_plan_bw)
938 type(
fft_t),
intent(in) :: fft_i
939 type(
fft_t),
intent(inout) :: fft_o
943 if (fft_o%slot > 0)
then
946 assert(fft_i%slot >= 1.and.fft_i%slot <= fft_max)
957 function fft_pack(dims3, howmany)
result(dims4)
958 integer,
intent(in) :: dims3(3)
959 integer,
intent(in) :: howmany
962 dims4(1:3) = dims3(1:3)
967 subroutine fft_get_dims(fft, rs_n_global, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
968 type(
fft_t),
intent(in) :: fft
969 integer,
intent(out) :: rs_n_global(1:3)
970 integer,
intent(out) :: fs_n_global(1:3)
971 integer,
intent(out) :: rs_n(1:3)
972 integer,
intent(out) :: fs_n(1:3)
973 integer,
intent(out) :: rs_istart(1:3)
974 integer,
intent(out) :: fs_istart(1:3)
981 rs_n_global(1:3) =
fft_array(slot)%rs_n_global(1:3)
982 fs_n_global(1:3) =
fft_array(slot)%fs_n_global(1:3)
985 rs_istart(1:3) =
fft_array(slot)%rs_istart(1:3)
986 fs_istart(1:3) =
fft_array(slot)%fs_istart(1:3)
993 pure function pad_feq(ii, nn, mode)
994 integer,
intent(in) :: ii,nn
995 logical,
intent(in) ::
mode
1001 if (ii <= nn/2 + 1)
then
1018 integer function fft_size(size, factors, parity)
1019 integer,
intent(in) :: size
1020 integer,
intent(in) :: factors(:)
1021 integer,
intent(in) :: parity
1025 integer,
allocatable :: exponents(:)
1029 nfactors = ubound(factors, dim = 1)
1031 safe_allocate(exponents(1:nfactors))
1036 if (nondiv == 1 .and. mod(
fft_size, 2) == parity)
exit
1040 safe_deallocate_a(exponents)
1048 integer,
intent(in) :: num
1049 integer,
intent(in) :: nfactors
1050 integer,
intent(in) :: factors(:)
1051 integer,
contiguous,
intent(out) :: exponents(:)
1052 integer,
intent(out) :: nondiv
1059 do ifactor = 1, nfactors
1060 exponents(ifactor) = 0
1062 if (mod(nondiv, factors(ifactor)) /= 0)
exit
1063 nondiv = nondiv/factors(ifactor)
1064 exponents(ifactor) = exponents(ifactor) + 1
1075 type(
fft_t),
intent(in) :: fft
1077 real(real64) :: fullsize
1081 fullsize = real(fft%howmany, real64)*product(real(fft%fs_n(1:3), real64))
1089 pure subroutine fft_gg_transform(gg_in, temp, periodic_dim, latt, qq, gg, modg2)
1090 integer,
contiguous,
intent(in) :: gg_in(:)
1091 real(real64),
contiguous,
intent(in) :: temp(:)
1092 integer,
intent(in) :: periodic_dim
1094 real(real64),
contiguous,
intent(in) :: qq(:)
1095 real(real64),
contiguous,
intent(out) :: gg(:)
1096 real(real64),
intent(out) :: modg2
1100 gg(1:3) = real(gg_in(1:3), real64)
1101 gg(1:periodic_dim) = gg(1:periodic_dim) + qq(1:periodic_dim)
1102 gg(1:3) = gg(1:3) * temp(1:3)
1103 gg(1:3) = matmul(latt%klattice_primitive(1:3,1:3),gg(1:3))
1104 modg2 = sum(gg(1:3)**2)
1113 type(
fft_t),
intent(in) :: fft
1116 scaling_factor =
m_one
1118 select case (
fft_array(fft%slot)%library)
1121 scaling_factor =
m_one/real(
fft_array(fft%slot)%rs_n_global(1), real64)
1122 scaling_factor = scaling_factor/real(
fft_array(fft%slot)%rs_n_global(2), real64)
1123 scaling_factor = scaling_factor/real(
fft_array(fft%slot)%rs_n_global(3), real64)
1134 real(real64) function
fft_get_ecut_from_box(box_dim, fs_istart, latt, gspacing, periodic_dim, qq) result(ecut)
1135 integer,
contiguous,
intent(in) :: box_dim(:)
1136 integer,
contiguous,
intent(in) :: fs_istart(:)
1138 real(real64),
contiguous,
intent(in) :: gspacing(:)
1139 integer,
intent(in) :: periodic_dim
1140 real(real64),
contiguous,
intent(in) :: qq(:)
1142 integer :: lx, ix, iy, iz, idir, idir2, idir3
1143 real(real64) :: dminsq, gg(3), modg2
1144 integer :: box_dim_(3), ixx(3)
1145 integer :: ming(3), maxg(3)
1149 assert(periodic_dim > 0)
1151 box_dim_(1:periodic_dim) = box_dim(1:periodic_dim)
1152 if (periodic_dim < 3) box_dim_(periodic_dim+1:3) = 1
1157 do idir = 1, periodic_dim
1158 do lx = 1, box_dim(idir)
1159 ix = fs_istart(idir) + lx - 1
1161 ming(idir) = min(ming(idir), ixx(idir))
1162 maxg(idir) = max(maxg(idir), ixx(idir))
1164 maxg(idir) = min(abs(ming(idir)), maxg(idir))
1169 do idir = 1, periodic_dim
1170 idir2 = mod(idir, 3)+1
1171 idir3 = mod(idir+1, 3)+1
1174 ixx(idir) = -maxg(idir)
1175 do iy = -maxg(idir2), maxg(idir2)
1177 do iz = -maxg(idir3), maxg(idir3)
1180 dminsq = min(dminsq, sum(gg(1:periodic_dim)**2))
1184 ixx(idir) = maxg(idir)
1185 do iy = -maxg(idir2), maxg(idir2)
1187 do iz = -maxg(idir3), maxg(idir3)
1190 dminsq = min(dminsq, sum(gg(1:periodic_dim)**2))
1201#include "fft_inc.F90"
1204#include "complex.F90"
1205#include "fft_inc.F90"
1208#include "real32.F90"
1209#include "fft_inc.F90"
1212#include "complex32.F90"
1213#include "fft_inc.F90"
if write to the Free Software Franklin Fifth USA !The assertions are ignored if the code is compiled in not debug mode(NDEBUG ! is defined). Otherwise it is merely a logical assertion that
double log(double __x) __attribute__((__nothrow__
pure logical function, public accel_is_enabled()
type(accel_t), public accel
Fast Fourier Transform module. This module provides a single interface that works with different FFT ...
subroutine sfft_forward_single_3d(fft, in, out, norm)
subroutine zfft_forward_accel(fft, in, out)
subroutine dfft_backward_1d(fft, in, out)
integer, parameter cufft_z2d
subroutine get_exponents(num, nfactors, factors, exponents, nondiv)
subroutine, public fft_init_stage1(this, namespace, XX, nn, dft_nodes)
Some fft-libraries (only NFFT for the moment) need an additional precomputation stage that depends on...
subroutine, public fft_all_init(namespace)
initialize the table
real(real64) function, public fft_get_ecut_from_box(box_dim, fs_istart, latt, gspacing, periodic_dim, qq)
Given an fft box (fixed by the real-space grid), it returns the cutoff energy of the sphere that fits...
subroutine sfft_backward_accel(fft, in, out)
subroutine zfft_forward_many_3d(fft, in, out, norm)
subroutine zfft_backward_single_3d(fft, in, out, norm)
subroutine dfft_forward_accel(fft, in, out)
subroutine cfft_forward_single_3d(fft, in, out, norm)
subroutine dfft_forward_single_3d(fft, in, out, norm)
subroutine sfft_forward_many_3d(fft, in, out, norm)
subroutine, public fft_end(this)
subroutine cfft_backward_accel(fft, in, out)
real(real64) pure function, public fft_scaling_factor(fft)
This function returns the factor required to normalize a function after a forward and backward transf...
subroutine, public fft_init(this, nn, dim, type, library, optimize, optimize_parity, comm, mpi_grp, use_aligned, howmany, nthreads, single)
integer, public fft_default_lib
integer, parameter cufft_z2z
pure integer function, public pad_feq(ii, nn, mode)
convert between array index and G-vector
subroutine dfft_backward_single_3d(fft, in, out, norm)
subroutine zfft_backward_1d(fft, in, out)
subroutine cfft_backward_single_3d(fft, in, out, norm)
subroutine cfft_forward_many_3d(fft, in, out, norm)
integer, parameter, public fftlib_accel
integer function, dimension(4) fft_pack(dims3, howmany)
Rank-4 data-array shape (dims3, howmany) with the batch as the trailing (batch-last) axis.
subroutine, public fft_all_end()
delete all plans
logical, save, public fft_initialized
integer function fft_size(size, factors, parity)
subroutine fft_operation_count(fft)
subroutine sfft_forward_accel(fft, in, out)
subroutine zfft_backward_accel(fft, in, out)
integer, parameter cufft_c2r
subroutine sfft_backward_many_3d(fft, in, out, norm)
integer, parameter cufft_c2c
integer, parameter, public fft_real
subroutine, public fft_get_dims(fft, rs_n_global, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
integer, parameter, public fft_complex
integer, save fft_prepare_plan
integer, parameter, public fftlib_nfft
subroutine, public fft_copy(fft_i, fft_o)
subroutine dfft_backward_many_3d(fft, in, out, norm)
type(pnfft_t), save pnfft_options
subroutine cfft_backward_many_3d(fft, in, out, norm)
subroutine zfft_forward_single_3d(fft, in, out, norm)
integer, dimension(fft_max), save fft_refs
subroutine zfft_backward_many_3d(fft, in, out, norm)
subroutine dfft_forward_1d(fft, in, out)
subroutine cfft_forward_accel(fft, in, out)
integer, parameter cufft_d2z
type(fft_t), dimension(fft_max), save fft_array
integer, parameter fft_null
integer, parameter, public fftlib_pnfft
subroutine sfft_backward_single_3d(fft, in, out, norm)
subroutine dfft_forward_many_3d(fft, in, out, norm)
integer, parameter cufft_r2c
subroutine zfft_forward_1d(fft, in, out)
pure subroutine, public fft_gg_transform(gg_in, temp, periodic_dim, latt, qq, gg, modg2)
Convert FFT grid index into the Cartesian reciprocal-space vector .
integer, parameter, public fftlib_pfft
subroutine dfft_backward_accel(fft, in, out)
integer, parameter, public fftlib_fftw
subroutine, public fftw_free_memory(is_real, drs_data, zrs_data, fs_data)
subroutine, public fftw_get_dims(rs_n, is_real, fs_n)
subroutine, public fftw_prepare_plan_single(plan, dim, n, howmany, is_real, sign, flags, sin_, cin_, cout_)
Single-precision counterpart of fftw_prepare_plan. Only the aligned-memory path is supported (the sin...
subroutine, public fftw_alloc_memory_single(rs_dims, is_real, fs_dims, srs_data, crs_data, cfs_data)
subroutine, public fftw_free_memory_single(is_real, srs_data, crs_data, cfs_data)
subroutine, public fftw_prepare_plan(plan, dim, n, howmany, is_real, sign, flags_, din_, cin_, cout_)
subroutine, public fftw_alloc_memory(rs_dims, is_real, fs_dims, drs_data, zrs_data, fs_data)
Allocate the FFTW work buffers from the rank-4 (batch-last) dimension arrays (see fft_pack).
Non-equispaced FFTs through the C interface of the FINUFFT library.
subroutine, public finufft_write_info(finufft)
subroutine, public finufft_precompute(finufft, X1, X2, X3, period)
Hand the spatial nodes to FINUFFT.
subroutine, public finufft_read_options(finufft, namespace)
subroutine, public finufft_init(finufft, finufft_options, N, dim, M, nthreads)
subroutine, public finufft_end(finufft)
real(real64), parameter, public m_two
real(real64), parameter, public m_huge
real(real64), parameter, public m_zero
real(real64), parameter, public m_half
real(real64), parameter, public m_one
subroutine, public messages_not_implemented(feature, namespace)
subroutine, public messages_warning(no_lines, all_nodes, namespace)
subroutine, public messages_new_line()
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
subroutine, public messages_fatal(no_lines, only_root_writes, namespace)
subroutine, public messages_input_error(namespace, var, details, row, column)
subroutine, public messages_info(no_lines, iunit, debug_only, stress, all_nodes, namespace)
type(mpi_comm), parameter, public mpi_comm_undefined
used to indicate a communicator has not been initialized
type(mpi_grp_t), public mpi_world
The low level module to work with the PFFT library. http://www-user.tu-chemnitz.de/~mpip/software....
subroutine, public pfft_prepare_plan_r2c(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
subroutine, public pfft_prepare_plan_c2c(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
subroutine, public pfft_decompose(n_proc, dim1, dim2)
Decompose all available processors in 2D processor grid, most equally possible.
subroutine, public pfft_prepare_plan_c2r(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
subroutine, public pfft_get_dims(rs_n_global, mpi_comm, is_real, alloc_size, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
The includes for the PFFT.
The low level module to work with the PNFFT library. http://www-user.tu-chemnitz.de/~mpip/software....
subroutine, public pnfft_copy_params(in, out)
subroutine, public pnfft_set_sp_nodes(pnfft, namespace, X)
subroutine, public pnfft_init_plan(pnfft, pnfft_options, comm, fs_n_global, fs_n, fs_istart, rs_n, rs_istart)
subroutine, public pnfft_write_info(pnfft)
subroutine, public pnfft_guru_options(pnfft, namespace)
subroutine, public pnfft_end(pnfft)
subroutine, public pnfft_init_procmesh(pnfft, mpi_grp, comm)
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
This module defines the unit system, used for input and output.
type(unit_t), public unit_megabytes
For large amounts of data (natural code units are bytes)
This is defined even when running serial.