Octopus
fft.F90
Go to the documentation of this file.
1!! Copyright (C) 2002-2006 M. Marques, A. Castro, A. Rubio, G. Bertsch
2!! Copyright (C) 2011 J. Alberdi-Rodriguez, P. Garcia RisueƱo, M. Oliveira
3!!
4!! This program is free software; you can redistribute it and/or modify
5!! it under the terms of the GNU General Public License as published by
6!! the Free Software Foundation; either version 2, or (at your option)
7!! any later version.
8!!
9!! This program is distributed in the hope that it will be useful,
10!! but WITHOUT ANY WARRANTY; without even the implied warranty of
11!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12!! GNU General Public License for more details.
13!!
14!! You should have received a copy of the GNU General Public License
15!! along with this program; if not, write to the Free Software
16!! Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17!! 02110-1301, USA.
18!!
19
20#include "global.h"
21
25module fft_oct_m
26 use, intrinsic :: iso_c_binding
27 use, intrinsic :: ieee_arithmetic
28
29 use accel_oct_m
30 use fftw_oct_m
32 use debug_oct_m
33 use global_oct_m
34 use, intrinsic :: iso_fortran_env
39 use mpi_oct_m
41#ifdef HAVE_NFFT
43#endif
44#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
45 use omp_lib
46#endif
47 use parser_oct_m
48 use pfft_oct_m
50 use pnfft_oct_m
52 use types_oct_m
55
56 implicit none
57
58 private
59 public :: &
60 fft_t, &
63 fft_init, &
65 fft_end, &
66 fft_copy, &
68 pad_feq, &
80
81
83 integer, public, parameter :: &
84 FFT_NONE = 0, &
85 fft_real = 1, &
86 fft_complex = 2
87
88 integer, public, parameter :: &
89 FFTLIB_NONE = 0, &
90 fftlib_fftw = 1, &
91 fftlib_pfft = 2, &
92 fftlib_accel = 3, &
93 fftlib_nfft = option__fftlibrary__nfft, &
94 fftlib_pnfft = 5
95
96 integer, parameter :: &
97 FFT_MAX = 10, &
98 fft_null = -1
99
100
101 type fft_t
102 private
103 integer :: slot = 0
104
105 integer, public :: type
106 integer, public :: library
107 integer :: howmany
112
113 type(MPI_Comm) :: comm
114 integer :: rs_n_global(3)
115 integer :: fs_n_global(3)
116 integer :: rs_n(3)
117 integer :: fs_n(3)
118 integer :: rs_istart(1:3)
119 integer :: fs_istart(1:3)
120
122 integer, public :: stride_rs(1:3)
123 integer, public :: stride_fs(1:3)
124
125 type(c_ptr) :: planf
126 type(c_ptr) :: planb
127 !integer(ptrdiff_t_kind) :: pfft_planf !< PFFT plan for forward transform
128 !integer(ptrdiff_t_kind) :: pfft_planb !< PFFT plan for backward transform
129
132 real(real64), contiguous, pointer, public :: drs_data(:,:,:,:)
133 complex(real64), contiguous, pointer, public :: zrs_data(:,:,:,:)
134 complex(real64), contiguous, pointer, public :: fs_data(:,:,:,:)
137 real(real32), contiguous, pointer, public :: srs_data(:,:,:,:)
138 complex(real32), contiguous, pointer, public :: crs_data(:,:,:,:)
139 complex(real32), contiguous, pointer, public :: cfs_data(:,:,:,:)
140 logical, public :: single = .false.
141 type(c_ptr) :: cuda_plan_fw
142 type(c_ptr) :: cuda_plan_bw
143#ifdef HAVE_NFFT
144 type(finufft_t), public :: finufft
145#endif
146 type(pnfft_t), public :: pnfft
147
148 logical, public :: aligned_memory
149 end type fft_t
150
151 interface dfft_forward
153 end interface dfft_forward
154
155 interface zfft_forward
157 end interface zfft_forward
158
159 interface dfft_backward
161 end interface dfft_backward
162
163 interface zfft_backward
165 end interface zfft_backward
166
167 ! Single-precision host and accelerator transforms
168 interface sfft_forward
170 end interface sfft_forward
171
172 interface cfft_forward
174 end interface cfft_forward
176 interface sfft_backward
178 end interface sfft_backward
180 interface cfft_backward
182 end interface cfft_backward
183
184 logical, save, public :: fft_initialized = .false.
185 integer, save :: fft_refs(fft_max)
186 type(fft_t), save :: fft_array(fft_max)
187 logical :: fft_optimize
188 integer, save :: fft_prepare_plan
189 integer, public :: fft_default_lib = -1
190#ifdef HAVE_NFFT
191 type(finufft_t), save :: finufft_options
192#endif
193 type(pnfft_t), save :: pnfft_options
195 integer, parameter :: &
196 cufft_r2c = int(z'2a'), &
197 cufft_c2r = int(z'2c'), &
198 cufft_c2c = int(z'29'), &
199 cufft_d2z = int(z'6a'), &
200 cufft_z2d = int(z'6c'), &
201 cufft_z2z = int(z'69')
202
203contains
205 ! ---------------------------------------------------------
207 subroutine fft_all_init(namespace)
208 type(namespace_t), intent(in) :: namespace
210 integer :: ii, fft_default
211#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
212 integer :: iret
213#endif
215 push_sub(fft_all_init)
216
219 !%Variable FFTOptimize
220 !%Type logical
221 !%Default yes
222 !%Section Mesh::FFTs
223 !%Description
224 !% Should <tt>octopus</tt> optimize the FFT dimensions?
225 !% This means that the mesh to which FFTs are applied is not taken to be as small
226 !% as possible: some points may be added to each direction in order to get a "good number"
227 !% for the performance of the FFT algorithm.
228 !% The best FFT grid dimensions are given by <math>2^a 3^b 5^c 7^d 11^e 13^f</math>
229 !% where <math>a,b,c,d</math> are arbitrary and <math>e,f</math> are 0 or 1.
230 !% (<a href=http://www.fftw.org/doc/Complex-DFTs.html>ref</a>).
231 !% In some cases, namely when using
232 !% the split-operator, or Suzuki-Trotter propagators, this option should be turned off.
233 !% For spatial FFTs in periodic directions, the grid is never optimized, but a warning will
234 !% be written if the number is not good, with a suggestion of a better one to use, so you
235 !% can try a different spacing if you want to get a good number.
236 !%End
237 call parse_variable(namespace, 'FFTOptimize', .true., fft_optimize)
238 do ii = 1, fft_max
239 fft_refs(ii) = fft_null
240 end do
241
242 !%Variable FFTPreparePlan
243 !%Type integer
244 !%Default fftw_measure
245 !%Section Mesh::FFTs
246 !%Description
247 !% The FFTs are performed in octopus with the help of <a href=http://www.fftw.org>FFTW</a> and similar packages.
248 !% Before doing the actual computations, this package prepares a "plan", which means that
249 !% the precise numerical strategy to be followed to compute the FFT is machine/compiler-dependent,
250 !% and therefore the software attempts to figure out which is this precise strategy (see the
251 !% FFTW documentation for details). This plan preparation, which has to be done for each particular
252 !% FFT shape, can be done exhaustively and carefully (slow), or merely estimated. Since this is
253 !% a rather critical numerical step, by default it is done carefully, which implies a longer initial
254 !% initialization, but faster subsequent computations. You can change this behaviour by changing
255 !% this <tt>FFTPreparePlan</tt> variable, and in this way you can force FFTW to do a fast guess or
256 !% estimation of which is the best way to perform the FFT.
257 !%Option fftw_measure 0
258 !% This plan implies a longer initialization, but involves a more careful analysis
259 !% of the strategy to follow, and therefore more efficient FFTs. A side effect of the runtime
260 !% choices is that this plan can introduce slight numerical fluctuations between runs.
261 !%Option fftw_estimate 64
262 !% This is the "fast initialization" scheme, in which the plan is merely guessed from "reasonable"
263 !% assumptions. This is the default option, as it guarantees stable results
264 !%Option fftw_patient 32
265 !% It is like fftw_measure, but considers a wider range of algorithms and often produces a
266 !% "more optimal" plan (especially for large transforms), but at the expense of several times
267 !% longer planning time (especially for large transforms).
268 !%Option fftw_exhaustive 8
269 !% It is like fftw_patient, but considers an even wider range of algorithms,
270 !% including many that we think are unlikely to be fast, to produce the most optimal
271 !% plan but with a substantially increased planning time.
272 !%End
273 call parse_variable(namespace, 'FFTPreparePlan', fftw_estimate, fft_prepare_plan)
274 if (.not. varinfo_valid_option('FFTPreparePlan', fft_prepare_plan)) then
275 call messages_input_error(namespace, 'FFTPreparePlan')
276 end if
277
278 !%Variable FFTLibrary
279 !%Type integer
280 !%Section Mesh::FFTs
281 !%Default fftw
282 !%Description
283 !% (experimental) You can select the FFT library to use.
284 !%Option fftw 1
285 !% Uses FFTW3 library.
286 !%Option pfft 2
287 !% (experimental) Uses PFFT library, which has to be linked.
288 !%Option accel 3
289 !% Uses a GPU accelerated library. This only
290 !% works if Octopus was compiled with HIP, or CUDA support.
291 !%Option nfft
292 !% (experimental) Uses the non-equispaced FFT of the FINUFFT library, which has to be linked.
293 !% On the equispaced cube this is the same transform as <tt>fftw</tt>, up to
294 !% <tt>FINUFFTTolerance</tt>, but slower and neither batched nor parallel in domains.
295 !%End
296 fft_default = fftlib_fftw
297 if(accel_is_enabled()) then
298 fft_default = fftlib_accel
299 end if
300 call parse_variable(namespace, 'FFTLibrary', fft_default, fft_default_lib)
301
302 if (.not. varinfo_valid_option('FFTLibrary', fft_default_lib)) then
303 call messages_input_error(namespace, 'FFTLibrary')
304 endif
305
306#ifndef HAVE_NFFT
307 if (fft_default_lib == fftlib_nfft) then
308 message(1) = 'FFTLibrary = nfft requires FINUFFT but that library was not linked.'
309 call messages_fatal(1, namespace=namespace)
310 end if
311#endif
312
313 if (fft_default_lib == fftlib_accel) then
314#if ! defined(HAVE_CUDA)
315 call messages_write('You have selected the Accelerated FFT, but Octopus was compiled', new_line = .true.)
316 call messages_write('without CUDA support.')
317 call messages_fatal()
318#endif
319 if (.not. accel_is_enabled()) then
320 call messages_write('You have selected the accelerated FFT, but acceleration is disabled.')
321 call messages_fatal()
322 end if
323 end if
324
325#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
326 if (omp_get_max_threads() > 1) then
327
328 call messages_write('Info: Initializing Multi-threaded FFTW')
329 call messages_info()
330
331 iret = fftw_init_threads()
332 if (iret == 0) then
333 call messages_write('Initialization of FFTW3 threads failed.')
334 call messages_fatal()
335 end if
336 call fftw_plan_with_nthreads(omp_get_max_threads())
337
338 end if
339#endif
340#ifdef HAVE_NFFT
341 call finufft_read_options(finufft_options, namespace)
342#endif
343 call pnfft_guru_options(pnfft_options, namespace)
344
345 pop_sub(fft_all_init)
346 end subroutine fft_all_init
347
348
349 ! ---------------------------------------------------------
351 subroutine fft_all_end()
352 integer :: ii
353
354 push_sub(fft_all_end)
355
356 do ii = 1, fft_max
357 if (fft_refs(ii) /= fft_null) then
358 call fft_end(fft_array(ii))
359 end if
360 end do
361
362#ifdef HAVE_PFFT
363 call pfft_cleanup()
364#endif
365
366#if defined(HAVE_OPENMP) && defined(HAVE_FFTW3_THREADS)
367 call fftw_cleanup_threads()
368#else
369 call fftw_cleanup()
370#endif
371 ! the single-precision planner is never threaded, so it always needs the serial cleanup
372 call fftwf_cleanup()
373
374 fft_initialized = .false.
375
376 pop_sub(fft_all_end)
377 end subroutine fft_all_end
378
379 ! ---------------------------------------------------------
380 subroutine fft_init(this, nn, dim, type, library, optimize, optimize_parity, comm, mpi_grp, use_aligned, &
381 howmany, nthreads, single)
382 type(fft_t), intent(inout) :: this
383 integer, intent(inout) :: nn(3)
384 integer, intent(in) :: dim
385 integer, intent(in) :: type
386 integer, intent(in) :: library
387 logical, intent(in) :: optimize(3)
388 integer, intent(in) :: optimize_parity(3)
390 type(mpi_comm), optional, intent(out) :: comm
391 type(mpi_grp_t), optional, intent(in) :: mpi_grp
392 logical, optional, intent(in) :: use_aligned
393 integer, optional, intent(in) :: howmany
394 integer, optional, intent(in) :: nthreads
395 logical, optional, intent(in) :: single
396
397 integer :: ii, jj, fft_dim, idir, column_size, row_size, n3
398 integer :: n_1, n_2, n_3, nn_temp(3)
399 integer :: library_
400 type(mpi_grp_t) :: mpi_grp_
401 integer(int64) :: number_points, alloc_size
402
403#ifdef HAVE_PFFT
404 integer :: ierror
405#endif
406
407 push_sub(fft_init)
408
409 assert(fft_initialized)
410 assert(type == fft_real .or. type == fft_complex)
411
412 mpi_grp_ = mpi_world
413 if (present(mpi_grp)) mpi_grp_ = mpi_grp
414
415 this%aligned_memory = optional_default(use_aligned, .false.)
416 this%howmany = optional_default(howmany, 1)
417 assert(this%howmany > 0)
418 this%single = optional_default(single, .false.)
419 ! Single-precision FFTW requires aligned memory; accelerator buffers are external.
420 if (this%single) then
421 this%aligned_memory = .true.
422 end if
423
424 ! First, figure out the dimensionality of the FFT.
425 fft_dim = 0
426 do ii = 1, dim
427 if (nn(ii) <= 1) exit
428 fft_dim = fft_dim + 1
429 end do
430
431 if (fft_dim == 0) then
432 message(1) = "Internal error in fft_init: apparently, a 1x1x1 FFT is required."
433 call messages_fatal(1)
434 end if
435
436 if (fft_dim > 3) call messages_not_implemented('FFT for dimension > 3')
437
438 library_ = library
439
440 ! Only FFTW and the accelerator have single-precision plans; without this check the
441 ! failure would surface as 'Invalid FFT library' at the first transform.
442 if (this%single .and. library_ /= fftlib_fftw .and. library_ /= fftlib_accel) then
443 call messages_not_implemented('single-precision FFT for this FFT library')
444 end if
445
446 nn_temp(1:fft_dim) = nn(1:fft_dim)
447
448 select case (library_)
449 case (fftlib_accel)
450 ! FFT optimization
451 if(any(optimize_parity(1:fft_dim) > 1)) then
452 message(1) = "Internal error in fft_init: optimize_parity must be negative, 0, or 1."
453 call messages_fatal(1)
454 end if
455
456 do ii = 1, fft_dim
457 nn_temp(ii) = fft_size(nn(ii), (/2, 3, 5, 7/), optimize_parity(ii))
458 if (fft_optimize .and. optimize(ii)) nn(ii) = nn_temp(ii)
459 end do
460
461 case (fftlib_nfft)
462 assert(this%howmany == 1) ! TODO: NFFT does not support howmany > 1
463
464 do ii = 1, fft_dim
465 !NFFT likes even grids
466 !The underlying FFT grids are optimized inside the backend
467 if (int(nn(ii)/2)*2 /= nn(ii) .and. (fft_optimize .and. optimize(ii)))&
468 nn(ii)=nn(ii)+1
469 end do
470
471 case (fftlib_pnfft)
472 assert(this%howmany == 1) ! TODO: PNFFT does not support howmany > 1
473
474 do ii = 1, fft_dim
475 !also PNFFT likes even grids
476 if (int(nn(ii)/2)*2 /= nn(ii)) nn(ii) = nn(ii) + 1
477 end do
478
479 if (fft_dim < 3) then
480 call messages_not_implemented('PNFFT support for dimension < 3')
481 end if
482
483 case default
484
485 if (fft_dim < 3 .and. library_ == fftlib_pfft) then
486 call messages_not_implemented('PFFT support for dimension < 3')
487 end if
488
489 ! FFT optimization
490 if (any(optimize_parity(1:fft_dim) > 1)) then
491 message(1) = "Internal error in fft_init: optimize_parity must be negative, 0, or 1."
492 call messages_fatal(1)
493 end if
494
495 do ii = 1, fft_dim
496 call loct_fft_optimize(nn_temp(ii), optimize_parity(ii))
497 if (fft_optimize .and. optimize(ii)) nn(ii) = nn_temp(ii)
498 end do
499
500 end select
501
502 ! find out if fft has already been allocated
503 jj = 0
504 do ii = fft_max, 1, -1
505 if (fft_refs(ii) /= fft_null) then
506 if (all(nn(1:dim) == fft_array(ii)%rs_n_global(1:dim)) .and. type == fft_array(ii)%type &
507 .and. library_ == fft_array(ii)%library .and. library_ /= fftlib_nfft &
508 .and. library_ /= fftlib_pnfft &
509 .and. this%howmany == fft_array(ii)%howmany &
510 .and. (this%single .eqv. fft_array(ii)%single) &
511 .and. (this%aligned_memory .eqv. fft_array(ii)%aligned_memory)) then
512
513 ! NFFT and PNFFT plans are always allocated from scratch since they
514 ! are very likely to be different
515 this = fft_array(ii) ! return a copy
516 fft_refs(ii) = fft_refs(ii) + 1 ! increment the ref count
517 if (present(comm)) comm = fft_array(ii)%comm ! also return the MPI communicator
518 pop_sub(fft_init)
519 return
520 end if
521 else
522 jj = ii
523 end if
524 end do
525
526 if (jj == 0) then
527 message(1) = "Not enough slots for FFTs."
528 message(2) = "Please increase FFT_MAX in fft.F90 and recompile."
529 call messages_fatal(2)
530 end if
531
532 ! jj now contains an empty slot
533 fft_refs(jj) = 1
534 fft_array(jj)%slot = jj
535 fft_array(jj)%type = type
536 fft_array(jj)%library = library_
537 fft_array(jj)%howmany = this%howmany
538 fft_array(jj)%rs_n_global(1:dim) = nn(1:dim)
539 fft_array(jj)%rs_n_global(dim+1:) = 1
540 nullify(fft_array(jj)%drs_data)
541 nullify(fft_array(jj)%zrs_data)
542 nullify(fft_array(jj)%fs_data)
543 nullify(fft_array(jj)%srs_data)
544 nullify(fft_array(jj)%crs_data)
545 nullify(fft_array(jj)%cfs_data)
546 fft_array(jj)%single = this%single
547
548 fft_array(jj)%aligned_memory = this%aligned_memory
549
550 ! Initialize parallel communicator
551 select case (library_)
552 case (fftlib_pfft)
553#ifdef HAVE_PFFT
554 call pfft_init()
555
556 call pfft_decompose(mpi_grp_%size, column_size, row_size)
557
558 ierror = pfft_create_procmesh_2d(mpi_grp_%comm%MPI_VAL, column_size, row_size, fft_array(jj)%comm%MPI_VAL)
559
560 if (ierror /= 0) then
561 message(1) = "The number of rows and columns in PFFT processor grid is not equal to "
562 message(2) = "the number of processor in the MPI communicator."
563 message(3) = "Please check it."
564 call messages_fatal(3)
565 end if
566#endif
567
568 case (fftlib_pnfft)
569#ifdef HAVE_PNFFT
570 call pnfft_init_procmesh(fft_array(jj)%pnfft, mpi_grp_, fft_array(jj)%comm)
571#endif
572 case default
574
575 end select
576
577 if (present(comm)) comm = fft_array(jj)%comm
578
579 ! Get dimentions of arrays
580 select case (library_)
581 case (fftlib_fftw)
582 call fftw_get_dims(fft_array(jj)%rs_n_global, type == fft_real, fft_array(jj)%fs_n_global)
583 fft_array(jj)%rs_n = fft_array(jj)%rs_n_global
584 fft_array(jj)%fs_n = fft_array(jj)%fs_n_global
585 fft_array(jj)%rs_istart = 1
586 fft_array(jj)%fs_istart = 1
587
588 if (this%single) then
589 call fftw_alloc_memory_single(fft_pack(fft_array(jj)%rs_n_global, this%howmany), &
590 type == fft_real, fft_pack(fft_array(jj)%fs_n_global, this%howmany), &
591 fft_array(jj)%srs_data, fft_array(jj)%crs_data, fft_array(jj)%cfs_data)
592 else if (this%aligned_memory) then
593 call fftw_alloc_memory(fft_pack(fft_array(jj)%rs_n_global, this%howmany), type == fft_real, &
594 fft_pack(fft_array(jj)%fs_n_global, this%howmany), &
595 fft_array(jj)%drs_data, fft_array(jj)%zrs_data, fft_array(jj)%fs_data)
596 end if
597
598 case (fftlib_pfft)
599 assert(this%howmany == 1) ! PFFT does not support howmany > 1
600#ifdef HAVE_PFFT
601 call pfft_get_dims(fft_array(jj)%rs_n_global, fft_array(jj)%comm%MPI_VAL, type == fft_real, &
602 alloc_size, fft_array(jj)%fs_n_global, fft_array(jj)%rs_n, &
603 fft_array(jj)%fs_n, fft_array(jj)%rs_istart, fft_array(jj)%fs_istart)
604#endif
605
606 ! Allocate memory. Note that PFFT may need extra memory space
607 ! and that in fourier space the function will be transposed. PFFT runs unbatched, so the
608 ! rank-4 arrays keep the pre-batching rank-3 sizing with the trailing batch axis fixed to 1.
609 if (type == fft_real) then
610 n_1 = max(1, fft_array(jj)%rs_n(1))
611 n_2 = max(1, fft_array(jj)%rs_n(2))
612 n_3 = max(1, fft_array(jj)%rs_n(3))
613
614 n3 = ceiling(real(2*alloc_size)/real(n_1*n_2))
615 safe_allocate(fft_array(jj)%drs_data(1:n_1, 1:n_2, 1:n3, 1:1))
616 else
617 n3 = ceiling(real(alloc_size)/real(fft_array(jj)%rs_n(1)*fft_array(jj)%rs_n(2)))
618 safe_allocate(fft_array(jj)%zrs_data(1:fft_array(jj)%rs_n(1), 1:fft_array(jj)%rs_n(2), 1:n3, 1:1))
619 end if
620
621 n_1 = max(1, fft_array(jj)%fs_n(1))
622 n_2 = max(1, fft_array(jj)%fs_n(2))
623 n_3 = max(1, fft_array(jj)%fs_n(3))
624
625 n3 = ceiling(real(alloc_size)/real(n_3*n_1))
626 safe_allocate(fft_array(jj)%fs_data(1:n_3, 1:n_1, 1:n3, 1:1))
627
628 case (fftlib_accel)
629 call fftw_get_dims(fft_array(jj)%rs_n_global, (type == fft_real), fft_array(jj)%fs_n_global)
630 fft_array(jj)%rs_n = fft_array(jj)%rs_n_global
631 fft_array(jj)%fs_n = fft_array(jj)%fs_n_global
632 fft_array(jj)%rs_istart = 1
633 fft_array(jj)%fs_istart = 1
634
635 case (fftlib_nfft)
636 fft_array(jj)%fs_n_global = fft_array(jj)%rs_n_global
637 fft_array(jj)%rs_n = fft_array(jj)%rs_n_global
638 fft_array(jj)%fs_n = fft_array(jj)%fs_n_global
639 fft_array(jj)%rs_istart = 1
640 fft_array(jj)%fs_istart = 1
641
642 case (fftlib_pnfft)
643 fft_array(jj)%fs_n_global = fft_array(jj)%rs_n_global
644 fft_array(jj)%rs_n = fft_array(jj)%rs_n_global
645 fft_array(jj)%fs_n = fft_array(jj)%fs_n_global
646 fft_array(jj)%rs_istart = 1
647 fft_array(jj)%fs_istart = 1
648 ! indices partition is performed together with the plan preparation
649
650
651 end select
652
653 ! Prepare plans
654 select case (library_)
655 case (fftlib_fftw)
656 ! for these calls, always pass all optional array arguments
657 ! only the associated pointers will actually be present in the called routine
658 if (this%single) then
659 call fftw_prepare_plan_single(fft_array(jj)%planf, fft_dim, fft_array(jj)%rs_n_global, this%howmany, &
660 type == fft_real, fftw_forward, fft_prepare_plan, &
661 sin_=fft_array(jj)%srs_data, cin_=fft_array(jj)%crs_data, cout_=fft_array(jj)%cfs_data)
662 call fftw_prepare_plan_single(fft_array(jj)%planb, fft_dim, fft_array(jj)%rs_n_global, this%howmany, &
663 type == fft_real, fftw_backward, fft_prepare_plan, &
664 sin_=fft_array(jj)%srs_data, cin_=fft_array(jj)%crs_data, cout_=fft_array(jj)%cfs_data)
665 else
666 call fftw_prepare_plan(fft_array(jj)%planf, fft_dim, fft_array(jj)%rs_n_global, this%howmany, &
667 type == fft_real, fftw_forward, fft_prepare_plan, &
668 din_=fft_array(jj)%drs_data, cin_=fft_array(jj)%zrs_data, cout_=fft_array(jj)%fs_data)
669 call fftw_prepare_plan(fft_array(jj)%planb, fft_dim, fft_array(jj)%rs_n_global, this%howmany, &
670 type == fft_real, fftw_backward, fft_prepare_plan, &
671 din_=fft_array(jj)%drs_data, cin_=fft_array(jj)%zrs_data, cout_=fft_array(jj)%fs_data)
672 end if
673
674 case (fftlib_nfft)
675#ifdef HAVE_NFFT
676 call finufft_init(fft_array(jj)%finufft, finufft_options, fft_array(jj)%rs_n_global, &
677 fft_dim, fft_array(jj)%rs_n_global, nthreads=nthreads)
678#endif
679 case (fftlib_pfft)
680#ifdef HAVE_PFFT
681 if (type == fft_real) then
682 call pfft_prepare_plan_r2c(fft_array(jj)%planf, fft_array(jj)%rs_n_global, fft_array(jj)%drs_data, &
683 fft_array(jj)%fs_data, fftw_forward, fft_prepare_plan, fft_array(jj)%comm%MPI_VAL)
684 call pfft_prepare_plan_c2r(fft_array(jj)%planb, fft_array(jj)%rs_n_global, fft_array(jj)%fs_data, &
685 fft_array(jj)%drs_data, fftw_backward, fft_prepare_plan, fft_array(jj)%comm%MPI_VAL)
686 else
687 call pfft_prepare_plan_c2c(fft_array(jj)%planf, fft_array(jj)%rs_n_global, fft_array(jj)%zrs_data, &
688 fft_array(jj)%fs_data, fftw_forward, fft_prepare_plan, fft_array(jj)%comm%MPI_VAL)
689 call pfft_prepare_plan_c2c(fft_array(jj)%planb, fft_array(jj)%rs_n_global, fft_array(jj)%fs_data, &
690 fft_array(jj)%zrs_data, fftw_backward, fft_prepare_plan, fft_array(jj)%comm%MPI_VAL)
691 end if
692#endif
693 case (fftlib_pnfft)
694#ifdef HAVE_PNFFT
695 call pnfft_copy_params(this%pnfft, fft_array(jj)%pnfft) ! pass default parameters like in NFFT
696
697 ! NOTE:
698 ! PNFFT (likewise NFFT) breaks the symmetry between real space and Fourier space
699 ! by allowing the possibility to have an unstructured grid in rs and by
700 ! using different parallelizations (the rs is transposed w.r.t. fs).
701 ! Octopus, in fourier_space_m, uses the convention for which the mapping
702 ! between rs and fs is done with a forward transform (and fs->rs with backward).
703 ! This is exactly the opposite of the definitions used by all the libraries
704 ! performing FFTs (PNFFT and NFFT included) [see e.g. M. Frigo, and S. G. Johnson, Proc.
705 ! IEEE 93, 216-231 (2005)].
706 ! While this leads to no problem on ordinary ffts where fs and rs can be exchanged
707 ! it does makes a fundamental difference for PNFFT (for some reason I don`t know NFFT
708 ! is still symmetric).
709 ! Therefore, in order to perform rs->fs tranforms with PNFFT one should use the
710 ! backward transform.
711
712 call pnfft_init_plan(fft_array(jj)%pnfft, pnfft_options, comm, fft_array(jj)%fs_n_global, &
713 fft_array(jj)%fs_n, fft_array(jj)%fs_istart, fft_array(jj)%rs_n, fft_array(jj)%rs_istart)
714#endif
715 case (fftlib_accel)
716
717 fft_array(jj)%stride_rs(1) = 1
718 fft_array(jj)%stride_fs(1) = 1
719 do ii = 2, fft_dim
720 fft_array(jj)%stride_rs(ii) = fft_array(jj)%stride_rs(ii - 1)*fft_array(jj)%rs_n(ii - 1)
721 fft_array(jj)%stride_fs(ii) = fft_array(jj)%stride_fs(ii - 1)*fft_array(jj)%fs_n(ii - 1)
722 end do
723
724#ifdef HAVE_CUDA
725 if (type == fft_real) then
726 call cuda_fft_plan_many(fft_array(jj)%cuda_plan_fw, this%howmany, &
727 fft_array(jj)%rs_n_global(3), fft_array(jj)%rs_n_global(2), fft_array(jj)%rs_n_global(1), &
728 merge(cufft_r2c, cufft_d2z, this%single), accel%cuda_stream)
729 call cuda_fft_plan_many(fft_array(jj)%cuda_plan_bw, this%howmany, &
730 fft_array(jj)%rs_n_global(3), fft_array(jj)%rs_n_global(2), fft_array(jj)%rs_n_global(1), &
731 merge(cufft_c2r, cufft_z2d, this%single), accel%cuda_stream)
732 else
733 call cuda_fft_plan_many(fft_array(jj)%cuda_plan_fw, this%howmany, &
734 fft_array(jj)%rs_n_global(3), fft_array(jj)%rs_n_global(2), fft_array(jj)%rs_n_global(1), &
735 merge(cufft_c2c, cufft_z2z, this%single), accel%cuda_stream)
736 call cuda_fft_plan_many(fft_array(jj)%cuda_plan_bw, this%howmany, &
737 fft_array(jj)%rs_n_global(3), fft_array(jj)%rs_n_global(2), fft_array(jj)%rs_n_global(1), &
738 merge(cufft_c2c, cufft_z2z, this%single), accel%cuda_stream)
739 end if
740#endif
741
742 case default
743 call messages_write('Invalid FFT library.')
744 call messages_fatal()
745 end select
746
747 this = fft_array(jj)
748
749 ! Write information
750 if (.not. (library_ == fftlib_nfft .or. library_ == fftlib_pnfft)) then
751 call messages_write('Info: FFT grid dimensions =')
752 number_points = 1
753 do idir = 1, dim
754 call messages_write(fft_array(jj)%rs_n_global(idir))
755 if (idir < dim) call messages_write(" x ")
756 ! do the multiplication in a integer(int64) to avoid overflow for large grids
757 number_points = number_points * fft_array(jj)%rs_n_global(idir)
758 end do
759 call messages_new_line()
760
761 call messages_write(' Total grid size =')
762 call messages_write(number_points)
763 call messages_write(' (')
764 call messages_write(number_points*8.0_real64, units = unit_megabytes, fmt = '(f9.1)')
765 call messages_write(' )')
766 if (any(nn(1:fft_dim) /= nn_temp(1:fft_dim))) then
767 call messages_new_line()
768 call messages_write(' Inefficient FFT grid. A better grid would be: ')
769 do idir = 1, fft_dim
770 call messages_write(nn_temp(idir))
771 end do
772 end if
773 call messages_info()
774 end if
775
776 select case (library_)
777 case (fftlib_pfft)
778 write(message(1),'(a)') "Info: FFT library = PFFT"
779 write(message(2),'(a)') "Info: PFFT processor grid"
780 write(message(3),'(a, i9)') " No. of processors = ", mpi_grp_%size
781 write(message(4),'(a, i9)') " No. of columns in the proc. grid = ", column_size
782 write(message(5),'(a, i9)') " No. of rows in the proc. grid = ", row_size
783 write(message(6),'(a, i9)') " The size of integer is = ", c_intptr_t
784 call messages_info(6)
785
786 case (fftlib_pnfft)
787#ifdef HAVE_PNFFT
788 call messages_write("Info: FFT library = PNFFT")
789 call messages_info()
791#endif
792 case (fftlib_nfft)
793#ifdef HAVE_NFFT
794 call messages_write("Info: FFT library = FINUFFT")
795 call messages_info()
796 call finufft_write_info(fft_array(jj)%finufft)
797#endif
798 end select
799
800 pop_sub(fft_init)
801 end subroutine fft_init
802
803 ! ---------------------------------------------------------
807 subroutine fft_init_stage1(this, namespace, XX, nn, dft_nodes)
808 type(fft_t), intent(inout) :: this
811 type(namespace_t), intent(in) :: namespace
812 real(real64), contiguous, intent(in) :: xx(:,:)
813 integer, contiguous, optional, intent(in) :: nn(:)
815 logical, optional, intent(in) :: dft_nodes
816
817 integer :: slot
818
819 push_sub(fft_init_stage1)
820
821 assert(size(xx,2) == 3)
822
823 slot = this%slot
824 select case (fft_array(slot)%library)
825 case (fftlib_fftw)
826 !Do nothing
827 case (fftlib_nfft)
828#ifdef HAVE_NFFT
829 assert(present(nn))
830 if (optional_default(dft_nodes, .false.)) then
831 ! The forward transform evaluates the modes on the nodes, so the nodes are the
832 ! Fourier space points. On the FFT frequencies they give the mode ordering of
833 ! pad_feq and, through their spacing, the 1/N of FFTW.
834 block
835 integer :: idir, ii
836 real(real64), allocatable :: freq(:,:)
837
838 safe_allocate(freq(1:maxval(nn(1:3)), 1:3))
839 freq = m_zero
840 do idir = 1, 3
841 do ii = 1, nn(idir)
842 freq(ii, idir) = real(pad_feq(ii, nn(idir), .true.), real64)
843 end do
844 end do
845 call finufft_precompute(fft_array(slot)%finufft, &
846 freq(1:nn(1),1), freq(1:nn(2),2), freq(1:nn(3),3), period = real(nn(1:3), real64))
847 safe_deallocate_a(freq)
848 end block
849 else
850 call finufft_precompute(fft_array(slot)%finufft, &
851 xx(1:nn(1),1), xx(1:nn(2),2), xx(1:nn(3),3))
852 end if
853#endif
854 case (fftlib_pfft)
855 !Do nothing
856 case (fftlib_accel)
857 !Do nothing
858 case (fftlib_pnfft)
859#ifdef HAVE_PNFFT
860 call pnfft_set_sp_nodes(fft_array(slot)%pnfft, namespace, xx)
861#endif
862 case default
863 call messages_write('Invalid FFT library.')
864 call messages_fatal()
865 end select
866
867
868
869 pop_sub(fft_init_stage1)
870 end subroutine fft_init_stage1
871 ! ---------------------------------------------------------
872 subroutine fft_end(this)
873 type(fft_t), intent(inout) :: this
874
875 integer :: ii
876
877 push_sub(fft_end)
878
879 ii = this%slot
880 if (fft_refs(ii) == fft_null) then
881 message(1) = "Trying to deallocate FFT that has not been allocated."
882 call messages_warning(1)
883 else
884 if (fft_refs(ii) > 1) then
885 fft_refs(ii) = fft_refs(ii) - 1
886 else
887 select case (fft_array(ii)%library)
888 case (fftlib_fftw)
889 if (fft_array(ii)%single) then
890 call fftwf_destroy_plan(fft_array(ii)%planf)
891 call fftwf_destroy_plan(fft_array(ii)%planb)
892 call fftw_free_memory_single(this%type == fft_real, &
893 fft_array(ii)%srs_data, fft_array(ii)%crs_data, fft_array(ii)%cfs_data)
894 else
895 call fftw_destroy_plan(fft_array(ii)%planf)
896 call fftw_destroy_plan(fft_array(ii)%planb)
897
898 if (this%aligned_memory) then
899 call fftw_free_memory(this%type == fft_real, &
900 fft_array(ii)%drs_data, fft_array(ii)%zrs_data, fft_array(ii)%fs_data)
901 end if
902 end if
903
904 case (fftlib_pfft)
905#ifdef HAVE_PFFT
906 call pfft_destroy_plan(fft_array(ii)%planf)
907 call pfft_destroy_plan(fft_array(ii)%planb)
908#endif
909 safe_deallocate_p(fft_array(ii)%drs_data)
910 safe_deallocate_p(fft_array(ii)%zrs_data)
911 safe_deallocate_p(fft_array(ii)%fs_data)
912
913 case (fftlib_accel)
914#ifdef HAVE_CUDA
915 call cuda_fft_destroy(fft_array(ii)%cuda_plan_fw)
916 call cuda_fft_destroy(fft_array(ii)%cuda_plan_bw)
917#endif
918
919 case (fftlib_nfft)
920#ifdef HAVE_NFFT
921 call finufft_end(fft_array(ii)%finufft)
922#endif
923 case (fftlib_pnfft)
924#ifdef HAVE_PNFFT
925 call pnfft_end(fft_array(ii)%pnfft)
926#endif
927 end select
928 fft_refs(ii) = fft_null
929 end if
930 end if
931 this%slot = 0
932
933 pop_sub(fft_end)
934 end subroutine fft_end
935
936 ! ---------------------------------------------------------
937 subroutine fft_copy(fft_i, fft_o)
938 type(fft_t), intent(in) :: fft_i
939 type(fft_t), intent(inout) :: fft_o
940
941 push_sub(fft_copy)
942
943 if (fft_o%slot > 0) then
944 call fft_end(fft_o)
945 end if
946 assert(fft_i%slot >= 1.and.fft_i%slot <= fft_max)
947 assert(fft_refs(fft_i%slot) > 0)
948
949 fft_o = fft_i
950 fft_refs(fft_i%slot) = fft_refs(fft_i%slot) + 1
952 pop_sub(fft_copy)
953 end subroutine fft_copy
954
955 ! ---------------------------------------------------------
957 function fft_pack(dims3, howmany) result(dims4)
958 integer, intent(in) :: dims3(3)
959 integer, intent(in) :: howmany
960 integer :: dims4(4)
961
962 dims4(1:3) = dims3(1:3)
963 dims4(4) = howmany
964 end function fft_pack
965
966 ! ---------------------------------------------------------
967 subroutine fft_get_dims(fft, rs_n_global, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
968 type(fft_t), intent(in) :: fft
969 integer, intent(out) :: rs_n_global(1:3)
970 integer, intent(out) :: fs_n_global(1:3)
971 integer, intent(out) :: rs_n(1:3)
972 integer, intent(out) :: fs_n(1:3)
973 integer, intent(out) :: rs_istart(1:3)
974 integer, intent(out) :: fs_istart(1:3)
975
976 integer :: slot
977
978 push_sub(fft_get_dims)
979
980 slot = fft%slot
981 rs_n_global(1:3) = fft_array(slot)%rs_n_global(1:3)
982 fs_n_global(1:3) = fft_array(slot)%fs_n_global(1:3)
983 rs_n(1:3) = fft_array(slot)%rs_n(1:3)
984 fs_n(1:3) = fft_array(slot)%fs_n(1:3)
985 rs_istart(1:3) = fft_array(slot)%rs_istart(1:3)
986 fs_istart(1:3) = fft_array(slot)%fs_istart(1:3)
987
988 pop_sub(fft_get_dims)
989 end subroutine fft_get_dims
990
991 ! ---------------------------------------------------------
993 pure function pad_feq(ii, nn, mode)
994 integer, intent(in) :: ii,nn
995 logical, intent(in) :: mode
996 integer :: pad_feq
997
998 ! no push_sub: called too frequently
999
1000 if (mode) then ! index to frequency number
1001 if (ii <= nn/2 + 1) then
1002 pad_feq = ii - 1
1003 else
1004 pad_feq = ii - nn - 1
1005 end if
1006 else ! frequency number to index
1007 if (ii >= 0) then
1008 pad_feq = ii + 1
1009 else
1010 pad_feq = ii + nn + 1
1011 end if
1012 end if
1013
1014 end function pad_feq
1015
1016 ! -------------------------------------------------------
1017
1018 integer function fft_size(size, factors, parity)
1019 integer, intent(in) :: size
1020 integer, intent(in) :: factors(:)
1021 integer, intent(in) :: parity
1022
1023 integer :: nfactors
1024 integer :: nondiv
1025 integer, allocatable :: exponents(:)
1026
1027 push_sub(fft_size)
1028
1029 nfactors = ubound(factors, dim = 1)
1030
1031 safe_allocate(exponents(1:nfactors))
1033 fft_size = size
1034 do
1035 call get_exponents(fft_size, nfactors, factors, exponents, nondiv)
1036 if (nondiv == 1 .and. mod(fft_size, 2) == parity) exit
1037 fft_size = fft_size + 1
1038 end do
1039
1040 safe_deallocate_a(exponents)
1041
1042 pop_sub(fft_size)
1043 end function fft_size
1044
1045 ! -------------------------------------------------------
1046
1047 subroutine get_exponents(num, nfactors, factors, exponents, nondiv)
1048 integer, intent(in) :: num
1049 integer, intent(in) :: nfactors
1050 integer, intent(in) :: factors(:)
1051 integer, contiguous, intent(out) :: exponents(:)
1052 integer, intent(out) :: nondiv
1053
1054 integer :: ifactor
1055
1056 push_sub(get_exponents)
1057
1058 nondiv = num
1059 do ifactor = 1, nfactors
1060 exponents(ifactor) = 0
1061 do
1062 if (mod(nondiv, factors(ifactor)) /= 0) exit
1063 nondiv = nondiv/factors(ifactor)
1064 exponents(ifactor) = exponents(ifactor) + 1
1065 end do
1066 end do
1067
1068 pop_sub(get_exponents)
1069 end subroutine get_exponents
1071
1072 ! ----------------------------------------------------------
1073
1074 subroutine fft_operation_count(fft)
1075 type(fft_t), intent(in) :: fft
1076
1077 real(real64) :: fullsize
1078
1079 push_sub(fft_operation_count)
1080
1081 fullsize = real(fft%howmany, real64)*product(real(fft%fs_n(1:3), real64))
1082 call profiling_count_operations(5.0_real64*fullsize*log(fullsize)/log(m_two))
1083
1084 pop_sub(fft_operation_count)
1085 end subroutine fft_operation_count
1086
1089 pure subroutine fft_gg_transform(gg_in, temp, periodic_dim, latt, qq, gg, modg2)
1090 integer, contiguous, intent(in) :: gg_in(:)
1091 real(real64), contiguous, intent(in) :: temp(:)
1092 integer, intent(in) :: periodic_dim
1093 type(lattice_vectors_t), intent(in) :: latt
1094 real(real64), contiguous, intent(in) :: qq(:)
1095 real(real64), contiguous, intent(out) :: gg(:)
1096 real(real64), intent(out) :: modg2
1097
1098 ! no PUSH_SUB, called too frequently
1099
1100 gg(1:3) = real(gg_in(1:3), real64)
1101 gg(1:periodic_dim) = gg(1:periodic_dim) + qq(1:periodic_dim)
1102 gg(1:3) = gg(1:3) * temp(1:3)
1103 gg(1:3) = matmul(latt%klattice_primitive(1:3,1:3),gg(1:3))
1104 modg2 = sum(gg(1:3)**2)
1105
1106 end subroutine fft_gg_transform
1107
1108 ! ----------------------------------------------------------
1109
1112 real(real64) pure function fft_scaling_factor(fft) result(scaling_factor)
1113 type(fft_t), intent(in) :: fft
1114
1115 ! for the moment this factor is handled by the backwards transform for most libraries
1116 scaling_factor = m_one
1117
1118 select case (fft_array(fft%slot)%library)
1119 case (fftlib_accel)
1120#ifdef HAVE_CUDA
1121 scaling_factor = m_one/real(fft_array(fft%slot)%rs_n_global(1), real64)
1122 scaling_factor = scaling_factor/real(fft_array(fft%slot)%rs_n_global(2), real64)
1123 scaling_factor = scaling_factor/real(fft_array(fft%slot)%rs_n_global(3), real64)
1124#endif
1125 end select
1126
1127 end function fft_scaling_factor
1128
1129 ! ----------------------------------------------------------
1132 !
1133 ! Inspired by the routine bounds from Abinit
1134 real(real64) function fft_get_ecut_from_box(box_dim, fs_istart, latt, gspacing, periodic_dim, qq) result(ecut)
1135 integer, contiguous, intent(in) :: box_dim(:)
1136 integer, contiguous, intent(in) :: fs_istart(:)
1137 type(lattice_vectors_t), intent(in) :: latt
1138 real(real64), contiguous, intent(in) :: gspacing(:)
1139 integer, intent(in) :: periodic_dim
1140 real(real64), contiguous, intent(in) :: qq(:)
1141
1142 integer :: lx, ix, iy, iz, idir, idir2, idir3
1143 real(real64) :: dminsq, gg(3), modg2
1144 integer :: box_dim_(3), ixx(3)
1145 integer :: ming(3), maxg(3)
1146
1147 ! no PUSH_SUB, called too frequently
1148
1149 assert(periodic_dim > 0)
1150
1151 box_dim_(1:periodic_dim) = box_dim(1:periodic_dim)
1152 if (periodic_dim < 3) box_dim_(periodic_dim+1:3) = 1
1153
1154 ! We first need to remove asymmetric planes for the case of even FFT grids
1155 ming = 1
1156 maxg = 1
1157 do idir = 1, periodic_dim
1158 do lx = 1, box_dim(idir)
1159 ix = fs_istart(idir) + lx - 1
1160 ixx(idir) = pad_feq(ix, box_dim(idir), .true.)
1161 ming(idir) = min(ming(idir), ixx(idir))
1162 maxg(idir) = max(maxg(idir), ixx(idir))
1163 end do
1164 maxg(idir) = min(abs(ming(idir)), maxg(idir))
1165 end do
1166
1167 ! Given the boundaries, we can search the min distance, which gives us the the cutoff energy
1168 dminsq = m_huge
1169 do idir = 1, periodic_dim
1170 idir2 = mod(idir, 3)+1
1171 idir3 = mod(idir+1, 3)+1
1172
1173 ! Negative plane
1174 ixx(idir) = -maxg(idir)
1175 do iy = -maxg(idir2), maxg(idir2)
1176 ixx(idir2) = iy
1177 do iz = -maxg(idir3), maxg(idir3)
1178 ixx(idir3) = iz
1179 call fft_gg_transform(ixx, gspacing, periodic_dim, latt, qq, gg, modg2)
1180 dminsq = min(dminsq, sum(gg(1:periodic_dim)**2))
1181 end do
1182 end do
1183 ! Positive plane
1184 ixx(idir) = maxg(idir)
1185 do iy = -maxg(idir2), maxg(idir2)
1186 ixx(idir2) = iy
1187 do iz = -maxg(idir3), maxg(idir3)
1188 ixx(idir3) = iz
1189 call fft_gg_transform(ixx, gspacing, periodic_dim, latt, qq, gg, modg2)
1190 dminsq = min(dminsq, sum(gg(1:periodic_dim)**2))
1191 end do
1192 end do
1193 end do
1194
1195 ecut = m_half * dminsq
1196
1197 end function fft_get_ecut_from_box
1198
1199#include "undef.F90"
1200#include "real.F90"
1201#include "fft_inc.F90"
1202
1203#include "undef.F90"
1204#include "complex.F90"
1205#include "fft_inc.F90"
1206
1207#include "undef.F90"
1208#include "real32.F90"
1209#include "fft_inc.F90"
1210
1211#include "undef.F90"
1212#include "complex32.F90"
1213#include "fft_inc.F90"
1214
1215end module fft_oct_m
1216
1217!! Local Variables:
1218!! mode: f90
1219!! coding: utf-8
1220!! End:
subroutine optimize()
if write to the Free Software Franklin Fifth USA !The assertions are ignored if the code is compiled in not debug mode(NDEBUG ! is defined). Otherwise it is merely a logical assertion that
double log(double __x) __attribute__((__nothrow__
pure logical function, public accel_is_enabled()
Definition: accel.F90:391
type(accel_t), public accel
Definition: accel.F90:225
Fast Fourier Transform module. This module provides a single interface that works with different FFT ...
Definition: fft.F90:97
subroutine sfft_forward_single_3d(fft, in, out, norm)
Definition: fft.F90:2044
subroutine zfft_forward_accel(fft, in, out)
Definition: fft.F90:1750
subroutine dfft_backward_1d(fft, in, out)
Definition: fft.F90:1537
integer, parameter cufft_z2d
Definition: fft.F90:267
subroutine get_exponents(num, nfactors, factors, exponents, nondiv)
Definition: fft.F90:1006
subroutine, public fft_init_stage1(this, namespace, XX, nn, dft_nodes)
Some fft-libraries (only NFFT for the moment) need an additional precomputation stage that depends on...
Definition: fft.F90:791
logical fft_optimize
Definition: fft.F90:259
subroutine, public fft_all_init(namespace)
initialize the table
Definition: fft.F90:280
real(real64) function, public fft_get_ecut_from_box(box_dim, fs_istart, latt, gspacing, periodic_dim, qq)
Given an fft box (fixed by the real-space grid), it returns the cutoff energy of the sphere that fits...
Definition: fft.F90:1093
subroutine sfft_backward_accel(fft, in, out)
Definition: fft.F90:2254
subroutine zfft_forward_many_3d(fft, in, out, norm)
Definition: fft.F90:1667
subroutine zfft_backward_single_3d(fft, in, out, norm)
Definition: fft.F90:1791
subroutine dfft_forward_accel(fft, in, out)
Definition: fft.F90:1352
subroutine cfft_forward_single_3d(fft, in, out, norm)
Definition: fft.F90:2359
subroutine dfft_forward_single_3d(fft, in, out, norm)
Definition: fft.F90:1232
subroutine sfft_forward_many_3d(fft, in, out, norm)
Definition: fft.F90:2069
subroutine, public fft_end(this)
Definition: fft.F90:831
subroutine cfft_backward_accel(fft, in, out)
Definition: fft.F90:2569
real(real64) pure function, public fft_scaling_factor(fft)
This function returns the factor required to normalize a function after a forward and backward transf...
Definition: fft.F90:1071
subroutine, public fft_init(this, nn, dim, type, library, optimize, optimize_parity, comm, mpi_grp, use_aligned, howmany, nthreads, single)
Definition: fft.F90:429
integer, public fft_default_lib
Definition: fft.F90:261
integer, parameter cufft_z2z
Definition: fft.F90:267
pure integer function, public pad_feq(ii, nn, mode)
convert between array index and G-vector
Definition: fft.F90:952
subroutine dfft_backward_single_3d(fft, in, out, norm)
Definition: fft.F90:1393
subroutine zfft_backward_1d(fft, in, out)
Definition: fft.F90:1935
subroutine cfft_backward_single_3d(fft, in, out, norm)
Definition: fft.F90:2470
subroutine cfft_forward_many_3d(fft, in, out, norm)
Definition: fft.F90:2384
integer, parameter, public fftlib_accel
Definition: fft.F90:160
integer function, dimension(4) fft_pack(dims3, howmany)
Rank-4 data-array shape (dims3, howmany) with the batch as the trailing (batch-last) axis.
Definition: fft.F90:916
subroutine, public fft_all_end()
delete all plans
Definition: fft.F90:406
logical, save, public fft_initialized
Definition: fft.F90:256
integer function fft_size(size, factors, parity)
Definition: fft.F90:977
subroutine fft_operation_count(fft)
Definition: fft.F90:1033
subroutine sfft_forward_accel(fft, in, out)
Definition: fft.F90:2134
subroutine zfft_backward_accel(fft, in, out)
Definition: fft.F90:1914
integer, parameter cufft_c2r
Definition: fft.F90:267
subroutine sfft_backward_many_3d(fft, in, out, norm)
Definition: fft.F90:2180
integer, parameter cufft_c2c
Definition: fft.F90:267
integer, parameter, public fft_real
Definition: fft.F90:155
subroutine, public fft_get_dims(fft, rs_n_global, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
Definition: fft.F90:926
integer, parameter, public fft_complex
Definition: fft.F90:155
integer, save fft_prepare_plan
Definition: fft.F90:260
integer, parameter, public fftlib_nfft
Definition: fft.F90:160
subroutine, public fft_copy(fft_i, fft_o)
Definition: fft.F90:896
subroutine dfft_backward_many_3d(fft, in, out, norm)
Definition: fft.F90:1418
type(pnfft_t), save pnfft_options
Definition: fft.F90:265
subroutine cfft_backward_many_3d(fft, in, out, norm)
Definition: fft.F90:2495
subroutine zfft_forward_single_3d(fft, in, out, norm)
Definition: fft.F90:1642
integer, dimension(fft_max), save fft_refs
Definition: fft.F90:257
subroutine zfft_backward_many_3d(fft, in, out, norm)
Definition: fft.F90:1816
subroutine dfft_forward_1d(fft, in, out)
Definition: fft.F90:1374
subroutine cfft_forward_accel(fft, in, out)
Definition: fft.F90:2449
integer, parameter cufft_d2z
Definition: fft.F90:267
type(fft_t), dimension(fft_max), save fft_array
Definition: fft.F90:258
integer, parameter fft_null
Definition: fft.F90:168
integer, parameter, public fftlib_pnfft
Definition: fft.F90:160
subroutine sfft_backward_single_3d(fft, in, out, norm)
Definition: fft.F90:2155
subroutine dfft_forward_many_3d(fft, in, out, norm)
Definition: fft.F90:1257
integer, parameter cufft_r2c
Definition: fft.F90:267
subroutine zfft_forward_1d(fft, in, out)
Definition: fft.F90:1772
pure subroutine, public fft_gg_transform(gg_in, temp, periodic_dim, latt, qq, gg, modg2)
Convert FFT grid index into the Cartesian reciprocal-space vector .
Definition: fft.F90:1048
integer, parameter, public fftlib_pfft
Definition: fft.F90:160
subroutine dfft_backward_accel(fft, in, out)
Definition: fft.F90:1516
integer, parameter, public fftlib_fftw
Definition: fft.F90:160
subroutine, public fftw_free_memory(is_real, drs_data, zrs_data, fs_data)
Definition: fftw.F90:371
subroutine, public fftw_get_dims(rs_n, is_real, fs_n)
Definition: fftw.F90:329
subroutine, public fftw_prepare_plan_single(plan, dim, n, howmany, is_real, sign, flags, sin_, cin_, cout_)
Single-precision counterpart of fftw_prepare_plan. Only the aligned-memory path is supported (the sin...
Definition: fftw.F90:402
subroutine, public fftw_alloc_memory_single(rs_dims, is_real, fs_dims, srs_data, crs_data, cfs_data)
Definition: fftw.F90:480
subroutine, public fftw_free_memory_single(is_real, srs_data, crs_data, cfs_data)
Definition: fftw.F90:507
subroutine, public fftw_prepare_plan(plan, dim, n, howmany, is_real, sign, flags_, din_, cin_, cout_)
Definition: fftw.F90:177
subroutine, public fftw_alloc_memory(rs_dims, is_real, fs_dims, drs_data, zrs_data, fs_data)
Allocate the FFTW work buffers from the rank-4 (batch-last) dimension arrays (see fft_pack).
Definition: fftw.F90:344
Non-equispaced FFTs through the C interface of the FINUFFT library.
Definition: finufft.F90:95
subroutine, public finufft_write_info(finufft)
Definition: finufft.F90:446
subroutine, public finufft_precompute(finufft, X1, X2, X3, period)
Hand the spatial nodes to FINUFFT.
Definition: finufft.F90:218
subroutine, public finufft_read_options(finufft, namespace)
Definition: finufft.F90:148
subroutine, public finufft_init(finufft, finufft_options, N, dim, M, nthreads)
Definition: finufft.F90:186
subroutine, public finufft_end(finufft)
Definition: finufft.F90:480
real(real64), parameter, public m_two
Definition: global.F90:175
real(real64), parameter, public m_huge
Definition: global.F90:191
real(real64), parameter, public m_zero
Definition: global.F90:173
real(real64), parameter, public m_half
Definition: global.F90:179
real(real64), parameter, public m_one
Definition: global.F90:174
subroutine, public messages_not_implemented(feature, namespace)
Definition: messages.F90:1048
subroutine, public messages_warning(no_lines, all_nodes, namespace)
Definition: messages.F90:501
subroutine, public messages_new_line()
Definition: messages.F90:1069
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
Definition: messages.F90:138
subroutine, public messages_fatal(no_lines, only_root_writes, namespace)
Definition: messages.F90:386
subroutine, public messages_input_error(namespace, var, details, row, column)
Definition: messages.F90:671
subroutine, public messages_info(no_lines, iunit, debug_only, stress, all_nodes, namespace)
Definition: messages.F90:570
type(mpi_comm), parameter, public mpi_comm_undefined
used to indicate a communicator has not been initialized
Definition: mpi.F90:115
type(mpi_grp_t), public mpi_world
Definition: mpi.F90:317
The low level module to work with the PFFT library. http://www-user.tu-chemnitz.de/~mpip/software....
Definition: pfft.F90:105
subroutine, public pfft_prepare_plan_r2c(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
Definition: pfft.F90:170
subroutine, public pfft_prepare_plan_c2c(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
Definition: pfft.F90:238
subroutine, public pfft_decompose(n_proc, dim1, dim2)
Decompose all available processors in 2D processor grid, most equally possible.
Definition: pfft.F90:129
subroutine, public pfft_prepare_plan_c2r(plan, n, in, out, fft_sign, flags, mpi_comm)
Octopus subroutine to prepare a PFFT plan real to complex.
Definition: pfft.F90:204
subroutine, public pfft_get_dims(rs_n_global, mpi_comm, is_real, alloc_size, fs_n_global, rs_n, fs_n, rs_istart, fs_istart)
Definition: pfft.F90:265
The includes for the PFFT.
Definition: pfft.F90:94
The low level module to work with the PNFFT library. http://www-user.tu-chemnitz.de/~mpip/software....
Definition: pnfft.F90:107
subroutine, public pnfft_copy_params(in, out)
Definition: pnfft.F90:283
subroutine, public pnfft_set_sp_nodes(pnfft, namespace, X)
Definition: pnfft.F90:475
subroutine, public pnfft_init_plan(pnfft, pnfft_options, comm, fs_n_global, fs_n, fs_istart, rs_n, rs_istart)
Definition: pnfft.F90:341
subroutine, public pnfft_write_info(pnfft)
Definition: pnfft.F90:298
subroutine, public pnfft_guru_options(pnfft, namespace)
Definition: pnfft.F90:181
subroutine, public pnfft_end(pnfft)
Definition: pnfft.F90:451
subroutine, public pnfft_init_procmesh(pnfft, mpi_grp, comm)
Definition: pnfft.F90:247
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
Definition: profiling.F90:122
This module defines the unit system, used for input and output.
type(unit_t), public unit_megabytes
For large amounts of data (natural code units are bytes)
This is defined even when running serial.
Definition: mpi.F90:121
int true(void)