Octopus
nl_operator.F90
Go to the documentation of this file.
1!! Copyright (C) 2002-2006 M. Marques, A. Castro, A. Rubio, G. Bertsch
2!!
3!! This program is free software; you can redistribute it and/or modify
4!! it under the terms of the GNU General Public License as published by
5!! the Free Software Foundation; either version 2, or (at your option)
6!! any later version.
7!!
8!! This program is distributed in the hope that it will be useful,
9!! but WITHOUT ANY WARRANTY; without even the implied warranty of
10!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11!! GNU General Public License for more details.
12!!
13!! You should have received a copy of the GNU General Public License
14!! along with this program; if not, write to the Free Software
15!! Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16!! 02110-1301, USA.
17!!
18
19#include "global.h"
20
24 use accel_oct_m
25 use batch_oct_m
27 use debug_oct_m
28 use global_oct_m
30 use index_oct_m
31 use iso_c_binding
32 use math_oct_m
33 use mesh_oct_m
35 use mpi_oct_m
40 use parser_oct_m
42 use space_oct_m
43 use sort_oct_m
45 use types_oct_m
47
48 implicit none
49
50 private
51 public :: &
72
78 private
79 integer :: nri = 0
80 integer :: np = 0
81 integer, allocatable :: imin(:)
82 integer, allocatable :: imax(:)
83 integer, allocatable :: ri(:, :)
84 integer, allocatable :: ri_pos(:,:,:)
85 integer, allocatable :: ri_neg(:,:,:)
86 end type nl_operator_index_t
87
88 integer, public, parameter :: &
89 OP_GENERAL = 1, &
90 op_symmetric = 2, &
92
94 type nl_operator_t
95 private
96 type(stencil_t), public :: stencil
97 type(mesh_t), pointer :: mesh => null()
98 integer, allocatable :: nn(:)
99 integer, public :: np = 0
100 ! When running in parallel mode, the next three arrays are unique on each node.
101 real(real64), allocatable, public :: w(:,:)
102
103 logical, public :: const_w = .true.
104
105 type(accel_mem_t), public :: buff_weights
106 type(accel_mem_t), public :: buff_half_weights
107
108 integer, public :: symmetry = op_general
109
110 character(len=40) :: label
111
113 integer, public :: nri = 0
114 integer, allocatable, public :: ri(:,:)
115 integer, allocatable, public :: rimap(:)
116 integer, allocatable, public :: rimap_inv(:)
117
119 integer :: npairs = 0
120 real(real64),allocatable :: wpair(:)
121 real(real64) :: wcenter
122 integer, allocatable :: ri_pos(:,:,:)
123 integer, allocatable :: ri_neg(:,:,:)
124 integer :: max_allocated_ri_pair = 0
125
126 integer :: ninner = 0
127 integer :: nouter = 0
128
129 type(nl_operator_index_t) :: inner
130 type(nl_operator_index_t) :: outer
131
132 type(accel_kernel_t) :: kernel
133 type(accel_mem_t) :: buff_imin
134 type(accel_mem_t) :: buff_imax
135 type(accel_mem_t) :: buff_ri
136 type(accel_mem_t) :: buff_map
137 type(accel_mem_t) :: buff_all
138 type(accel_mem_t) :: buff_inner
139 type(accel_mem_t) :: buff_outer
140
142 type(accel_mem_t), public :: buff_wpair
143 type(accel_mem_t), public :: buff_half_wpair
144 type(accel_mem_t), public :: buff_ri_pos
145 type(accel_mem_t), public :: buff_ri_neg
146 type(accel_mem_t), public :: buff_map_sym
147 integer :: max_allocated_ri_pair_gpu = 0
148
149 ! For multigrid solvers
150 type(nl_operator_t), public, pointer :: coarser => null()
154 integer, parameter :: &
156 op_vec = 1, &
158 op_max = op_vec
159
160 integer, parameter :: &
161 op_invmap = 1, &
162 op_map = 2
163
164 integer, public, parameter :: op_all = 3, op_inner = 1, op_outer = 2
165
166 interface
167 integer function op_is_available(opid, type)
168 implicit none
169 integer, intent(in) :: opid, type
170 end function op_is_available
171 end interface
172
173 integer :: dfunction_global = -1
174 integer :: zfunction_global = -1
175 integer :: function_accel
176 logical :: use_symmetries = .false.
178contains
179
180 ! ---------------------------------------------------------
183 subroutine nl_operator_global_init(namespace)
184 type(namespace_t), intent(in) :: namespace
186 integer :: default
189
190 !%Variable OperateDouble
191 !%Type integer
192 !%Section Execution::Optimization
193 !%Default optimized
194 !%Description
195 !% This variable selects the subroutine used to apply non-local
196 !% operators over the grid for real functions.
197 !%Option fortran 0
198 !% The standard Fortran function.
199 !%Option optimized 1
200 !% This version is optimized using vector primitives (if available).
201 !%End
203 !%Variable OperateComplex
204 !%Type integer
205 !%Section Execution::Optimization
206 !%Default optimized
207 !%Description
208 !% This variable selects the subroutine used to apply non-local
209 !% operators over the grid for complex functions.
210 !%Option fortran 0
211 !% The standard Fortran function.
212 !%Option optimized 1
213 !% This version is optimized using vector primitives (if available).
214 !%End
216 default = op_vec
218 call parse_variable(namespace, 'OperateDouble', default, dfunction_global)
219 if (.not. varinfo_valid_option('OperateDouble', dfunction_global)) call messages_input_error(namespace, 'OperateDouble')
220
221 call parse_variable(namespace, 'OperateComplex', default, zfunction_global)
222 if (.not. varinfo_valid_option('OperateComplex', zfunction_global)) call messages_input_error(namespace, 'OperateComplex')
223
224 !%Variable OperateUseSymmetries
225 !%Type logical
226 !%Section Execution::Optimization
227 !%Default yes
228 !%Description
229 !% This variable selects if the operators are built using symmetries or not.
230 !% Ignored for curvilinear meshes.
231 !%End
232 call parse_variable(namespace, 'OperateUseSymmetries', .true., use_symmetries)
233
234 if (accel_is_enabled()) then
235
236 !%Variable OperateAccel
237 !%Type integer
238 !%Default map
239 !%Section Execution::Optimization
240 !%Description
241 !% This variable selects the subroutine used to apply non-local
242 !% operators over the grid when an accelerator device is used.
243 !%Option invmap 1
244 !% The standard implementation ported to GPUs.
245 !%Option map 2
246 !% A different version, more suitable for GPUs.
247 !%End
248 call parse_variable(namespace, 'OperateAccel', op_map, function_accel)
249
250 call messages_obsolete_variable(namespace, 'OperateOpenCL', 'OperateAccel')
251
252 end if
253
256
257 ! ---------------------------------------------------------
258
259 subroutine nl_operator_global_end()
260 push_sub(nl_operator_global_end)
261
263 end subroutine nl_operator_global_end
264
265 ! ---------------------------------------------------------
269 subroutine nl_operator_init(op, label, symm)
270 type(nl_operator_t), intent(inout) :: op
271 character(len=*), intent(in) :: label
272 integer, optional, intent(in) :: symm
273
274 push_sub(nl_operator_init)
275
276 op%label = label
277 op%symmetry = op_general
278 if (use_symmetries) then
279 op%symmetry = optional_default(symm, op_general)
280 end if
281
282 pop_sub(nl_operator_init)
283 end subroutine nl_operator_init
284
285 ! ---------------------------------------------------------
295 subroutine nl_operator_build(space, mesh, op, np, const_w, regenerate)
296 class(space_t), intent(in) :: space
297 type(mesh_t), target, intent(in) :: mesh
298 type(nl_operator_t), intent(inout) :: op
299 integer, intent(in) :: np
300 logical, optional, intent(in) :: const_w
301 logical, optional, intent(in) :: regenerate
302
303 integer :: ii, jj, p1(space%dim), time, current
304 integer, allocatable :: st1(:), st2(:), st1r(:)
305 integer :: ir, maxp, iinner, iouter
306 logical :: change, force_change
307 character(len=200) :: flags
308 character(len=32) :: indirect_arg
309 integer, allocatable :: inner_points(:), outer_points(:), all_points(:)
310
311 push_sub(nl_operator_build)
312
313 op%const_w = optional_default(const_w, .false.)
314 if (mesh%parallel_in_domains .and. .not. op%const_w) then
315 call messages_experimental('Domain parallelization with curvilinear coordinates')
316 end if
317
318 assert(np > 0)
319
320 ! store values in structure
321 op%np = np
322 op%mesh => mesh
323 if (.not. optional_default(regenerate, .false.)) then
324 op%const_w = optional_default(const_w, .false.)
325 end if
326
327 ! grouping points by symmetric pairs requires constant weights
328 if (.not. op%const_w) op%symmetry = op_general
329
330 ! allocate weights op%w
331 if (op%const_w) then
332 safe_allocate(op%w(1:op%stencil%size, 1))
333 message(1) = 'Debug: nl_operator_build: working with constant weights.'
334 call messages_info(1, debug_only=.true.)
335 else
336 safe_allocate(op%w(1:op%stencil%size, 1:op%np))
337 message(1) = 'Debug: nl_operator_build: working with non-constant weights.'
338 call messages_info(1, debug_only=.true.)
339 end if
340
341 ! set initially to zero
342 op%w = m_zero
343
344 ! Build lookup table
345 safe_allocate(st1(1:op%stencil%size))
346 safe_allocate(st1r(1:op%stencil%size))
347 safe_allocate(st2(1:op%stencil%size))
348
349 op%nri = 0
350 do time = 1, 2
351 st2 = 0
352 do ii = 1, np
353 p1 = 0
354 call mesh_local_index_to_coords(mesh, ii, p1)
355
356 do jj = 1, op%stencil%size
357 ! Get local index of p1 plus current stencil point.
358 st1(jj) = mesh_local_index_from_coords(mesh, p1 + op%stencil%points(:, jj))
359
360 assert(st1(jj) > 0)
361 end do
362
363 st1(1:op%stencil%size) = st1(1:op%stencil%size) - ii
364
365 change = any(st1 /= st2)
366
367 !the next is to detect when we move from a point that does not
368 !have boundary points as neighbours to one that has
369 force_change = any(st1 + ii > mesh%np) .and. all(st2 + ii - 1 <= mesh%np)
370
371 ! if the stencil changes
372 if (change .or. force_change) then
373 !store it
374 st2(:) = st1(:)
375
376 !first time, just count
377 if (time == 1) op%nri = op%nri + 1
378
379 !second time, store
380 if (time == 2) then
381 current = current + 1
382 op%ri(1:op%stencil%size, current) = st1(1:op%stencil%size)
383 end if
384 end if
385
386 if (time == 2) op%rimap(ii) = current
387
388 end do
389
390 !after counting, allocate
391 if (time == 1) then
392 safe_deallocate_a(op%ri)
393 safe_deallocate_a(op%rimap)
394 safe_deallocate_a(op%rimap_inv)
395
396 safe_allocate(op%ri(1:op%stencil%size, 1:op%nri))
397 safe_allocate(op%rimap(1:op%np))
398 safe_allocate(op%rimap_inv(1:op%nri + 1))
399 op%ri = 0
400 op%rimap = 0
401 op%rimap_inv = 0
402 current = 0
403
404 ! the sizes
405 if (mesh%use_curvilinear) then
406 safe_allocate(op%nn(1:op%nri))
407 ! for the moment all the sizes are the same
408 op%nn = op%stencil%size
409 end if
410 end if
411
412 end do
413
414 !the inverse mapping
415 op%rimap_inv(1) = 0
416 do jj = 1, op%np
417 op%rimap_inv(op%rimap(jj) + 1) = jj
418 end do
419 op%rimap_inv(op%nri + 1) = op%np
420
421 safe_deallocate_a(st1)
422 safe_deallocate_a(st1r)
423 safe_deallocate_a(st2)
424
425 if (op%mesh%parallel_in_domains) then
426 !now build the arrays required to apply the nl_operator by parts
427
428 !count points
429 op%inner%nri = 0
430 op%outer%nri = 0
431 do ir = 1, op%nri
432 maxp = op%rimap_inv(ir + 1) + maxval(op%ri(1:op%stencil%size, ir))
433 if (maxp <= np) then
434 !inner point
435 op%inner%nri = op%inner%nri + 1
436 assert(op%inner%nri <= op%nri)
437 else
438 !outer point
439 op%outer%nri = op%outer%nri + 1
440 assert(op%outer%nri <= op%nri)
441 end if
442 end do
443
444 assert(op%inner%nri + op%outer%nri == op%nri)
445
446 if (optional_default(regenerate, .false.)) then
447 safe_deallocate_a(op%inner%imin)
448 safe_deallocate_a(op%inner%imax)
449 safe_deallocate_a(op%inner%ri)
450 safe_deallocate_a(op%outer%imin)
451 safe_deallocate_a(op%outer%imax)
452 safe_deallocate_a(op%outer%ri)
453 end if
454 safe_allocate(op%inner%imin(1:op%inner%nri + 1))
455 safe_allocate(op%inner%imax(1:op%inner%nri))
456 safe_allocate(op%inner%ri(1:op%stencil%size, 1:op%inner%nri))
457
458 safe_allocate(op%outer%imin(1:op%outer%nri + 1))
459 safe_allocate(op%outer%imax(1:op%outer%nri))
460 safe_allocate(op%outer%ri(1:op%stencil%size, 1:op%outer%nri))
461
462 !now populate the arrays
463 iinner = 0
464 iouter = 0
465 op%inner%np = 0
466 op%outer%np = 0
467 do ir = 1, op%nri
468 maxp = op%rimap_inv(ir + 1) + maxval(op%ri(1:op%stencil%size, ir))
469 if (maxp <= np) then
470 !inner point
471 iinner = iinner + 1
472 op%inner%imin(iinner) = op%rimap_inv(ir)
473 op%inner%imax(iinner) = op%rimap_inv(ir + 1)
474 op%inner%np = op%inner%np + op%rimap_inv(ir + 1) - op%rimap_inv(ir)
475 op%inner%ri(1:op%stencil%size, iinner) = op%ri(1:op%stencil%size, ir)
476 else
477 !outer point
478 iouter = iouter + 1
479 op%outer%imin(iouter) = op%rimap_inv(ir)
480 op%outer%imax(iouter) = op%rimap_inv(ir + 1)
481 op%outer%np = op%outer%np + op%rimap_inv(ir + 1) - op%rimap_inv(ir)
482 op%outer%ri(1:op%stencil%size, iouter) = op%ri(1:op%stencil%size, ir)
483 end if
484 end do
485
486 !verify that all points in the inner operator are actually inner
487 do ir = 1, op%inner%nri
488 do ii = op%inner%imin(ir) + 1, op%inner%imax(ir)
489 assert(all(ii + op%inner%ri(1:op%stencil%size, ir) <= mesh%np))
490 end do
491 end do
492
493 end if
494
495 if (accel_is_enabled() .and. op%const_w) then
496
497 write(flags, '(i5)') op%stencil%size
498 flags='-DSTENCIL_SIZE='//trim(adjustl(flags))
499
500 select case (function_accel)
501 case (op_invmap)
502 if (op%symmetry /= op_general) then
503 call messages_not_implemented("OperateUseSymmetries=yes with OperateAccel=invmap")
504 end if
505 call accel_kernel_build(op%kernel, 'operate.cu', 'operate<double>', flags)
506 case (op_map)
507 if (op%mesh%parallel_in_domains) then
508 indirect_arg = 'Indexing::Indirect'
509 else
510 indirect_arg = 'Indexing::Direct'
511 end if
512 select case (op%symmetry)
513 case (op_general)
514 call accel_kernel_build(op%kernel, 'operate.cu', &
515 'operate_map<double, '//trim(indirect_arg)//'>', flags)
516 case (op_symmetric)
517 call accel_kernel_build(op%kernel, 'operate.cu', &
518 'operate_map<double, '//trim(indirect_arg)//', Symmetry::Symmetric>', flags)
519 case (op_antisymmetric)
520 call accel_kernel_build(op%kernel, 'operate.cu', &
521 'operate_map<double, '//trim(indirect_arg)//', Symmetry::Antisymmetric>', flags)
522 end select
523 end select
524
525 ! conversion to i8 needed to avoid integer overflow
526 call accel_create_buffer(op%buff_ri, accel_mem_read_only, type_integer, int(op%nri, int64)*op%stencil%size)
527 call accel_write_buffer(op%buff_ri, op%stencil%size, op%nri, op%ri)
528
529 select case (function_accel)
530 case (op_invmap)
531 call accel_create_buffer(op%buff_imin, accel_mem_read_only, type_integer, op%nri)
532 call accel_write_buffer(op%buff_imin, op%nri, op%rimap_inv(1:))
533 call accel_create_buffer(op%buff_imax, accel_mem_read_only, type_integer, op%nri)
534 call accel_write_buffer(op%buff_imax, op%nri, op%rimap_inv(2:))
535
536 case (op_map)
537
539 call accel_write_buffer(op%buff_map, op%mesh%np, (op%rimap - 1)*op%stencil%size)
540
541 if (op%symmetry /= op_general) then
542 call accel_create_buffer(op%buff_map_sym, accel_mem_read_only, type_integer, pad(op%mesh%np, accel_max_block_size()))
543 call accel_write_buffer(op%buff_map_sym, op%mesh%np, (op%rimap - 1)*(op%stencil%size/2))
544 end if
545
546 if (op%mesh%parallel_in_domains) then
547
548 safe_allocate(inner_points(1:op%mesh%np))
549 safe_allocate(outer_points(1:op%mesh%np))
550 safe_allocate(all_points(1:op%mesh%np))
551
552 op%ninner = 0
553 op%nouter = 0
554
555 do ii = 1, op%mesh%np
556 all_points(ii) = ii - 1
557 maxp = ii + maxval(op%ri(1:op%stencil%size, op%rimap(ii)))
558 if (maxp <= op%mesh%np) then
559 op%ninner = op%ninner + 1
560 inner_points(op%ninner) = ii - 1
561 else
562 op%nouter = op%nouter + 1
563 outer_points(op%nouter) = ii - 1
564 end if
565 end do
566
568 call accel_write_buffer(op%buff_all, op%mesh%np, all_points)
569
571 call accel_write_buffer(op%buff_inner, op%ninner, inner_points)
572
574 call accel_write_buffer(op%buff_outer, op%nouter, outer_points)
575
576 safe_deallocate_a(inner_points)
577 safe_deallocate_a(outer_points)
578 safe_deallocate_a(all_points)
579
580 end if
581 end select
582 end if
583
584 pop_sub(nl_operator_build)
585
586 end subroutine nl_operator_build
587
588 ! ---------------------------------------------------------
589 subroutine nl_operator_output_weights(this)
590 type(nl_operator_t), intent(inout) :: this
591
592 integer :: istencil, idir
593
595
596 write(message(1), '(3a)') 'Debug info: Finite difference weights for ', trim(this%label), '.'
597 write(message(2), '(a)') ' Spacing:'
598 do idir = 1, this%mesh%box%dim
599 write(message(2), '(a,f16.8)') trim(message(2)), this%mesh%spacing(idir)
600 end do
601 call messages_info(2, debug_only=.true.)
602
603 do istencil = 1, this%stencil%size
604 select case(this%mesh%box%dim)
605 case(1)
606 write(message(1), '(a,i3,1i4,f25.10)') ' ', istencil, this%stencil%points(1:1, istencil), this%w(istencil, 1)
607 case(2)
608 write(message(1), '(a,i3,2i4,f25.10)') ' ', istencil, this%stencil%points(1:2, istencil), this%w(istencil, 1)
609 case(3)
610 write(message(1), '(a,i3,3i4,f25.10)') ' ', istencil, this%stencil%points(1:3, istencil), this%w(istencil, 1)
611 end select
612 call messages_info(1, debug_only=.true.)
613 end do
614
616
617 end subroutine nl_operator_output_weights
618
619 ! ---------------------------------------------------------
620 subroutine nl_operator_end(op)
621 type(nl_operator_t), intent(inout) :: op
622
623 push_sub(nl_operator_end)
624
625 if (accel_is_enabled() .and. op%const_w) then
627 end if
628
629 safe_deallocate_a(op%inner%imin)
630 safe_deallocate_a(op%inner%imax)
631 safe_deallocate_a(op%inner%ri)
632 safe_deallocate_a(op%outer%imin)
633 safe_deallocate_a(op%outer%imax)
634 safe_deallocate_a(op%outer%ri)
635
636 safe_deallocate_a(op%w)
637
638 safe_deallocate_a(op%ri)
639 safe_deallocate_a(op%rimap)
640 safe_deallocate_a(op%rimap_inv)
641 safe_deallocate_a(op%nn)
642
643 safe_deallocate_a(op%wpair)
644 safe_deallocate_a(op%ri_pos)
645 safe_deallocate_a(op%ri_neg)
646
647 safe_deallocate_a(op%inner%ri_pos)
648 safe_deallocate_a(op%inner%ri_neg)
649 safe_deallocate_a(op%outer%ri_pos)
650 safe_deallocate_a(op%outer%ri_neg)
651 call stencil_end(op%stencil)
652
653 pop_sub(nl_operator_end)
654 end subroutine nl_operator_end
655
656
657 subroutine nl_operator_clear_gpu_buffers(op)
658 type(nl_operator_t), intent(inout) :: op
659
662 call accel_free_buffer(op%buff_ri)
663 select case (function_accel)
664 case (op_invmap)
665 call accel_free_buffer(op%buff_imin)
666 call accel_free_buffer(op%buff_imax)
667
668 case (op_map)
669 call accel_free_buffer(op%buff_map)
670 if (op%symmetry /= op_general) then
671 call accel_free_buffer(op%buff_map_sym)
672 end if
673 if (op%mesh%parallel_in_domains) then
674 call accel_free_buffer(op%buff_all)
675 call accel_free_buffer(op%buff_inner)
676 call accel_free_buffer(op%buff_outer)
677 end if
678 end select
679
680 call accel_free_buffer(op%buff_weights)
681 call accel_free_buffer(op%buff_half_weights)
682
683 if (op%symmetry /= op_general) then
684 call accel_free_buffer(op%buff_wpair)
685 call accel_free_buffer(op%buff_half_wpair)
686 if (op%max_allocated_ri_pair_gpu > 0) then
687 call accel_free_buffer(op%buff_ri_pos)
688 call accel_free_buffer(op%buff_ri_neg)
689 op%max_allocated_ri_pair_gpu = 0
690 end if
691 end if
694 end subroutine nl_operator_clear_gpu_buffers
695
696 ! ---------------------------------------------------------
697 integer pure function nl_operator_get_index(op, is, ip) result(res)
698 type(nl_operator_t), intent(in) :: op
699 integer, intent(in) :: is
700 integer, intent(in) :: ip
701
702 res = ip + op%ri(is, op%rimap(ip))
703 end function nl_operator_get_index
704
705 ! ---------------------------------------------------------
706
708 type(nl_operator_t), intent(inout) :: op
709
711
712 ! Update the GPU weights
713 if (accel_is_enabled() .and. op%const_w) then
714 call accel_create_buffer(op%buff_weights, accel_mem_read_only, type_float, op%stencil%size)
715 call accel_create_buffer(op%buff_half_weights, accel_mem_read_only, type_float, op%stencil%size)
716 if (op%symmetry /= op_general) then
717 call accel_create_buffer(op%buff_wpair, accel_mem_read_only, type_float, op%stencil%size/2)
718 call accel_create_buffer(op%buff_half_wpair, accel_mem_read_only, type_float, op%stencil%size/2)
719 end if
720 end if
721
724
725
726 ! ---------------------------------------------------------
727
728 subroutine nl_operator_update_gpu_buffers(op)
729 type(nl_operator_t), intent(inout) :: op
730
731 integer(int64) :: buf_size
732
734
735 ! Update the GPU weights
736 if (accel_is_enabled() .and. op%const_w) then
737 call accel_write_buffer(op%buff_weights, op%stencil%size, op%w(:, 1))
738 call accel_write_buffer(op%buff_half_weights, op%stencil%size, -m_half*op%w(:, 1))
739
740 if (op%symmetry /= op_general) then
741 call accel_write_buffer(op%buff_wpair, op%npairs, op%wpair)
742 call accel_write_buffer(op%buff_half_wpair, op%npairs, -m_half*op%wpair)
743
744 ! (Re)allocate the pair-index buffers if max_allocated_ri_pair has grown
745 if (op%max_allocated_ri_pair > op%max_allocated_ri_pair_gpu) then
746 if (op%max_allocated_ri_pair_gpu > 0) then
747 call accel_free_buffer(op%buff_ri_pos)
748 call accel_free_buffer(op%buff_ri_neg)
749 end if
750 buf_size = int(op%npairs, int64)*op%nri*op%max_allocated_ri_pair
751 call accel_create_buffer(op%buff_ri_pos, accel_mem_read_only, type_integer, buf_size)
752 call accel_create_buffer(op%buff_ri_neg, accel_mem_read_only, type_integer, buf_size)
753 op%max_allocated_ri_pair_gpu = op%max_allocated_ri_pair
754 end if
755
756 if (op%max_allocated_ri_pair > 0) then
757 call accel_write_buffer(op%buff_ri_pos, op%npairs, op%nri, op%max_allocated_ri_pair, op%ri_pos)
758 call accel_write_buffer(op%buff_ri_neg, op%npairs, op%nri, op%max_allocated_ri_pair, op%ri_neg)
759 end if
760 end if
761 end if
762
764 end subroutine nl_operator_update_gpu_buffers
765
766 ! ---------------------------------------------------------
767
768 integer pure function nl_operator_np_zero_bc(op) result(np_bc)
769 type(nl_operator_t), intent(in) :: op
770
771 integer :: jj, ii
772
773 np_bc = 0
774 do jj = 1, op%nri
775 ii = op%rimap_inv(jj + 1) + maxval(op%ri(1:op%stencil%size, jj))
776 np_bc = max(np_bc, ii)
777 end do
778
780
781
782 ! ---------------------------------------------------------
784 subroutine nl_operator_remove_zero_weight_points(op, space, mesh)
785 type(nl_operator_t), intent(inout) :: op
786 type(space_t), intent(in) :: space
787 class(mesh_t), intent(in) :: mesh
788
789 integer :: ip, size
790 real(real64), parameter :: tol = 1.0e-14_real64
791 real(real64) :: max_weight, new_w(op%stencil%size)
792 integer :: new_points(space%dim, op%stencil%size)
793
794 if (.not. op%const_w) return
795
797
798 max_weight = maxval(abs(op%w(:, 1)))
799 size = 0
800 do ip = 1, op%stencil%size
801 ! For (anti)symmetric operators, points are kept or removed together with their mirror partner,
802 ! otherwise the pairing of symmetric points fails
803 if (max(abs(op%w(ip, 1)), abs(partner_weight(ip))) > tol * max_weight) then
804 size = size +1
805 new_w(size) = op%w(ip, 1)
806 new_points(:,size) = op%stencil%points(:, ip)
807 end if
808 end do
809
810 ! We regenerate the stencil without the zero-weight points
811 op%stencil%size = size
812 safe_deallocate_a(op%stencil%points)
813 safe_allocate(op%stencil%points(space%dim, op%stencil%size))
814 op%stencil%points(:, :) = new_points(:, 1:size)
815 safe_deallocate_a(op%w)
817 call nl_operator_build(space, mesh, op, mesh%np, const_w=op%const_w, regenerate=.true.)
818 op%w(1:size, 1) = new_w(1:size)
819
820 !Update Stencil%center
821 call stencil_init_center(op%stencil)
822
824
825 contains
826
828 real(real64) function partner_weight(ip) result(wp)
829 integer, intent(in) :: ip
830
831 integer :: jp
832
833 wp = op%w(ip, 1)
834 if (op%symmetry == op_general) return
835 do jp = 1, op%stencil%size
836 if (all(op%stencil%points(:, jp) + op%stencil%points(:, ip) == 0)) then
837 wp = op%w(jp, 1)
838 return
839 end if
840 end do
841 end function partner_weight
843
845 subroutine group_by_pairs_sym(size, ldf, offsets, wre, ri, nri, npairs, wpair, pair_pos, pair_neg, wcenter)
846 integer, intent(in) :: size
847 integer, intent(in) :: ldf
848 integer, contiguous, intent(in) :: offsets(:, :)
849 real(real64), contiguous, intent(in) :: wre(:)
850 integer, contiguous, intent(in) :: ri(:, :)
851 integer, intent(in) :: nri
852 integer, intent(out) :: npairs
853 real(real64), contiguous, intent(inout) :: wpair(:)
854 integer, contiguous, intent(inout) :: pair_pos(:,:), pair_neg(:,:)
855 real(real64), intent(out) :: wcenter
857 logical, allocatable :: used(:)
858 integer :: i, j, ndim, s
859 logical :: same
860 integer, allocatable :: idx(:)
861 real(real64) :: max_mismatch
862
863 real(real64), parameter :: tol = 1.0e-11_real64
864
865 push_sub(group_by_pairs_sym)
866
867 assert(mod(size,2) == 1)
868
869 safe_allocate(used(1:size))
870 used = .false.
871 npairs = 0
872 max_mismatch = m_zero
873
874 ndim = ubound(offsets, dim=1)
875
876 safe_allocate(idx(1:size))
877 call robust_sort_by_abs(wre, offsets, idx)
878
879 do i = 1, size
880 if (used(i)) cycle
881
882 if (all(offsets(:, idx(i))==0)) then
883 wcenter = wre(idx(i))
884 used(i) = .true.
885 cycle
886 end if
887
888 ! Try to find symmetric partner j. The partner is defined by the opposite offset only.
889 ! For large stencils (e.g. cube, high order) the numerically computed weights of the two
890 ! partners can differ by more than tol; they are then averaged, which symmetrizes the operator.
891 do j = i+1, size
892 if (used(j)) cycle
893
894 ! Offsets equal and opposite
895 if (any(offsets(:,idx(j))+offsets(:, idx(i)) /= 0)) cycle
896
897 ! Weight equality
898 same = abs(wre(idx(i)) - wre(idx(j))) <= tol*max(m_one, abs(wre(idx(i))))
899 if (.not. same) max_mismatch = max(max_mismatch, abs(wre(idx(i)) - wre(idx(j))))
901 npairs = npairs + 1
902 do s = 1, nri
903 pair_pos(npairs, s) = ri(idx(i), s) * 2**ldf
904 pair_neg(npairs, s) = ri(idx(j), s) * 2**ldf
905 end do
906 wpair(npairs) = m_half*(wre(idx(i)) + wre(idx(j)))
907
908 used(i) = .true.
909 used(j) = .true.
910 exit
911 end do
912 end do
913
914 assert(npairs == size/2)
915
916 if (max_mismatch > m_zero) then
917 write(message(1), '(a,es10.3)') 'Symmetric stencil weights differ between partner points by up to ', max_mismatch
918 message(2) = 'They have been averaged. The stencil weights may be ill-conditioned (try a lower DerivativesOrder).'
919 call messages_warning(2)
920 end if
921
922 safe_deallocate_a(idx)
923
924 pop_sub(group_by_pairs_sym)
925 end subroutine group_by_pairs_sym
926
928 subroutine group_by_pairs_antisym(size, ldf, offsets, wre, ri, nri, npairs, wpair, pair_pos, pair_neg)
929 integer, intent(in) :: size
930 integer, intent(in) :: ldf
931 integer, contiguous, intent(in) :: offsets(:, :)
932 real(real64), contiguous, intent(in) :: wre(:)
933 integer, contiguous, intent(in) :: ri(:, :)
934 integer, intent(in) :: nri
935 integer, intent(out) :: npairs
936 real(real64), contiguous, intent(inout) :: wpair(:)
937 integer, contiguous, intent(inout) :: pair_pos(:,:), pair_neg(:,:)
938
939 logical, allocatable :: used(:)
940 integer :: i, j, ndim, s
941 logical :: same
942 integer, allocatable :: idx(:)
943 real(real64) :: max_mismatch
944
945 real(real64), parameter :: tol = 1.0e-11_real64
946
947 push_sub(group_by_pairs_antisym)
948
949 assert(mod(size,2) == 0)
950
951 safe_allocate(used(1:size))
952 used = .false.
953 npairs = 0
954 max_mismatch = m_zero
955
956 ndim = ubound(offsets, dim=1)
957
958 safe_allocate(idx(1:size))
959 call robust_sort_by_abs(wre, offsets, idx)
960
961 ! Max pairs = n/2
962 do i = 1, size
963 if (used(i)) cycle
964
965 ! Try to find symmetric partner j: the partner is defined by the opposite offset only
966 ! (see group_by_pairs_sym); weights are antisymmetrized
967 do j = i+1, size
968 if (used(j)) cycle
969
970 ! Offsets equal and opposite
971 if (any(offsets(:,idx(j))+offsets(:, idx(i)) /= 0)) cycle
972
973 ! Weight equality
974 same = abs(wre(idx(i)) + wre(idx(j))) <= tol*max(m_one, abs(wre(idx(i))))
975 if (.not. same) max_mismatch = max(max_mismatch, abs(wre(idx(i)) + wre(idx(j))))
976
977 npairs = npairs + 1
978 do s = 1, nri
979 pair_pos(npairs, s) = ri(idx(i), s) * 2**ldf
980 pair_neg(npairs, s) = ri(idx(j), s) * 2**ldf
981 end do
982 wpair(npairs) = m_half*(wre(idx(i)) - wre(idx(j)))
983
984 used(i) = .true.
985 used(j) = .true.
986 exit
987 end do
988 end do
989
990 assert(npairs == size/2)
991
992 if (max_mismatch > m_zero) then
993 write(message(1), '(a,es10.3)') 'Antisymmetric stencil weights differ between partner points by up to ', max_mismatch
994 message(2) = 'They have been averaged. The stencil weights may be ill-conditioned (try a lower DerivativesOrder).'
995 call messages_warning(2)
996 end if
997
998 safe_deallocate_a(idx)
1001
1003 subroutine nl_operator_build_symmetric_weights(op, max_size)
1004 type(nl_operator_t), intent(inout) :: op
1005 integer, optional, intent(in) :: max_size
1006
1007 integer :: ldf, start, end, ipair
1008
1009 if (op%symmetry == op_general) return
1010
1012
1013 assert(op%const_w)
1014
1015 if(present(max_size)) then
1016 start = op%max_allocated_ri_pair + 1
1017 end = max_size
1018 call reallocate_array(op%ri_pos, op%stencil%size/2, op%nri, op%max_allocated_ri_pair, end)
1019 call reallocate_array(op%ri_neg, op%stencil%size/2, op%nri, op%max_allocated_ri_pair, end)
1020 if (op%mesh%parallel_in_domains) then
1021 call reallocate_array(op%inner%ri_pos, op%stencil%size/2, op%inner%nri, op%max_allocated_ri_pair, end)
1022 call reallocate_array(op%inner%ri_neg, op%stencil%size/2, op%inner%nri, op%max_allocated_ri_pair, end)
1023 call reallocate_array(op%outer%ri_pos, op%stencil%size/2, op%outer%nri, op%max_allocated_ri_pair, end)
1024 call reallocate_array(op%outer%ri_neg, op%stencil%size/2, op%outer%nri, op%max_allocated_ri_pair, end)
1025 end if
1026 else if (allocated(op%wpair)) then
1027 ! The weights op%w have been modified: recompute the pairs for all allocated levels
1028 start = 1
1029 end = op%max_allocated_ri_pair
1030 else
1031 ! Max pairs = n/2
1032 safe_allocate(op%wpair(1:op%stencil%size/2))
1033 start = 1
1034 ! conf%target_states_block_size is only set for electron systems
1035 if (conf%target_states_block_size > 0) then
1036 end = log2(conf%target_states_block_size)+1
1037 else
1038 end = 1
1039 end if
1040
1041 safe_allocate(op%ri_pos(1:op%stencil%size/2, 1:op%nri, 1:end))
1042 safe_allocate(op%ri_neg(1:op%stencil%size/2, 1:op%nri, 1:end))
1043 if (op%mesh%parallel_in_domains) then
1044 safe_allocate(op%inner%ri_pos(1:op%stencil%size/2, 1:op%inner%nri, 1:end))
1045 safe_allocate(op%inner%ri_neg(1:op%stencil%size/2, 1:op%inner%nri, 1:end))
1046 safe_allocate(op%outer%ri_pos(1:op%stencil%size/2, 1:op%outer%nri, 1:end))
1047 safe_allocate(op%outer%ri_neg(1:op%stencil%size/2, 1:op%outer%nri, 1:end))
1048 end if
1049 end if
1050 op%max_allocated_ri_pair = end
1051
1052 do ldf = start-1, end-1
1053 select case(op%symmetry)
1054 case(op_symmetric)
1055 call group_by_pairs_sym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%ri, op%nri, &
1056 op%npairs, op%wpair, op%ri_pos(:,:,ldf+1), op%ri_neg(:,:,ldf+1), op%wcenter)
1057 if (op%mesh%parallel_in_domains) then
1058 call group_by_pairs_sym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%inner%ri, op%inner%nri, &
1059 op%npairs, op%wpair, op%inner%ri_pos(:,:,ldf+1), op%inner%ri_neg(:,:,ldf+1), op%wcenter)
1060 call group_by_pairs_sym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%outer%ri, op%outer%nri, &
1061 op%npairs, op%wpair, op%outer%ri_pos(:,:,ldf+1), op%outer%ri_neg(:,:,ldf+1), op%wcenter)
1062 end if
1063 case(op_antisymmetric)
1064 call group_by_pairs_antisym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%ri, op%nri, &
1065 op%npairs, op%wpair, op%ri_pos(:,:,ldf+1), op%ri_neg(:,:,ldf+1))
1066 if (op%mesh%parallel_in_domains) then
1067 call group_by_pairs_antisym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%inner%ri, op%inner%nri, &
1068 op%npairs, op%wpair, op%inner%ri_pos(:,:,ldf+1), op%inner%ri_neg(:,:,ldf+1))
1069 call group_by_pairs_antisym(op%stencil%size, ldf, op%stencil%points, op%w(:,1), op%outer%ri, op%outer%nri, &
1070 op%npairs, op%wpair, op%outer%ri_pos(:,:,ldf+1), op%outer%ri_neg(:,:,ldf+1))
1071 end if
1072 end select
1073 end do
1074
1075 if (.not. present(max_size)) then
1076 write(message(1), '(3a)') 'Debug info: Sorted weights for ', trim(op%label), '.'
1077 call messages_info(1, debug_only=.true.)
1078
1079 do ipair = 1, op%npairs
1080 write(message(1), '(a,i3,f25.10,2(1x,i8))') ' ', ipair, op%wpair(ipair), op%ri_pos(ipair,1,1), op%ri_neg(ipair,1,1)
1081 call messages_info(1, debug_only=.true.)
1082 end do
1083 end if
1084
1087
1089 subroutine reallocate_array(ri, stencil_size, nri, old_size, new_size)
1090 integer, allocatable, intent(inout) :: ri(:,:,:)
1091 integer, intent(in) :: stencil_size, nri
1092 integer, intent(in) :: old_size, new_size
1093
1094 integer, allocatable :: tmp(:,:,:)
1095
1096 safe_allocate_source_a(tmp, ri)
1097 safe_deallocate_a(ri)
1098 safe_allocate(ri(1:stencil_size, 1:nri, 1:new_size))
1099 ri(:,:,1:old_size) = tmp
1100 safe_deallocate_a(tmp)
1101 end subroutine reallocate_array
1102
1103#include "undef.F90"
1104#include "real.F90"
1105#include "nl_operator_inc.F90"
1106
1107#include "undef.F90"
1108#include "complex.F90"
1109#include "nl_operator_inc.F90"
1110
1111end module nl_operator_oct_m
1112
1113!! Local Variables:
1114!! mode: f90
1115!! coding: utf-8
1116!! End:
subroutine, public accel_free_buffer(this, async)
Definition: accel.F90:967
subroutine, public accel_kernel_build(this, file_name, kernel_name, flags)
Compile the program that contains a given kernel.
Definition: accel.F90:1800
pure logical function, public accel_is_enabled()
Definition: accel.F90:391
integer, parameter, public accel_mem_read_only
Definition: accel.F90:162
integer pure function, public accel_max_block_size()
Definition: accel.F90:1188
This module implements batches of mesh functions.
Definition: batch.F90:112
Module implementing boundary conditions in Octopus.
Definition: boundaries.F90:101
real(real64), parameter, public m_zero
Definition: global.F90:173
This module implements the index, used for the mesh points.
Definition: index.F90:101
This module is intended to contain "only mathematical" functions and procedures.
Definition: math.F90:94
This module defines the meshes, which are used in Octopus.
Definition: mesh.F90:97
integer function, public mesh_local_index_from_coords(mesh, ix)
This function returns the local index of the point for a given vector of integer coordinates.
Definition: mesh.F90:924
subroutine, public mesh_local_index_to_coords(mesh, ip, ix)
Given a local point index, this function returns the set of integer coordinates of the point.
Definition: mesh.F90:936
subroutine, public messages_not_implemented(feature, namespace)
Definition: messages.F90:1048
subroutine, public messages_obsolete_variable(namespace, name, rep)
Definition: messages.F90:980
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
Definition: messages.F90:138
subroutine, public messages_input_error(namespace, var, details, row, column)
Definition: messages.F90:671
subroutine, public messages_experimental(name, namespace)
Definition: messages.F90:1020
subroutine, public messages_info(no_lines, iunit, debug_only, stress, all_nodes, namespace)
Definition: messages.F90:570
This module handles the communicators for the various parallelization strategies.
Definition: multicomm.F90:124
This module defines non-local operators.
Definition: nl_operator.F90:95
subroutine, public dnl_operator_operate_diag(op, fo)
integer, parameter op_map
integer, parameter op_max
integer, parameter op_vec
subroutine, public nl_operator_init(op, label, symm)
initialize an instance of a non-local operator by setting the label
subroutine, public dnl_operator_operate_batch(op, fi, fo, ghost_update, profile, points, factor, async)
subroutine nl_operator_clear_gpu_buffers(op)
subroutine, public nl_operator_build_symmetric_weights(op, max_size)
Builds (or rebuild) the necessary arrays for symmetric and antisymmetric stencils.
integer, parameter, public op_all
integer dfunction_global
subroutine group_by_pairs_sym(size, ldf, offsets, wre, ri, nri, npairs, wpair, pair_pos, pair_neg, wcenter)
Take a list of weights and offsets and build pairs of symmetric points with common weights.
subroutine group_by_pairs_antisym(size, ldf, offsets, wre, ri, nri, npairs, wpair, pair_pos, pair_neg)
Take a list of weights and offsets and build pairs of symmetric points with common weights.
subroutine, public dnl_operator_operate(op, fi, fo, ghost_update, profile, points)
subroutine, public nl_operator_update_gpu_buffers(op)
subroutine, public nl_operator_global_init(namespace)
initialize global settings for non-local operators
subroutine, public nl_operator_output_weights(this)
integer, parameter, public op_general
subroutine, public nl_operator_end(op)
integer, parameter, public op_inner
subroutine, public znl_operator_operate(op, fi, fo, ghost_update, profile, points)
subroutine, public nl_operator_remove_zero_weight_points(op, space, mesh)
Removes the zero-weight points for constant weight stencils.
integer zfunction_global
integer, parameter op_invmap
integer, parameter, public op_symmetric
subroutine, public nl_operator_global_end()
integer, parameter, public op_outer
subroutine, public nl_operator_build(space, mesh, op, np, const_w, regenerate)
Creates the nonlocal operators for the stencils used for finite differences.
subroutine, public znl_operator_operate_batch(op, fi, fo, ghost_update, profile, points, factor, async)
integer pure function, public nl_operator_np_zero_bc(op)
integer, parameter, public op_antisymmetric
subroutine, public znl_operator_operate_diag(op, fo)
integer, parameter op_fortran
subroutine reallocate_array(ri, stencil_size, nri, old_size, new_size)
Reallocate an ri array.
integer pure function, public nl_operator_get_index(op, is, ip)
subroutine, public nl_operator_allocate_gpu_buffers(op)
integer, parameter op_min
This module contains interfaces for routines in operate.c.
Definition: operate_f.F90:96
Some general things and nomenclature:
Definition: par_vec.F90:150
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
Definition: profiling.F90:122
This module is intended to contain "only mathematical" functions and procedures.
Definition: sort.F90:96
This module defines stencils used in Octopus.
Definition: stencil.F90:114
subroutine, public stencil_end(this)
Definition: stencil.F90:194
type(type_t), parameter, public type_integer
Definition: types.F90:115
real(real64) function partner_weight(ip)
Weight of the point with the opposite offset (the point itself for general operators)
Describes mesh distribution to nodes.
Definition: mesh.F90:165
index type for non-local operators
data type for non local operators
int true(void)