Octopus
mesh_init.F90
Go to the documentation of this file.
1!! Copyright (C) 2002-2006 M. Marques, A. Castro, A. Rubio, G. Bertsch
2!! Copyright (C) 2021 S. Ohlmann
3!!
4!! This program is free software; you can redistribute it and/or modify
5!! it under the terms of the GNU General Public License as published by
6!! the Free Software Foundation; either version 2, or (at your option)
7!! any later version.
8!!
9!! This program is distributed in the hope that it will be useful,
10!! but WITHOUT ANY WARRANTY; without even the implied warranty of
11!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12!! GNU General Public License for more details.
13!!
14!! You should have received a copy of the GNU General Public License
15!! along with this program; if not, write to the Free Software
16!! Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17!! 02110-1301, USA.
18!!
19
20#include "global.h"
21
24module mesh_init_oct_m
25 use accel_oct_m
26 use types_oct_m
28 use box_oct_m
32 use debug_oct_m
33 use global_oct_m
34 use iihash_oct_m
35 use index_oct_m
36 use math_oct_m
38 use mesh_oct_m
42 use mpi_oct_m
47 use parser_oct_m
51 use sort_oct_m
52 use space_oct_m
54 use utils_oct_m
55
56 implicit none
57
58 private
59 public :: &
63
64
65 integer, parameter :: INNER_POINT = 1
66 integer, parameter :: ENLARGEMENT_POINT = 2
67 integer, parameter :: BOUNDARY = -1
68
69contains
70
71! ---------------------------------------------------------
78 subroutine mesh_init_stage_1(mesh, namespace, space, box, coord_system, spacing, enlarge)
79 class(mesh_t), intent(inout) :: mesh
80 type(namespace_t), intent(in) :: namespace
81 class(space_t), intent(in) :: space
82 class(box_t), target, intent(in) :: box
83 class(coordinate_system_t), target, intent(in) :: coord_system
84 real(real64), intent(in) :: spacing(1:space%dim)
85 integer, intent(in) :: enlarge(1:space%dim)
86
87 integer :: idir, jj, delta
88 real(real64) :: x(space%dim), chi(space%dim), spacing_new(-1:1), box_bounds(2, space%dim), tol
89 logical :: out
90
91 push_sub_with_profile(mesh_init_stage_1)
92
93 mesh%box => box
94
95 safe_allocate(mesh%spacing(1:space%dim))
96 mesh%spacing = spacing ! this number can change in the following
97
98 mesh%use_curvilinear = coord_system%local_basis
99 ! the volume element is per point for curvilinear coordinates
100 mesh%use_vol_pp = mesh%use_curvilinear
101 mesh%coord_system => coord_system
102
103 call index_init(mesh%idx, space%dim)
104 mesh%idx%enlarge = enlarge
105
106 ! get box bounds along the axes that generate the grid points
107 select type (coord_system)
108 class is (affine_coordinates_t)
109 box_bounds = box%bounds(coord_system%basis)
110 class default
111 box_bounds = box%bounds()
112 end select
113
114 tol = box_boundary_delta*max(m_one, maxval(abs(box_bounds)))
115
116 ! adjust nr
117 do idir = 1, space%dim
118 chi = m_zero
119 ! the upper border
120 jj = 0
121 out = .false.
122 do while(.not.out)
123 jj = jj + 1
124 chi(idir) = real(jj, real64) * mesh%spacing(idir)
125 if (mesh%use_curvilinear) then
126 x = coord_system%to_cartesian(chi)
127 out = x(idir) > maxval(abs(box_bounds(:, idir))) + tol
128 else
129 ! do the same comparison here as in simul_box_contains_points
130 out = chi(idir) > maxval(abs(box_bounds(:, idir))) + tol
131 end if
132 end do
133 mesh%idx%nr(2, idir) = jj - 1
134 end do
135
136 ! we have a symmetric mesh (for now)
137 mesh%idx%nr(1,:) = -mesh%idx%nr(2,:)
139 ! we have to adjust a couple of things for the periodic directions
140 do idir = 1, space%periodic_dim
141 if (mesh%idx%nr(2, idir) == 0) then
142 ! this happens if Spacing > box size
143 mesh%idx%nr(2, idir) = 1
144 mesh%idx%nr(1, idir) = -1
145 end if
146
147 ! We have to adjust the spacing to be commensurate with the box,
148 ! for this we scan the possible values of the grid size around the
149 ! one we selected. We choose the size that has the spacing closest
150 ! to the requested one.
151 do delta = -1, 1
152 spacing_new(delta) = m_two*maxval(abs(box_bounds(:, idir))) / real(2 * mesh%idx%nr(2, idir) + 1 - delta, real64)
153 spacing_new(delta) = abs(spacing_new(delta) - spacing(idir))
154 end do
155
156 delta = minloc(spacing_new, dim = 1) - 2
157
158 assert(delta >= -1)
159 assert(delta <= 1)
160
161 mesh%spacing(idir) = m_two*maxval(abs(box_bounds(:, idir))) / real(2 * mesh%idx%nr(2, idir) + 1 - delta, real64)
162
163 ! we need to adjust the grid by adding or removing one point
164 if (delta == -1) then
165 mesh%idx%nr(1, idir) = mesh%idx%nr(1, idir) - 1
166 else if (delta == 1) then
167 mesh%idx%nr(2, idir) = mesh%idx%nr(2, idir) - 1
168 end if
169
170 end do
171
172 if ( any(abs(mesh%spacing - spacing) > 1.e-6_real64) ) then
173 call messages_write('The spacing has been modified to make it commensurate with the periodicity of the system.')
174 call messages_warning()
175 end if
176
177 do idir = space%periodic_dim + 1, space%dim
178 if (mesh%idx%nr(2, idir) == 0) then
179 write(message(1),'(a,i2)') 'Spacing > box size in direction ', idir
180 call messages_fatal(1, namespace=namespace)
181 end if
182 end do
183
184 mesh%idx%ll = mesh%idx%nr(2, :) - mesh%idx%nr(1, :) + 1
185 ! compute strides for cubic indices
186 mesh%idx%stride(:) = 1
187 do idir = 2, space%dim+1
188 mesh%idx%stride(idir) = mesh%idx%stride(idir-1) * &
189 (mesh%idx%ll(idir-1) + 2*mesh%idx%enlarge(idir-1))
190 end do
191
192 pop_sub_with_profile(mesh_init_stage_1)
193 end subroutine mesh_init_stage_1
194
195 ! ---------------------------------------------------------
201 !
202 subroutine mesh_init_stage_2(mesh, namespace, space, box, stencil, grp, regenerate)
203 class(mesh_t), intent(inout) :: mesh
204 type(namespace_t), intent(in) :: namespace
205 class(space_t), intent(in) :: space
206 class(box_t), intent(in) :: box
207 type(stencil_t), intent(in) :: stencil
208 type(mpi_grp_t), intent(in) :: grp
209 logical, optional, intent(in) :: regenerate
210
211 integer :: is
212 real(real64) :: chi(1:space%dim)
213 real(real64) :: pos(space%dim)
214 integer :: point(space%dim), point_stencil(space%dim), grid_sizes(space%dim)
215 integer(int64) :: global_size
216 integer(int32) :: local_size
217 integer(int64) :: ispatial, ispatialb, istart, iend, spatial_size, ipg
218 integer :: ip, ib, ib2, np, np_boundary, ii
219 logical :: found
220 type(lihash_t) :: spatial_to_boundary
221 integer(int64), allocatable :: boundary_to_spatial(:), boundary_to_spatial_reordered(:)
222 integer(int64), allocatable :: grid_to_spatial(:), grid_to_spatial_initial(:), grid_to_spatial_reordered(:)
223 integer(int64), allocatable :: spatial_to_grid(:)
224 integer, allocatable :: sizes(:)
225 integer(int64), allocatable :: offsets(:)
226 integer :: size_boundary
227#ifdef HAVE_MPI
228 integer(int64), pointer :: ptr(:)
229 type(mpi_grp_t) :: internode_grp, intranode_grp
230#endif
231
232 push_sub_with_profile(mesh_init_stage_2)
233
234 if (.not. optional_default(regenerate, .false.)) then
235 ! enlarge mesh for boundary points
236 mesh%idx%nr(1, :) = mesh%idx%nr(1, :) - mesh%idx%enlarge(:)
237 mesh%idx%nr(2, :) = mesh%idx%nr(2, :) + mesh%idx%enlarge(:)
238 end if
239
240 !%Variable MeshIndexType
241 !%Type integer
242 !%Default idx_cubic
243 !%Section Mesh
244 !%Description
245 !% Determine index type. Must be the same for restarting a calculation.
246 !%Option idx_cubic 1
247 !% Cubic indices are used to map the spatial information to the grid points.
248 !%Option idx_hilbert 2
249 !% A Hilbert space-filling curve is used to map the spatial information to
250 !% the grid points.
251 !%End
252 call parse_variable(namespace, 'MeshIndexType', idx_cubic, mesh%idx%type)
253
254 grid_sizes = mesh%idx%nr(2, :) - mesh%idx%nr(1, :) + 1
255 mesh%idx%offset = grid_sizes/2
256 if (space%dim > 1 .and. any(grid_sizes > 2**(int(63/space%dim, int64)))) then
257 write(message(1), '(A, I10, A, I2, A)') "Error: grid too large, more than ", 2**(int(63/space%dim, int64)), &
258 " points in one direction for ", space%dim, " dimensions. This is not supported."
259 call messages_fatal(1, namespace=namespace)
260 end if
261 global_size = product(int(grid_sizes, int64))
262 ! compute the bits per dimension: grid_sizes(i) <= 2**bits
263 mesh%idx%bits = maxval(ceiling(log(real(grid_sizes, real64) )/log(2.0_real64)))
264
265 if (mesh%idx%type == idx_cubic) then
266 spatial_size = global_size
267 else if (mesh%idx%type == idx_hilbert) then
268 spatial_size = 2**(space%dim*mesh%idx%bits)
269 end if
270
271 ! use block data decomposition of spatial indices
272 istart = spatial_size * grp%rank/grp%size
273 iend = spatial_size * (grp%rank+1)/grp%size - 1
274 if (.not. (iend - istart + 1 < huge(0_int32))) then
275 write(message(1), '(A, I10, A, I2, A)') "Error: local grid too large, more than ", &
276 huge(0_int32), " points. This is not supported. Maybe use more MPI ranks?"
277 call messages_fatal(1, namespace=namespace)
278 end if
279 local_size = int(iend - istart + 1, int32)
280
281 safe_allocate(grid_to_spatial_initial(1:local_size))
282
283 ! get inner grid indices
284 ip = 1
285 do ispatial = istart, iend
286 call index_spatial_to_point(mesh%idx, space%dim, ispatial, point)
287 ! first check if point is outside bounding box
288 if (any(point < mesh%idx%nr(1, :) + mesh%idx%enlarge)) cycle
289 if (any(point > mesh%idx%nr(2, :) - mesh%idx%enlarge)) cycle
290 ! then check if point is inside simulation box
291 chi = real(point, real64) * mesh%spacing
292 pos = mesh%coord_system%to_cartesian(chi)
293 if (.not. box%contains_point(pos)) cycle
294 grid_to_spatial_initial(ip) = ispatial
295 assert(ip + 1 < huge(ip))
296 ip = ip + 1
297 end do
298 np = ip - 1
299
300 call rebalance_array(grp,grid_to_spatial_initial(1:np), grid_to_spatial, sizes)
301 np = sizes(grp%rank)
302
303 safe_deallocate_a(grid_to_spatial_initial)
304
305 safe_allocate(spatial_to_grid(grid_to_spatial(1):grid_to_spatial(np)))
306 safe_deallocate_a(sizes)
307
308 !$omp parallel do
309 do ispatial = grid_to_spatial(1), grid_to_spatial(np)
310 spatial_to_grid(ispatial) = -1
311 end do
312 !$omp parallel do
313 do ip = 1, np
314 spatial_to_grid(grid_to_spatial(ip)) = ip
315 end do
316
317 ! get local boundary indices
318 call lihash_init(spatial_to_boundary)
319 size_boundary = np
320 safe_allocate(boundary_to_spatial(1:size_boundary))
321 ib = 1
322 do ip = 1, np
323 call index_spatial_to_point(mesh%idx, space%dim, grid_to_spatial(ip), point)
324 do is = 1, stencil%size
325 if (stencil%center == is) cycle
326 point_stencil(1:space%dim) = point(1:space%dim) + stencil%points(1:space%dim, is)
327 ! check if point is in inner part
328 call index_point_to_spatial(mesh%idx, space%dim, ispatialb, point_stencil)
329 assert(ispatialb >= 0)
330 if (ispatialb >= lbound(spatial_to_grid, dim=1, kind=int64) .and. &
331 ispatialb <= ubound(spatial_to_grid, dim=1, kind=int64)) then
332 if (spatial_to_grid(ispatialb) > 0) cycle
333 end if
334 ! then check if point is inside simulation box
335 chi = real(point_stencil, real64) * mesh%spacing
336 pos = mesh%coord_system%to_cartesian(chi)
337 if (box%contains_point(pos)) cycle
338 ! it has to be a boundary point now
339 ! check if already counted
340 ib2 = lihash_lookup(spatial_to_boundary, ispatialb, found)
341 if (found) cycle
342 boundary_to_spatial(ib) = ispatialb
343 call lihash_insert(spatial_to_boundary, ispatialb, ib)
344 ib = ib + 1
345 ! enlarge array
346 if (ib >= size_boundary) then
347 size_boundary = size_boundary * 2
348 call make_array_larger(boundary_to_spatial, size_boundary)
349 end if
350 end do
351 end do
352 np_boundary = ib - 1
353 call lihash_end(spatial_to_boundary)
354 safe_deallocate_a(spatial_to_grid)
355
356 ! reorder inner points
357 call reorder_points(namespace, space, grp, mesh%idx, grid_to_spatial, grid_to_spatial_reordered)
358 safe_deallocate_a(grid_to_spatial)
359
360 call rebalance_array(grp,grid_to_spatial_reordered, grid_to_spatial, sizes)
361 np = sizes(grp%rank)
362 mesh%np_global = sizes(0)
363 do ii = 1, grp%size - 1
364 mesh%np_global = mesh%np_global + sizes(ii)
365 end do
366 safe_deallocate_a(sizes)
367 safe_deallocate_a(grid_to_spatial_reordered)
368
369 ! reorder boundary points
370 call make_array_larger(boundary_to_spatial, np_boundary)
371 call reorder_points(namespace, space, grp, mesh%idx, boundary_to_spatial, boundary_to_spatial_reordered)
372 safe_deallocate_a(boundary_to_spatial)
373
374 call rebalance_array(grp,boundary_to_spatial_reordered, boundary_to_spatial, sizes)
375 safe_deallocate_a(boundary_to_spatial_reordered)
376
377 ! global grid size
378 np_boundary = sizes(grp%rank)
379 mesh%np_part_global = mesh%np_global + sizes(0)
380 do ii = 1, grp%size - 1
381 mesh%np_part_global = mesh%np_part_global + sizes(ii)
382 end do
383 safe_deallocate_a(sizes)
384
385
386 ! get global indices
387#ifdef HAVE_MPI
388 ! create shared memory window and fill it only on root
389 call create_intranode_communicator(grp, intranode_grp, internode_grp)
390 call lmpi_create_shared_memory_window(mesh%np_part_global, intranode_grp, &
391 mesh%idx%window_grid_to_spatial, mesh%idx%grid_to_spatial_global)
392#else
393 safe_allocate(mesh%idx%grid_to_spatial_global(1:mesh%np_part_global))
394#endif
395 ! inner grid
396 call get_sizes_offsets(np, sizes, offsets, grp)
397 call grp%gatherv(grid_to_spatial, np, mpi_integer8, &
398 mesh%idx%grid_to_spatial_global, sizes, offsets, mpi_integer8, 0)
399
400 ! boundary indices
401 call get_sizes_offsets(np_boundary, sizes, offsets, grp)
402 call grp%gatherv(boundary_to_spatial, np_boundary, mpi_integer8, &
403 mesh%idx%grid_to_spatial_global(mesh%np_global+1:), sizes, offsets, mpi_integer8, 0)
404
405 ! fill global hash map
406#ifdef HAVE_MPI
407 ! create shared memory window and fill it only on root
408 call lmpi_create_shared_memory_window(spatial_size, intranode_grp, &
409 mesh%idx%window_spatial_to_grid, ptr)
410 mesh%idx%spatial_to_grid_global(0:spatial_size-1) => ptr(1:spatial_size)
411#else
412 safe_allocate(mesh%idx%spatial_to_grid_global(0:spatial_size-1))
413#endif
414 if (grp%is_root()) then
415 ! fill only on root, then broadcast
416 !$omp parallel do
417 do ispatial = 0, spatial_size-1
418 mesh%idx%spatial_to_grid_global(ispatial) = 0
419 end do
420 !$omp parallel do
421 do ipg = 1, mesh%np_part_global
422 mesh%idx%spatial_to_grid_global(mesh%idx%grid_to_spatial_global(ipg)) = ipg
423 end do
424 end if
425
426#ifdef HAVE_MPI
427 ! now broadcast the global arrays to local rank 0 on each node
428 if (intranode_grp%is_root()) then
429 call internode_grp%bcast(mesh%idx%grid_to_spatial_global(1), mesh%np_part_global, mpi_integer8, 0)
430 call internode_grp%bcast(mesh%idx%spatial_to_grid_global(0), spatial_size, mpi_integer8, 0)
431 end if
432 call lmpi_sync_shared_memory_window(mesh%idx%window_grid_to_spatial, intranode_grp)
433 call lmpi_sync_shared_memory_window(mesh%idx%window_spatial_to_grid, intranode_grp)
434#endif
435
436 safe_deallocate_a(offsets)
437 safe_deallocate_a(sizes)
438
439 safe_deallocate_a(boundary_to_spatial)
440 safe_deallocate_a(grid_to_spatial)
441
442 pop_sub_with_profile(mesh_init_stage_2)
443 end subroutine mesh_init_stage_2
444
445! ---------------------------------------------------------
450! ---------------------------------------------------------
451 subroutine mesh_init_stage_3(mesh, namespace, space, stencil, mc, parent, regenerate)
452 class(mesh_t), intent(inout) :: mesh
453 type(namespace_t), intent(in) :: namespace
454 class(space_t), intent(in) :: space
455 type(stencil_t), intent(in) :: stencil
456 type(multicomm_t), intent(in) :: mc
457 type(mesh_t), optional, intent(in) :: parent
458 logical, optional, intent(in) :: regenerate
459
460 integer :: ip, jj(space%dim), np
461
462 push_sub_with_profile(mesh_init_stage_3)
463
464 call mpi_grp_init(mesh%mpi_grp, mc%group_comm(p_strategy_domains))
465
466 ! check if we are running in parallel in domains
467 mesh%parallel_in_domains = (mesh%mpi_grp%size > 1)
468
469 call checksum_calculate(1, mesh%np_part_global, mesh%idx%grid_to_spatial_global(1), &
470 mesh%idx%checksum)
471
472 if (mesh%parallel_in_domains) then
473 call do_partition()
474 else
475 ! When running serially those two are the same.
476 assert(mesh%np_part_global < huge(mesh%np_part))
477 mesh%np = i8_to_i4(mesh%np_global)
478 mesh%np_part = i8_to_i4(mesh%np_part_global)
479
480 ! These must be initialized for par_vec_gather, par_vec_scatter to work
481 ! as copy operations when running without domain parallelization.
482 mesh%pv%np_global = mesh%np_global
483 mesh%pv%np_ghost = 0
484 mesh%pv%np_bndry = mesh%np_part - mesh%np
485 mesh%pv%npart = 1
486 mesh%pv%xlocal = 1
487 end if
488
489 ! Compute mesh%x
490 safe_allocate(mesh%x(1:space%dim, 1:mesh%np_part))
491 safe_allocate(mesh%x_t(1:mesh%np_part, 1:space%dim))
492 do ip = 1, mesh%np_part
493 mesh%x(1:space%dim, ip) = mesh_x_global(mesh, mesh_local2global(mesh, ip))
494 mesh%x_t(ip, 1:space%dim) = mesh%x(1:space%dim, ip)
495 end do
496
497 ! save chi, i.e. primitive coordinates
498 safe_allocate(mesh%chi(1:space%dim, 1:mesh%np_part))
499 do ip = 1, mesh%np_part
500 call mesh_local_index_to_coords(mesh, ip, jj)
501 mesh%chi(:, ip) = jj*mesh%spacing
502 end do
503
504 call mesh_get_vol_pp()
505
506 ! save inverse jacobian
507 if (mesh%coord_system%local_basis) then
508 np = mesh%np_part
509 else
510 np = 1
511 end if
512 safe_allocate(mesh%jacobian_inverse(1:space%dim, 1:space%dim, np))
513 do ip = 1, np
514 mesh%jacobian_inverse(:, :, ip) = mesh%coord_system%jacobian_inverse(mesh%chi(:, ip))
515 end do
516
517 pop_sub_with_profile(mesh_init_stage_3)
518
519 contains
520 ! ---------------------------------------------------------
521 subroutine do_partition()
522#ifdef HAVE_MPI
523 integer :: jj, ipart, jpart
524 integer(int64) :: ipg, jpg
525 integer, allocatable :: gindex(:), gedges(:)
526 logical, allocatable :: nb(:, :)
527 integer :: idx(space%dim), jx(space%dim)
528 type(mpi_comm) :: graph_comm
529 integer :: iedge, nnb
530 logical :: use_topo, reorder, partition_print
531 integer :: ierr
532
533 logical :: has_virtual_partition = .false.
534 integer :: vsize
535 type(restart_t) :: restart_load, restart_dump
536 integer, allocatable :: part_vec(:)
537
539
540 !Try to load the partition from the restart files
541 if (.not. optional_default(regenerate, .false.)) then
542 call restart_load%init(namespace, restart_partition, restart_type_load, mc, ierr, mesh=mesh, exact=.true.)
543 if (ierr == 0) call mesh_partition_load(restart_load, mesh, ierr)
544 call restart_load%end()
545 else
546 ierr = 0
547 end if
548
549 if (ierr /= 0) then
550
551 !%Variable MeshPartitionVirtualSize
552 !%Type integer
553 !%Default mesh mpi_grp size
554 !%Section Execution::Parallelization
555 !%Description
556 !% Gives the possibility to change the partition nodes.
557 !% Afterward, it crashes.
558 !%End
559 call parse_variable(namespace, 'MeshPartitionVirtualSize', mesh%mpi_grp%size, vsize)
560
561 if (vsize /= mesh%mpi_grp%size) then
562 write(message(1),'(a,I7)') "Changing the partition size to", vsize
563 write(message(2),'(a)') "The execution will crash."
564 call messages_warning(2, namespace=namespace)
565 has_virtual_partition = .true.
566 else
567 has_virtual_partition = .false.
568 end if
569
570 if (.not. present(parent)) then
571 call mesh_partition(mesh, namespace, space, stencil, vsize)
572 else
573 ! if there is a parent grid, use its partition
574 call mesh_partition_from_parent(mesh, parent)
575 end if
576
577 !Now that we have the partitions, we save them
578 call restart_dump%init(namespace, restart_partition, restart_type_dump, mc, ierr, mesh=mesh)
579 call mesh_partition_dump(restart_dump, mesh, vsize, ierr)
580 call restart_dump%end()
581 end if
582
583 if (has_virtual_partition) then
584 call profiling_end(namespace)
585 call print_date("Calculation ended on ")
586 write(message(1),'(a)') "Execution has ended."
587 write(message(2),'(a)') "If you want to run your system, do not use MeshPartitionVirtualSize."
588 call messages_warning(2, namespace=namespace)
589 call messages_end()
590 call global_end()
591 stop
592 end if
593
594 !%Variable MeshUseTopology
595 !%Type logical
596 !%Default false
597 !%Section Execution::Parallelization
598 !%Description
599 !% (experimental) If enabled, <tt>Octopus</tt> will use an MPI virtual
600 !% topology to map the processors. This can improve performance
601 !% for certain interconnection systems.
602 !%End
603 call parse_variable(namespace, 'MeshUseTopology', .false., use_topo)
604
605 if (use_topo) then
606 ! At the moment we still need the global partition. This will be removed in near future.
607 safe_allocate(part_vec(1:mesh%np_global))
608 call partition_get_global(mesh%partition, part_vec(1:mesh%np_global))
609
610
611 ! generate a table of neighbours
612
613 safe_allocate(nb(1:mesh%mpi_grp%size, 1:mesh%mpi_grp%size))
614 nb = .false.
615
616 do ipg = 1, mesh%np_global
617 ipart = part_vec(ipg)
618 call mesh_global_index_to_coords(mesh, ipg, idx)
619 do jj = 1, stencil%size
620 jx = idx + stencil%points(:, jj)
621 jpg = mesh_global_index_from_coords(mesh, jx)
622 if (jpg > 0 .and. jpg <= mesh%np_global) then
623 jpart = part_vec(jpg)
624 if (ipart /= jpart ) nb(ipart, jpart) = .true.
625 end if
626 end do
627 end do
628 safe_deallocate_a(part_vec)
629
630 ! now generate the information of the graph
631
632 safe_allocate(gindex(1:mesh%mpi_grp%size))
633 safe_allocate(gedges(1:count(nb)))
634
635 ! and now generate it
636 iedge = 0
637 do ipart = 1, mesh%mpi_grp%size
638 do jpart = 1, mesh%mpi_grp%size
639 if (nb(ipart, jpart)) then
640 iedge = iedge + 1
641 gedges(iedge) = jpart - 1
642 end if
643 end do
644 gindex(ipart) = iedge
645 end do
646
647 assert(iedge == count(nb))
648
649 reorder = .true.
650 call mpi_graph_create(mesh%mpi_grp%comm, mesh%mpi_grp%size, gindex, gedges, reorder, graph_comm)
651
652 ! we have a new communicator
653 call mpi_grp_init(mesh%mpi_grp, graph_comm)
654
655 safe_deallocate_a(nb)
656 safe_deallocate_a(gindex)
657 safe_deallocate_a(gedges)
658
659 end if
660
661 if (optional_default(regenerate, .false.)) call par_vec_end(mesh%pv)
662 call par_vec_init(mesh%mpi_grp, mesh%np_global, mesh%idx, stencil,&
663 space, mesh%partition, mesh%pv, namespace)
664
665 ! check the number of ghost neighbours in parallel
666 nnb = 0
667 jpart = mesh%pv%partno
668 do ipart = 1, mesh%pv%npart
669 if (ipart == jpart) cycle
670 if (mesh%pv%ghost_scounts(ipart) /= 0) nnb = nnb + 1
671 end do
672 assert(nnb >= 0 .and. nnb < mesh%pv%npart)
673
674 ! Set local point numbers.
675 mesh%np = mesh%pv%np_local
676 mesh%np_part = mesh%np + mesh%pv%np_ghost + mesh%pv%np_bndry
677
678 !%Variable PartitionPrint
679 !%Type logical
680 !%Default true
681 !%Section Execution::Parallelization
682 !%Description
683 !% (experimental) If disabled, <tt>Octopus</tt> will not compute
684 !% nor print the partition information, such as local points,
685 !% no. of neighbours, ghost points and boundary points.
686 !%End
687 call parse_variable(namespace, 'PartitionPrint', .true., partition_print)
688
689 if (partition_print) then
690 call mesh_partition_write_info(mesh, namespace=namespace)
691 call mesh_partition_messages_debug(mesh, namespace)
692 end if
693#endif
694
696 end subroutine do_partition
697
698
699 ! ---------------------------------------------------------
701 subroutine mesh_get_vol_pp()
702
703 integer :: ip, np
704
706
707 np = 1
708 if (mesh%use_vol_pp) np = mesh%np_part
709 ! If no local point, we should not try to access the arrays
710 if (mesh%np_part == 0) np = 0
711
712 safe_allocate(mesh%vol_pp(1:np))
713
714 do ip = 1, np
715 mesh%vol_pp(ip) = product(mesh%spacing)
716 end do
717
718 do ip = 1, np
719 mesh%vol_pp(ip) = mesh%vol_pp(ip)*mesh%coord_system%jacobian_determinant(mesh%chi(:, ip))
720 end do
721
722 if (mesh%use_vol_pp .or. mesh%np_part == 0) then
723 mesh%volume_element = m_one
724 else
725 mesh%volume_element = mesh%vol_pp(1)
726 end if
727
728 ! the weights on the device, for weighted reductions of device batches
729 if (mesh%use_vol_pp .and. accel_is_enabled()) then
730 call accel_create_buffer(mesh%vol_pp_buffer, accel_mem_read_only, type_float, max(np, 1))
731 call accel_create_buffer(mesh%sqrt_vol_pp_buffer, accel_mem_read_only, type_float, max(np, 1))
732 if (np > 0) then
733 call accel_write_buffer(mesh%vol_pp_buffer, np, mesh%vol_pp)
734 call accel_write_buffer(mesh%sqrt_vol_pp_buffer, np, sqrt(mesh%vol_pp))
735 end if
736 end if
737
739 end subroutine mesh_get_vol_pp
740
741 end subroutine mesh_init_stage_3
742
747 subroutine rebalance_array(grp, data_input, data_output, output_sizes)
748 type(mpi_grp_t), intent(in) :: grp
749 integer(int64), contiguous, intent(in) :: data_input(:)
750 integer(int64), allocatable, intent(out) :: data_output(:)
751 integer, allocatable, optional, intent(out) :: output_sizes(:)
752
753 integer, allocatable :: initial_sizes(:), final_sizes(:)
754 integer(int64), allocatable :: initial_offsets(:), final_offsets(:)
755 integer, allocatable :: scounts(:), sdispls(:)
756 integer, allocatable :: rcounts(:), rdispls(:)
757 integer :: irank
758 integer(int64) :: itmp
759
760 push_sub(rebalance_array)
761
762 ! collect current sizes of distributed array
763 safe_allocate(initial_sizes(0:grp%size-1))
764 call grp%allgather(size(data_input), 1, mpi_integer, initial_sizes(0), 1, mpi_integer)
765 safe_allocate(initial_offsets(0:grp%size))
766 initial_offsets(0) = 0
767 do irank = 1, grp%size
768 initial_offsets(irank) = initial_offsets(irank-1) + initial_sizes(irank-1)
769 end do
770
771 ! now redistribute the arrays
772 ! use block data decomposition of grid indices
773 safe_allocate(final_offsets(0:grp%size))
774 safe_allocate(final_sizes(0:grp%size-1))
775
776 do irank = 0, grp%size
777 final_offsets(irank) = sum(int(initial_sizes, int64)) * irank/grp%size
778 end do
779 do irank = 0, grp%size - 1
780 assert(final_offsets(irank + 1) - final_offsets(irank) < huge(0_int32))
781 final_sizes(irank) = int(final_offsets(irank + 1) - final_offsets(irank), int32)
782 end do
783
784 safe_allocate(scounts(0:grp%size-1))
785 safe_allocate(sdispls(0:grp%size-1))
786 safe_allocate(rcounts(0:grp%size-1))
787 safe_allocate(rdispls(0:grp%size-1))
788 ! determine communication pattern
789 scounts = 0
790 do irank = 0, grp%size - 1
791 ! get overlap of initial and final distribution
792 itmp = min(final_offsets(irank+1), initial_offsets(grp%rank+1)) - &
793 max(final_offsets(irank), initial_offsets(grp%rank))
794 assert(itmp < huge(0_int32))
795 if (itmp < 0) then
796 scounts(irank) = 0
797 else
798 scounts(irank) = int(itmp, int32)
799 end if
800 end do
801 sdispls(0) = 0
802 do irank = 1, grp%size - 1
803 sdispls(irank) = sdispls(irank - 1) + scounts(irank - 1)
804 end do
805 assert(sum(int(scounts, int64)) < huge(0_int32))
806 assert(sum(scounts) == initial_sizes(grp%rank))
807
808 rcounts = 0
809 do irank = 0, grp%size - 1
810 ! get overlap of initial and final distribution
811 itmp = min(final_offsets(grp%rank+1), initial_offsets(irank+1)) - &
812 max(final_offsets(grp%rank), initial_offsets(irank))
813 assert(itmp < huge(0_int32))
814 if (itmp < 0) then
815 rcounts(irank) = 0
816 else
817 rcounts(irank) = int(itmp, int32)
818 end if
819 end do
820 rdispls(0) = 0
821 do irank = 1, grp%size - 1
822 rdispls(irank) = rdispls(irank - 1) + rcounts(irank - 1)
823 end do
824 ! check for consistency between sending and receiving
825 assert(sum(rcounts) == final_sizes(grp%rank))
826
827 safe_allocate(data_output(1:final_sizes(grp%rank)))
828 call grp%alltoallv(data_input, scounts, sdispls, mpi_integer8, &
829 data_output, rcounts, rdispls, mpi_integer8)
830
831 ! save final sizes of array if optional argument present
832 if (present(output_sizes)) then
833 safe_allocate(output_sizes(0:grp%size-1))
834 output_sizes(:) = final_sizes(:)
835 end if
836
837 safe_deallocate_a(final_offsets)
838 safe_deallocate_a(final_sizes)
839
840 safe_deallocate_a(scounts)
841 safe_deallocate_a(sdispls)
842 safe_deallocate_a(rcounts)
843 safe_deallocate_a(rdispls)
844
845
846 pop_sub(rebalance_array)
847 end subroutine rebalance_array
848
853 subroutine reorder_points(namespace, space, grp, idx, grid_to_spatial, grid_to_spatial_reordered)
854 type(namespace_t), intent(in) :: namespace
855 class(space_t), intent(in) :: space
856 type(mpi_grp_t), intent(in) :: grp
857 type(index_t), intent(in) :: idx
858 integer(int64), contiguous, intent(in) :: grid_to_spatial(:)
859 integer(int64), allocatable, intent(out) :: grid_to_spatial_reordered(:)
860
861 integer :: bsize(space%dim), order, default
862 integer :: nn, idir, ipg, ip, number_of_blocks(space%dim)
863 type(block_t) :: blk
864 integer, parameter :: &
865 ORDER_BLOCKS = 1, &
866 order_original = 2, &
867 order_cube = 3
868 integer :: point(1:space%dim)
869 integer(int64), allocatable :: reorder_indices(:), reorder_recv(:)
870 integer, allocatable :: index_map(:), indices(:)
871 integer(int64), allocatable :: grid_to_spatial_recv(:)
872 integer, allocatable :: initial_sizes(:)
873 integer(int64), allocatable :: initial_offsets(:)
874 integer(int64) :: istart, iend, indstart, indend, spatial_size
875 integer :: irank, local_size, num_recv
876 integer :: iunique, nunique
877 integer :: direction
878 logical :: increase_with_dimension
879
880 integer, allocatable :: scounts(:), sdispls(:), rcounts(:), rdispls(:)
881 integer(int64), allocatable :: spatial_cutoff(:)
882
883 push_sub(reorder_points)
884
885 !%Variable MeshOrder
886 !%Type integer
887 !%Section Execution::Optimization
888 !%Description
889 !% This variable controls how the grid points are mapped to a
890 !% linear array for global arrays. For runs that are parallel
891 !% in domains, the local mesh order may be different (see
892 !% <tt>MeshLocalOrder</tt>).
893 !% The default is blocks when serial in domains and cube when
894 !% parallel in domains with the local mesh order set to blocks.
895 !%Option order_blocks 1
896 !% The grid is mapped using small parallelepipedic grids. The size
897 !% of the blocks is controlled by <tt>MeshBlockSize</tt>.
898 !%Option order_original 2
899 !% The original order of the indices is used to map the grid.
900 !%Option order_cube 3
901 !% The grid is mapped using a full cube, i.e. without blocking.
902 !%End
903 default = order_blocks
904 call parse_variable(namespace, 'MeshOrder', default, order)
905 ! no reordering in 1D necessary
906 if (space%dim == 1) then
907 order = order_original
908 end if
909
910 !%Variable MeshBlockDirection
911 !%Type integer
912 !%Section Execution::Optimization
913 !%Description
914 !% Determines the direction in which the dimensions are chosen to compute
915 !% the blocked index for sorting the mesh points (see MeshBlockSize).
916 !% The default is increase_with_dimensions, corresponding to xyz ordering
917 !% in 3D.
918 !%Option increase_with_dimension 1
919 !% The fastest changing index is in the first dimension, i.e., in 3D this
920 !% corresponds to ordering in xyz directions.
921 !%Option decrease_with_dimension 2
922 !% The fastest changing index is in the last dimension, i.e., in 3D this
923 !% corresponds to ordering in zyx directions.
924 !%End
925 call parse_variable(namespace, 'MeshBlockDirection', 1, direction)
926 increase_with_dimension = direction == 1
927 if (direction /= 1 .and. direction /= 2) then
928 call messages_input_error(namespace, 'MeshBlockDirection')
929 end if
930
931 select case (order)
932 case (order_original)
933 ! only copy points, they stay in their original ordering
934 safe_allocate(grid_to_spatial_reordered(1:size(grid_to_spatial)))
935 grid_to_spatial_reordered(1:size(grid_to_spatial)) = grid_to_spatial(1:size(grid_to_spatial))
936 case (order_blocks, order_cube)
937 if (order == order_cube) then
938 bsize = idx%nr(2, :) - idx%nr(1, :) + 1
939 else
940 !%Variable MeshBlockSize
941 !%Type block
942 !%Section Execution::Optimization
943 !%Description
944 !% To improve memory-access locality when calculating derivatives,
945 !% <tt>Octopus</tt> arranges mesh points in blocks. This variable
946 !% controls the size of this blocks in the different
947 !% directions. The default is selected according to the value of
948 !% the <tt>StatesBlockSize</tt> variable. (This variable only affects the
949 !% performance of <tt>Octopus</tt> and not the results.)
950 !%End
951 if (conf%target_states_block_size < 16) then
952 bsize(1) = 80 * 4 / abs(conf%target_states_block_size)
953 if (space%dim > 1) bsize(2) = 4
954 if (space%dim > 2) bsize(3:) = 10
955 else
956 bsize(1) = max(4 * 16 / abs(conf%target_states_block_size), 1)
957 if (space%dim > 1) bsize(2) = 15
958 if (space%dim > 2) bsize(3:) = 15
959 end if
960
961 if (parse_block(namespace, 'MeshBlockSize', blk) == 0) then
962 nn = parse_block_cols(blk, 0)
963 if (nn /= space%dim) then
964 message(1) = "Error: number of entries in MeshBlockSize must match the number of dimensions."
965 call messages_fatal(1, namespace=namespace)
966 end if
967 do idir = 1, nn
968 call parse_block_integer(blk, 0, idir - 1, bsize(idir))
969 end do
970 end if
971 end if
972
973 number_of_blocks = (idx%nr(2, :) - idx%nr(1, :) + 1) / bsize + 1
974
975
976 ! do the global reordering in parallel, use block data decomposition of global indices
977 ! reorder indices along blocked parallelepiped curve
978
979 ! collect current sizes of distributed array
980 safe_allocate(initial_sizes(0:grp%size-1))
981 call grp%allgather(size(grid_to_spatial), 1, mpi_integer, initial_sizes(0), 1, mpi_integer)
982 safe_allocate(initial_offsets(0:grp%size))
983 initial_offsets(0) = 0
984 do irank = 1, grp%size
985 initial_offsets(irank) = initial_offsets(irank-1) + initial_sizes(irank-1)
986 end do
987
988 ! get local range and size
989 istart = initial_offsets(grp%rank)
990 iend = initial_offsets(grp%rank + 1) - 1
991 assert(iend - istart + 1 < huge(0_int32))
992 local_size = int(iend - istart + 1, int32)
993 assert(local_size == initial_sizes(grp%rank))
994
995 ! compute new indices locally
996 safe_allocate(reorder_indices(1:local_size))
997 safe_allocate(indices(1:local_size))
998 safe_allocate(grid_to_spatial_reordered(1:local_size))
999 !$omp parallel do private(point)
1000 do ip = 1, local_size
1001 call index_spatial_to_point(idx, space%dim, grid_to_spatial(ip), point)
1002 point = point + idx%offset
1003 reorder_indices(ip) = get_blocked_index(space%dim, point, bsize, number_of_blocks, increase_with_dimension)
1004 end do
1005 ! parallel sort according to the new indices
1006 ! sort the local array
1007 call sort(reorder_indices, indices)
1008 ! save reordered indices to send to other processes
1009 !$omp parallel do
1010 do ip = 1, local_size
1011 grid_to_spatial_reordered(ip) = grid_to_spatial(indices(ip))
1012 end do
1013
1014 ! get minimum and maximum
1015 if(local_size > 0) then
1016 indstart = reorder_indices(1)
1017 indend = reorder_indices(local_size)
1018 else
1019 indstart = huge(1_int64)
1020 indend = 0
1021 end if
1022 call grp%allreduce_inplace(indstart, 1, mpi_integer8, mpi_min)
1023 call grp%allreduce_inplace(indend, 1, mpi_integer8, mpi_max)
1024 spatial_size = indend - indstart + 1
1025
1026 ! get index ranges for each rank
1027 safe_allocate(spatial_cutoff(0:grp%size-1))
1028 do irank = 0, grp%size - 1
1029 spatial_cutoff(irank) = spatial_size * (irank+1)/grp%size + indstart
1030 end do
1031
1032 safe_allocate(scounts(0:grp%size-1))
1033 safe_allocate(sdispls(0:grp%size-1))
1034 safe_allocate(rcounts(0:grp%size-1))
1035 safe_allocate(rdispls(0:grp%size-1))
1036 ! get send counts
1037 scounts = 0
1038 irank = 0
1039 ! the indices are ordered, so we can go through them and increase
1040 ! the rank to which they are associated to when we cross a cutoff
1041 do ip = 1, local_size
1042 if (reorder_indices(ip) >= spatial_cutoff(irank)) then
1043 ! this do loop is needed in case some ranks do not have any points
1044 do while (reorder_indices(ip) >= spatial_cutoff(irank))
1045 irank = irank + 1
1046 end do
1047 assert(irank < grp%size)
1048 end if
1049 scounts(irank) = scounts(irank) + 1
1050 end do
1051 safe_deallocate_a(spatial_cutoff)
1052 assert(sum(scounts) == local_size)
1053
1054 ! compute communication pattern (sdispls, rcounts, rdispls)
1055 sdispls(0) = 0
1056 do irank = 1, grp%size - 1
1057 sdispls(irank) = sdispls(irank - 1) + scounts(irank - 1)
1058 end do
1059
1060 call grp%alltoall(scounts, 1, mpi_integer, &
1061 rcounts, 1, mpi_integer)
1062
1063 rdispls(0) = 0
1064 do irank = 1, grp%size - 1
1065 rdispls(irank) = rdispls(irank - 1) + rcounts(irank - 1)
1066 end do
1067
1068 ! make sure the arrays get allocated also if we do not receive anything
1069 num_recv = max(sum(rcounts), 1)
1070 ! communicate the locally sorted indices
1071 safe_allocate(reorder_recv(1:num_recv))
1072 call grp%alltoallv(reorder_indices, scounts, sdispls, mpi_integer8, &
1073 reorder_recv, rcounts, rdispls, mpi_integer8)
1074 safe_deallocate_a(reorder_indices)
1075
1076 ! communicate the corresponding spatial indices
1077 safe_allocate(grid_to_spatial_recv(1:num_recv))
1078 call grp%alltoallv(grid_to_spatial_reordered, scounts, sdispls, mpi_integer8, &
1079 grid_to_spatial_recv, rcounts, rdispls, mpi_integer8)
1080 safe_deallocate_a(grid_to_spatial_reordered)
1081
1082 ! do k-way merge of sorted indices
1083 safe_allocate(reorder_indices(1:num_recv))
1084 safe_allocate(index_map(1:num_recv))
1085 if (sum(rcounts) > 0) then
1086 call merge_sorted_arrays(reorder_recv, rcounts, reorder_indices, index_map)
1087
1088 ! get number of unique indices, needed for boundary
1089 nunique = 1
1090 do ipg = 2, sum(rcounts)
1091 if (reorder_indices(ipg) /= reorder_indices(ipg-1)) then
1092 nunique = nunique + 1
1093 end if
1094 end do
1095
1096 ! reorder according to new order, but remove duplicate entries
1097 safe_allocate(grid_to_spatial_reordered(1:nunique))
1098 iunique = 1
1099 grid_to_spatial_reordered(iunique) = grid_to_spatial_recv(index_map(1))
1100 do ipg = 2, sum(rcounts)
1101 if (reorder_indices(ipg) /= reorder_indices(ipg-1)) then
1102 iunique = iunique + 1
1103 grid_to_spatial_reordered(iunique) = grid_to_spatial_recv(index_map(ipg))
1104 end if
1105 end do
1106 else
1107 safe_allocate(grid_to_spatial_reordered(1:0))
1108 end if
1109
1110 safe_deallocate_a(initial_offsets)
1111 safe_deallocate_a(initial_sizes)
1112
1113 safe_deallocate_a(reorder_indices)
1114 safe_deallocate_a(reorder_recv)
1115
1116 safe_deallocate_a(grid_to_spatial_recv)
1117 safe_deallocate_a(index_map)
1118 safe_deallocate_a(indices)
1119
1120 safe_deallocate_a(scounts)
1121 safe_deallocate_a(sdispls)
1122 safe_deallocate_a(rcounts)
1123 safe_deallocate_a(rdispls)
1124
1125 end select
1126 pop_sub(reorder_points)
1127 end subroutine reorder_points
1128
1131 subroutine get_sizes_offsets(local_size, sizes, offsets, mpi_grp)
1132 integer, intent(in) :: local_size
1133 integer, allocatable, intent(out) :: sizes(:)
1134 integer(int64), allocatable, intent(out) :: offsets(:)
1135 type(mpi_grp_t), intent(in) :: mpi_grp
1136
1137 integer :: irank
1138
1139 push_sub(get_sizes_offsets)
1140
1141 safe_allocate(sizes(0:mpi_grp%size-1))
1142 call mpi_grp%allgather(local_size, 1, mpi_integer, sizes(0), 1, mpi_integer)
1143 safe_allocate(offsets(0:mpi_grp%size))
1144 offsets(0) = 0
1145 do irank = 1, mpi_grp%size
1146 offsets(irank) = offsets(irank-1) + sizes(irank-1)
1147 end do
1148
1149 pop_sub(get_sizes_offsets)
1150 end subroutine get_sizes_offsets
1151
1152end module mesh_init_oct_m
1153
1154!! Local Variables:
1155!! mode: f90
1156!! coding: utf-8
1157!! End:
Box bounds along some axes.
Definition: box.F90:157
This is the common interface to a sorting routine. It performs the shell algorithm,...
Definition: sort.F90:133
double log(double __x) __attribute__((__nothrow__
double sqrt(double __x) __attribute__((__nothrow__
subroutine do_partition()
Definition: mesh_init.F90:584
subroutine mesh_get_vol_pp()
calculate the volume of integration
Definition: mesh_init.F90:592
pure logical function, public accel_is_enabled()
Definition: accel.F90:391
integer, parameter, public accel_mem_read_only
Definition: accel.F90:162
real(real64), parameter, public box_boundary_delta
Definition: box.F90:111
This module handles the calculation mode.
integer, parameter, public p_strategy_domains
parallelization in domains
real(real64), parameter, public m_two
Definition: global.F90:175
subroutine, public global_end()
Finalise parser varinfo file, and MPI.
Definition: global.F90:500
real(real64), parameter, public m_zero
Definition: global.F90:173
type(conf_t), public conf
Global instance of Octopus configuration.
Definition: global.F90:163
real(real64), parameter, public m_one
Definition: global.F90:174
This module implements a simple hash table for non-negative integer keys and integer values.
Definition: iihash.F90:104
integer function, public lihash_lookup(h, key, found)
Look up a value in the hash table h. If found is present, it indicates if key could be found in the t...
Definition: iihash.F90:313
subroutine, public lihash_insert(h, key, val)
Insert a (key, val) pair into the hash table h.
Definition: iihash.F90:287
subroutine, public lihash_end(h)
Free a hash table.
Definition: iihash.F90:265
subroutine, public lihash_init(h)
Initialize a hash table h.
Definition: iihash.F90:242
This module implements the index, used for the mesh points.
Definition: index.F90:101
subroutine, public index_init(idx, dim)
This subroutine allocates memory and initializes some components.
Definition: index.F90:197
integer(int64) function, public get_blocked_index(dim, point, bsize, number_of_blocks, increase_with_dimensions)
Definition: index.F90:476
subroutine, public index_spatial_to_point(idx, dim, ispatial, point)
Definition: index.F90:377
integer, parameter, public idx_hilbert
Definition: index.F90:146
subroutine, public index_point_to_spatial(idx, dim, ispatial, point)
Definition: index.F90:395
integer, parameter, public idx_cubic
Definition: index.F90:146
This module is intended to contain "only mathematical" functions and procedures.
Definition: math.F90:94
subroutine, public merge_sorted_arrays(array, sizes, merged, index_map)
This module contains subroutines, related to the initialization of the mesh.
Definition: mesh_init.F90:96
subroutine rebalance_array(grp, data_input, data_output, output_sizes)
re-distribute the points to improve load balancing
Definition: mesh_init.F90:638
subroutine, public mesh_init_stage_3(mesh, namespace, space, stencil, mc, parent, regenerate)
When running parallel in domains, stencil and np_stencil are needed to compute the ghost points....
Definition: mesh_init.F90:514
subroutine, public mesh_init_stage_1(mesh, namespace, space, box, coord_system, spacing, enlarge)
First stage mesh initialization.
Definition: mesh_init.F90:151
subroutine get_sizes_offsets(local_size, sizes, offsets, mpi_grp)
return the sizes and offsets of a distributed array for all tasks of a mpi group.
Definition: mesh_init.F90:1022
subroutine reorder_points(namespace, space, grp, idx, grid_to_spatial, grid_to_spatial_reordered)
reorder the points in the grid according to the variables MeshOrder and MeshLocalOrder
Definition: mesh_init.F90:744
subroutine, public mesh_init_stage_2(mesh, namespace, space, box, stencil, grp, regenerate)
This subroutine creates the global array of spatial indices and the inverse mapping.
Definition: mesh_init.F90:275
This module defines the meshes, which are used in Octopus.
Definition: mesh.F90:97
subroutine, public mesh_global_index_to_coords(mesh, ipg, ix)
Given a global point index, this function returns the set of integer coordinates of the point.
Definition: mesh.F90:914
integer(int64) function, public mesh_global_index_from_coords(mesh, ix)
This function returns the true global index of the point for a given vector of integer coordinates.
Definition: mesh.F90:905
subroutine, public mesh_local_index_to_coords(mesh, ip, ix)
Given a local point index, this function returns the set of integer coordinates of the point.
Definition: mesh.F90:936
integer(int64) function, public mesh_local2global(mesh, ip)
This function returns the global mesh index for a given local index.
Definition: mesh.F90:948
real(real64) function, dimension(1:mesh%box%dim), public mesh_x_global(mesh, ipg)
Given a global point index, this function returns the coordinates of the point.
Definition: mesh.F90:803
subroutine, public mesh_partition_from_parent(mesh, parent)
create a mesh partition from a given parent mesh
subroutine, public mesh_partition_write_info(mesh, iunit, namespace)
subroutine, public mesh_partition_messages_debug(mesh, namespace)
subroutine, public mesh_partition(mesh, namespace, space, lapl_stencil, vsize)
This routine converts the mesh given by grid points into a graph.
subroutine, public mesh_partition_dump(restart, mesh, vsize, ierr)
subroutine, public mesh_partition_load(restart, mesh, ierr)
subroutine, public messages_end()
Definition: messages.F90:249
subroutine, public messages_warning(no_lines, all_nodes, namespace)
Definition: messages.F90:501
subroutine, public print_date(str)
Definition: messages.F90:963
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
Definition: messages.F90:138
subroutine, public messages_fatal(no_lines, only_root_writes, namespace)
Definition: messages.F90:386
subroutine, public messages_input_error(namespace, var, details, row, column)
Definition: messages.F90:671
This module contains some common usage patterns of MPI routines.
Definition: mpi_lib.F90:94
subroutine mpi_grp_init(grp, comm)
Initialize MPI group instance.
Definition: mpi.F90:386
This module handles the communicators for the various parallelization strategies.
Definition: multicomm.F90:124
Some general things and nomenclature:
Definition: par_vec.F90:150
subroutine, public par_vec_end(pv)
Deallocate memory used by pv.
Definition: par_vec.F90:734
subroutine, public par_vec_init(mpi_grp, np_global, idx, stencil, space, partition, pv, namespace)
Initializes a par_vec_type object (parallel vector).
Definition: par_vec.F90:279
integer function, public parse_block(namespace, name, blk, check_varinfo_)
Definition: parser.F90:600
subroutine, public partition_get_global(partition, part_global, root)
Returns the global partition. If root is present, the partition is gathered only in that node....
Definition: partition.F90:375
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
Definition: profiling.F90:122
subroutine, public profiling_end(namespace)
Definition: profiling.F90:455
integer, parameter, public restart_partition
Definition: restart.F90:133
integer, parameter, public restart_type_dump
Definition: restart.F90:161
integer, parameter, public restart_type_load
Definition: restart.F90:161
This module is intended to contain "only mathematical" functions and procedures.
Definition: sort.F90:96
This module defines stencils used in Octopus.
Definition: stencil.F90:114
integer, dimension(7), parameter sizes
Definition: types.F90:129
type(type_t), parameter, public type_float
Definition: types.F90:113
This module is intended to contain simple general-purpose utility functions and procedures.
Definition: utils.F90:99
subroutine, public make_array_larger(array, new_size)
Definition: utils.F90:658
class to tell whether a point is inside or outside
Definition: box.F90:120
Describes mesh distribution to nodes.
Definition: mesh.F90:165
This is defined even when running serial.
Definition: mpi.F90:121
Stores all communicators and groups.
Definition: multicomm.F90:185
The class representing the stencil, which is used for non-local mesh operations.
Definition: stencil.F90:142
int true(void)