Octopus
mesh_partition.F90
Go to the documentation of this file.
1!! Copyright (C) 2002-2013 M. Marques, A. Castro, A. Rubio, G. Bertsch, M. Oliveira
2!! Copyright (C) 2021 S. Ohlmann
3!!
4!! This program is free software; you can redistribute it and/or modify
5!! it under the terms of the GNU General Public License as published by
6!! the Free Software Foundation; either version 2, or (at your option)
7!! any later version.
8!!
9!! This program is distributed in the hope that it will be useful,
10!! but WITHOUT ANY WARRANTY; without even the implied warranty of
11!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12!! GNU General Public License for more details.
13!!
14!! You should have received a copy of the GNU General Public License
15!! along with this program; if not, write to the Free Software
16!! Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17!! 02110-1301, USA.
18!!
19
20#include "global.h"
21
23 use debug_oct_m
24 use global_oct_m
25 use index_oct_m
26 use io_oct_m
27 use mesh_oct_m
28 use metis_oct_m
30 use mpi_oct_m
32 use parser_oct_m
36 use space_oct_m
39
40 implicit none
41
42 private
43 public :: &
50
51 integer, parameter :: &
52 METIS = 1, &
53 parmetis = 2, &
54 hilbert = 3
55
56 integer, parameter :: &
57 RCB = 1, &
58 graph = 2
59
60contains
61
62 ! ---------------------------------------------------------------
72 ! ---------------------------------------------------------------
73 subroutine mesh_partition(mesh, namespace, space, lapl_stencil, vsize)
74 type(mesh_t), intent(inout) :: mesh
75 type(namespace_t), intent(in) :: namespace
76 class(space_t), intent(in) :: space
77 type(stencil_t), intent(in) :: lapl_stencil
78 integer, intent(in) :: vsize
81
82 integer :: jp, jpart
83 integer(imetis) :: iv
84 integer(int64) :: ivtmp, inb
85 integer :: ix(space%dim), jx(space%dim)
86
87 integer :: npart
88 integer :: ipart
89
90 integer(imetis) :: ne_global
91 integer(imetis) :: nv_global
98 integer(imetis) :: ne
99 integer :: nv
100
101 integer(imetis), allocatable :: xadj_global(:)
102 integer(imetis), allocatable :: adjncy_global(:)
103 integer(imetis), allocatable :: adjncy(:)
104 integer(imetis), allocatable :: xadj(:)
105
106#ifdef HAVE_METIS
107 integer(imetis), allocatable :: options(:)
108 integer(imetis) :: edgecut
109#endif
110 real(real32), allocatable :: tpwgts(:)
111
112 integer :: iunit
113 integer, allocatable :: rcounts(:)
114 integer(imetis) :: tmp
115 integer(imetis), allocatable :: rdispls(:)
116 integer(imetis), allocatable :: part(:)
117 integer(imetis), allocatable :: part_global(:)
118 integer(imetis), allocatable :: vtxdist(:)
119
120 type(stencil_t) :: stencil
121 integer :: stencil_to_use, default_method, method
122 integer :: library
123 integer, parameter :: star = 1, laplacian = 2
124 integer(imetis), allocatable :: istart(:)
125 integer, allocatable :: lsize(:)
126
127 integer :: default, ierr
129 push_sub_with_profile(mesh_partition)
130
131 if (mesh%np_global == 0) then
132 message(1) = 'The mesh is empty and cannot be partitioned.'
133 call messages_fatal(1, namespace=namespace)
134 end if
135
136 !%Variable MeshPartitionPackage
137 !%Type integer
138 !%Section Execution::Parallelization
139 !%Description
140 !% Decides which library to use to perform the mesh partition.
141 !% By default ParMETIS is used when available, otherwise METIS is used.
142 !%Option metis 1
143 !% METIS library.
144 !%Option parmetis 2
145 !% (Experimental) Use ParMETIS library to perform the mesh partition.
146 !% Only available if the code was compiled with ParMETIS support.
147 !%Option part_hilbert 3
148 !% Use the ordering along the Hilbert curve for partitioning.
149 !%End
150 default = metis
151#ifdef HAVE_PARMETIS
152 default = parmetis
153#endif
154 call parse_variable(namespace, 'MeshPartitionPackage', default, library)
155
156#if !defined(HAVE_METIS)
157 if (library == metis) then
158 message(1) = 'METIS was requested, but Octopus was compiled without it.'
159 call messages_fatal(1, only_root_writes = .true.)
160 end if
161#endif
162#if !defined(HAVE_PARMETIS)
163 if (library == parmetis) then
164 message(1) = 'PARMETIS was requested, but Octopus was compiled without it.'
165 call messages_fatal(1, only_root_writes = .true.)
166 end if
167#endif
168
169 !%Variable MeshPartitionStencil
170 !%Type integer
171 !%Default stencil_star
172 !%Section Execution::Parallelization
173 !%Description
174 !% To partition the mesh, it is necessary to calculate the connection
175 !% graph connecting the points. This variable selects which stencil
176 !% is used to do this.
177 !%Option stencil_star 1
178 !% An order-one star stencil.
179 !%Option laplacian 2
180 !% The stencil used for the Laplacian is used to calculate the
181 !% partition. This in principle should give a better partition, but
182 !% it is slower and requires more memory.
183 !%End
184 call parse_variable(namespace, 'MeshPartitionStencil', star, stencil_to_use)
185
186 if (stencil_to_use == star) then
187 call stencil_star_get_lapl(stencil, mesh%box%dim, order = 1)
188 else if (stencil_to_use == laplacian) then
189 call stencil_copy(lapl_stencil, stencil)
190 else
191 call messages_input_error(namespace, 'MeshPartitionStencil')
192 end if
193
194 ! Shortcut to the global number of vertices
195 if (mesh%np_global > huge(nv_global)) then
196 message(1) = "Error: too many grid points for this version of metis/parmetis."
197 message(2) = "Please use the 64-bit version."
198 call messages_fatal(2, namespace=namespace)
199 end if
200 nv_global = i8_to_imetis(mesh%np_global)
201
202 call partition_init(mesh%partition, imetis_to_i8(nv_global), mesh%mpi_grp)
203
204 ! Get number of partitions.
205 npart = mesh%mpi_grp%size
206 ipart = mesh%mpi_grp%rank + 1
207
208 safe_allocate(istart(1:npart))
209 safe_allocate(lsize(1:npart))
210 safe_allocate(vtxdist(1:npart+1))
211 call partition_get_local_size(mesh%partition, ivtmp, nv)
212 assert(ivtmp <= huge(0_imetis))
213 iv = i8_to_imetis(ivtmp)
214
215 ! Allocate local output matrix
216 safe_allocate(part(1:nv))
217 if (library == hilbert) then
218 part = ipart
219 else
220
221 call mesh%mpi_grp%allgather(iv, 1, mpi_metis_int, istart(1), 1, mpi_metis_int)
222 call mesh%mpi_grp%allgather(nv, 1, mpi_integer, lsize(1), 1, mpi_integer)
223
224 vtxdist(1:npart) = istart(1:npart)
225 vtxdist(npart+1) = nv_global + 1
226
227 ! Create graph with each point being represented by a
228 ! vertex and edges between neighbouring points.
229 safe_allocate(xadj(1:nv+1))
230 safe_allocate(adjncy(1:i4_to_imetis(stencil%size - 1)*nv))
231 ne = 1
232 ! Iterate over number of vertices.
233 do iv = 1, lsize(ipart)
234 ! Get coordinates of point iv (vertex iv).
235 call mesh_global_index_to_coords(mesh, imetis_to_i8(istart(ipart)-1 + iv), ix)
236
237 ! Set entry in index table.
238 xadj(iv) = ne
239 ! Check all possible neighbours.
240 do jp = 1, stencil%size
241 if (jp == stencil%center) cycle
242
243 ! Store coordinates of possible neighbors, they
244 ! are needed several times in the check below.
245 jx = ix + stencil%points(:, jp)
246
247 if (all(jx >= mesh%idx%nr(1, :)) .and. all(jx <= mesh%idx%nr(2, :))) then
248 ! Only points inside the mesh or its enlargement
249 ! are included in the graph.
250 inb = mesh_global_index_from_coords(mesh, jx)
251 if (inb /= 0 .and. inb <= nv_global) then
252 ! Store a new edge and increment edge counter.
253 adjncy(ne) = i8_to_imetis(inb)
254 ne = ne + 1
255 end if
256 end if
257 end do
258 end do
259 ! We started with ne=1 for simplicity, so ne is off by one in the end.
260 ne = ne - 1
261
262 ! Set the total number of edges plus one as last index.
263 ! (NOTE: the plus one is because we are using Fortran numbering)
264 ! The reason is: neighbours of node i are stored in adjncy(xadj(i):xadj(i+1)-1).
265 ! Setting the last index as mentioned makes special handling of last element
266 ! unnecessary (this indexing is a METIS requirement).
267 xadj(nv+1) = ne + 1
268
269
270 if (debug%info .or. library == metis) then
271 !Gather the global xadj and adjncy arrays
272 safe_allocate(rcounts(1:npart))
273 safe_allocate(rdispls(1:npart))
274
275 safe_allocate(xadj_global(1:nv_global + 1))
276 xadj_global(1) = 1
277 rcounts(1:npart) = lsize(1:npart)
278 rdispls(1:npart) = vtxdist(1:npart) - 1
279 call mesh%mpi_grp%allgatherv(xadj(2:), lsize(ipart), mpi_metis_int, &
280 xadj_global(2:), rcounts, rdispls, mpi_metis_int)
281 do jpart = 2, npart
282 do iv = 1, lsize(jpart)
283 xadj_global(istart(jpart) + iv) = xadj_global(istart(jpart) + iv) + xadj_global(istart(jpart)) - 1
284 end do
285 end do
286
287 ne_global = xadj_global(nv_global + 1)
288 safe_allocate(adjncy_global(1:ne_global))
289 do jpart = 1, npart
290 rdispls(jpart) = xadj_global(vtxdist(jpart)) - 1
291 tmp = xadj_global(vtxdist(jpart+1)) - 1 - rdispls(jpart)
292 assert(tmp < huge(0_int32))
293 rcounts(jpart) = imetis_to_i4(tmp)
294 end do
295 assert(xadj(nv+1)-1 < huge(0_int32))
296 call mesh%mpi_grp%allgatherv(adjncy, imetis_to_i4(xadj(nv+1)-1), mpi_metis_int, &
297 adjncy_global, rcounts, rdispls, mpi_metis_int)
298
299 safe_deallocate_a(rcounts)
300 safe_deallocate_a(rdispls)
301 end if
302
303
304 if (debug%info) then
305 ! DEBUG output. Write graph to file mesh_graph.txt.
306 message(1) = 'Info: Adjacency lists of the graph representing the grid'
307 message(2) = 'Info: are stored in debug/mesh_partition/mesh_graph.txt.'
308 message(3) = 'Info: Compatible with METIS programs pmetis and kmetis.'
309 message(4) = 'Info: First line contains number of vertices and edges.'
310 message(5) = 'Info: Edges are not directed and appear twice in the lists.'
311 call messages_info(5)
312 if (mpi_world%is_root()) then
313 call io_mkdir('debug/mesh_partition', namespace, parents=.true.)
314 iunit = io_open('debug/mesh_partition/mesh_graph.txt', namespace, action='write')
315 write(iunit, *) nv_global, ne_global/2
316 do iv = 1, nv
317 write(iunit, *) adjncy_global(xadj_global(iv):xadj_global(iv+1) - 1)
318 end do
319 call io_close(iunit)
320 end if
321 message(1) = "Info: Done writing mesh_graph.txt."
322 call messages_info(1, namespace=namespace)
323 end if
324
325 ! The sum of all tpwgts elements has to be 1 and
326 ! we don`t care about the weights. So; 1/npart
327 safe_allocate(tpwgts(1:vsize))
328 tpwgts(1:vsize) = real(1.0, real32)/real(vsize, real32)
329
330 select case (library)
331 case (metis)
332
333#ifdef HAVE_METIS
334 safe_allocate(options(1:metis_noptions))
335 call oct_metis_setdefaultoptions(options(1))
336 options(metis_option_numbering) = 1
337#endif
338 if (vsize < 8) then
339 default_method = rcb
340 else
341 default_method = graph
342 end if
343
344 !%Variable MeshPartition
345 !%Type integer
346 !%Section Execution::Parallelization
347 !%Description
348 !% When using METIS to perform the mesh partitioning, decides which
349 !% algorithm is used. By default, <tt>graph</tt> partitioning
350 !% is used for 8 or more partitions, and <tt>rcb</tt> for fewer.
351 !%Option rcb 1
352 !% Recursive coordinate bisection partitioning.
353 !%Option graph 2
354 !% Graph partitioning (called 'k-way' by METIS).
355 !%End
356 call parse_variable(namespace, 'MeshPartition', default_method, method)
357
358 safe_allocate(part_global(1:nv_global))
359
360 !Now we can call METIS
361 select case (method)
362 case (rcb)
363 message(1) = 'Info: Using METIS 5 multilevel recursive bisection to partition the mesh.'
364 call messages_info(1, namespace=namespace)
365#ifdef HAVE_METIS
366 ierr = oct_metis_partgraphrecursive(nv_global, 1_imetis, xadj_global(1), adjncy_global(1), &
367 i4_to_imetis(vsize), tpwgts(1), 1.01_4, options(1), edgecut, part_global(1))
368#endif
369 call metis_error_code(ierr)
370 case (graph)
371 message(1) = 'Info: Using METIS 5 multilevel k-way algorithm to partition the mesh.'
372 call messages_info(1, namespace=namespace)
373#ifdef HAVE_METIS
374 ierr = oct_metis_partgraphkway(nv_global, 1_imetis, xadj_global(1), adjncy_global(1), &
375 i4_to_imetis(vsize), tpwgts(1), 1.01_4, options(1), edgecut, part_global(1))
376#endif
377 call metis_error_code(ierr)
378 case default
379 message(1) = 'Selected partition method is not available in METIS 5.'
380 call messages_fatal(1, namespace=namespace)
381 end select
382
383 part(1:nv) = part_global(istart(ipart):istart(ipart) + nv - 1)
384
385#ifdef HAVE_METIS
386 safe_deallocate_a(options)
387#endif
388 safe_deallocate_a(part_global)
389
390 case (parmetis)
391
392#ifdef HAVE_PARMETIS
393 ! Call to ParMETIS with no imbalance tolerance
394 safe_allocate(options(1:3))
395 ! For the moment we use default options
396 options = 0
397 write(message(1),'(a)') 'Info: Using ParMETIS multilevel k-way algorithm to partition the mesh.'
398 call messages_info(1, namespace=namespace)
399 call oct_parmetis_v3_partkway(vtxdist(1), xadj(1), adjncy(1), &
400 1_imetis, i4_to_imetis(mesh%mpi_grp%size), tpwgts(1), 1.05_4, &
401 options(1), edgecut, part(1), mesh%mpi_grp%comm%MPI_VAL)
402 safe_deallocate_a(options)
403#endif
404
405 end select
406
407 safe_deallocate_a(vtxdist)
408
409 if (debug%info .or. library == metis) then
410 safe_deallocate_a(xadj_global)
411 safe_deallocate_a(adjncy_global)
412 end if
413 safe_deallocate_a(tpwgts)
414 safe_deallocate_a(xadj)
415 safe_deallocate_a(adjncy)
416 safe_deallocate_a(istart)
417 safe_deallocate_a(lsize)
418 end if
419
420 assert(all(part(1:nv) > 0))
421 assert(all(part(1:nv) <= vsize))
422 call partition_set(mesh%partition, imetis_to_i4(part))
423
424 safe_deallocate_a(part)
425
426 call stencil_end(stencil)
427 pop_sub_with_profile(mesh_partition)
428
429 contains
430
431 subroutine metis_error_code(ierr)
432 integer, intent(in) :: ierr
433
435
436 select case (ierr)
437 case (metis_ok)
438 !Everything OK
439 case (metis_error_input)
440 message(1) = "Metis: Input error."
441 call messages_fatal(1, namespace=namespace)
442 case (metis_error_memory)
443 message(1) = "Metis: Unable to allocate required memory."
444 call messages_fatal(1, namespace=namespace)
445 case (metis_error)
446 message(1) = "Metis: Some type of error."
447 call messages_fatal(1, namespace=namespace)
448 end select
449
451 end subroutine metis_error_code
452
453 end subroutine mesh_partition
454
455 ! ----------------------------------------------------
458 subroutine mesh_partition_from_parent(mesh, parent)
459 type(mesh_t), intent(inout) :: mesh
460 type(mesh_t), intent(in) :: parent
461
462 integer :: np, ip_local
463 integer(int64) :: istart, ip_global
464 integer(int64), allocatable :: points(:)
465 integer, allocatable :: part(:)
466 integer :: idx(1:3)
467
469
470 !Initialize partition
471 call partition_init(mesh%partition, mesh%np_global, mesh%mpi_grp)
472 call partition_get_local_size(mesh%partition, istart, np)
473
474 !Get the partition number of each point from the parent mesh
475 safe_allocate(points(1:np))
476 safe_allocate(part(1:np))
477 do ip_local = 1, np
478 ip_global = istart + ip_local - 1
479 call mesh_global_index_to_coords(mesh, ip_global, idx)
480 points(ip_local) = mesh_global_index_from_coords(parent, 2*idx)
481 end do
482 call partition_get_partition_number(parent%partition, np, points, part)
483
484 !Set the partition
485 call partition_set(mesh%partition, part)
486
487 !Free memory
488 safe_deallocate_a(points)
489 safe_deallocate_a(part)
492 end subroutine mesh_partition_from_parent
493
494 ! ----------------------------------------------------
495 subroutine mesh_partition_dump(restart, mesh, vsize, ierr)
496 type(restart_t), intent(in) :: restart
497 type(mesh_t), intent(in) :: mesh
498 integer, intent(in) :: vsize
499 integer, intent(out) :: ierr
500
501 integer :: err
502 character(len=6) :: numstring
503
504 push_sub_with_profile(mesh_partition_dump)
505
506 ierr = 0
507
508 if (restart%skip()) then
509 pop_sub_with_profile(mesh_partition_dump)
510 return
511 end if
512
513 message(1) = "Debug: Writing mesh partition restart."
514 call messages_info(1, debug_only=.true.)
515
516 write(numstring, '(i6.6)') vsize
518 call partition_dump(mesh%partition, restart%dir(), 'inner_partition_'//trim(numstring)//'.obf', err)
519 if (err /= 0) then
520 message(1) = "Unable to write inner mesh partition to 'inner_partition_"//trim(numstring)//".obf'"
521 call messages_warning(1)
522 ierr = ierr + 1
523 end if
524
525 message(1) = "Debug: Writing mesh partition restart done."
526 call messages_info(1, debug_only=.true.)
527
528 pop_sub_with_profile(mesh_partition_dump)
529 end subroutine mesh_partition_dump
530
531 ! ----------------------------------------------------
532 subroutine mesh_partition_load(restart, mesh, ierr)
533 type(restart_t), intent(in) :: restart
534 type(mesh_t), intent(inout) :: mesh
535 integer, intent(out) :: ierr
536
537 integer :: err
538 character(len=6) :: numstring
539 character(len=MAX_PATH_LEN) :: filename, dir
540
541 push_sub(mesh_partition_load)
542
543 ierr = 0
544
545 if (restart%skip()) then
546 ierr = -1
547 pop_sub(mesh_partition_load)
548 return
549 end if
550
551 message(1) = "Debug: Reading mesh partition restart."
552 call messages_info(1, debug_only=.true.)
553
554 call partition_init(mesh%partition, mesh%np_global, mesh%mpi_grp)
555
556 write(numstring, '(i6.6)') mesh%mpi_grp%size
557 dir = restart%dir()
558
559 !Read inner partition
560 filename = 'inner_partition_'//numstring//'.obf'
561 if (.not. io_file_exists(trim(dir)//"/"//filename)) then
562 message(1) = "Unable to read inner mesh partition, file '"//trim(filename)//"' does not exist."
563 call messages_warning(1)
564 ierr = ierr + 1
565 else
566 call partition_load(mesh%partition, restart%dir(), filename, err)
567 if (err /= 0) then
568 message(1) = "Unable to read inner mesh partition from '"//trim(filename)//"'."
569 call messages_warning(1)
570 ierr = ierr + 2
571 end if
572 end if
573
574 ! Free the memory in case we were unable to read the partitions
575 if (ierr /= 0) then
576 call partition_end(mesh%partition)
577 end if
578
579 message(1) = "Debug: Reading mesh partition restart done."
580 call messages_info(1, debug_only=.true.)
581
582 pop_sub(mesh_partition_load)
583 end subroutine mesh_partition_load
584
585
586 ! ----------------------------------------------------------------------
587 subroutine mesh_partition_write_info(mesh, iunit, namespace)
588 type(mesh_t), intent(in) :: mesh
589 integer, optional, intent(in) :: iunit
590 type(namespace_t), optional, intent(in) :: namespace
592 integer(int64) :: npoints
593 integer :: npart
594 integer, allocatable :: nghost(:), nbound(:), nlocal(:), nneigh(:)
595 integer :: num_ghost, num_bound, num_local, num_neigh
596 real(real64) :: quality
597
598 integer :: ipart
599 logical, allocatable :: is_a_neigh(:)
600 real(real64) :: scal
601
602 push_sub_with_profile(mesh_partition_write_info)
603
604 ! Build information about the mesh partition
605 npart = mesh%mpi_grp%size
606 npoints = mesh%np_part_global
607
608 safe_allocate(nghost(1:npart))
609 safe_allocate(nbound(1:npart))
610 safe_allocate(nlocal(1:npart))
611 safe_allocate(nneigh(1:npart))
612 safe_allocate(is_a_neigh(1:npart))
613
614 ipart = mesh%mpi_grp%rank + 1
615
616 num_local = mesh%np
617 num_ghost = mesh%pv%np_ghost
618 num_bound = mesh%pv%np_bndry
619
620 is_a_neigh = mesh%pv%ghost_rcounts > 0
621 num_neigh = count(is_a_neigh(1:npart))
622
623 safe_deallocate_a(is_a_neigh)
624
625 call mesh%mpi_grp%gather(num_neigh, 1, mpi_integer, nneigh(1), 1, mpi_integer, 0)
626 call mesh%mpi_grp%gather(num_local, 1, mpi_integer, nlocal(1), 1, mpi_integer, 0)
627 call mesh%mpi_grp%gather(num_ghost, 1, mpi_integer, nghost(1), 1, mpi_integer, 0)
628 call mesh%mpi_grp%gather(num_bound, 1, mpi_integer, nbound(1), 1, mpi_integer, 0)
629
630 ! Calculate partition quality
631 scal = real(npart, real64) /npoints
632
633 quality = m_zero
634
635 quality = quality + (real(maxval(nlocal), real64) - real(minval(nlocal), real64))**3
636 quality = quality + (sum(real(nghost, real64) **2))
637
638 quality = m_one/(m_one + quality)
639
640
641 ! Only the root has the information (nlocal, nghost, ...)
642 if (mesh%mpi_grp%is_root()) then
643 ! Write information about the partition
644 message(1) = &
645 'Info: Mesh partition:'
646 message(2) = ''
647 call messages_info(2, iunit=iunit, namespace=namespace)
648
649 write(message(1),'(a,e16.6)') &
650 ' Partition quality:', quality
651 message(2) = ''
652 call messages_info(2, iunit=iunit, namespace=namespace)
653
654 write(message(1),'(a)') &
655 ' Neighbours Ghost points'
656 write(message(2),'(a,i5,a,i10)') &
657 ' Average : ', nint(sum(real(nneigh, real64) )/npart), ' ', nint(sum(real(nghost, real64) )/npart)
658 write(message(3),'(a,i5,a,i10)') &
659 ' Minimum : ', minval(nneigh), ' ', minval(nghost)
660 write(message(4),'(a,i5,a,i10)') &
661 ' Maximum : ', maxval(nneigh), ' ', maxval(nghost)
662 message(5) = ''
663 call messages_info(5, iunit=iunit, namespace=namespace)
664
665 do ipart = 1, npart
666 write(message(1),'(a,i5)') &
667 ' Nodes in domain-group ', ipart
668 write(message(2),'(a,i10,a,i10)') &
669 ' Neighbours :', nneigh(ipart), &
670 ' Local points :', nlocal(ipart)
671 write(message(3),'(a,i10,a,i10)') &
672 ' Ghost points :', nghost(ipart), &
673 ' Boundary points :', nbound(ipart)
674 call messages_info(3, iunit=iunit, namespace=namespace)
675 end do
676
677 message(1) = ''
678 call messages_info(1, iunit=iunit, namespace=namespace)
679 end if
680
681 safe_deallocate_a(nghost)
682 safe_deallocate_a(nbound)
683 safe_deallocate_a(nlocal)
684 safe_deallocate_a(nneigh)
685
686 pop_sub_with_profile(mesh_partition_write_info)
687 end subroutine mesh_partition_write_info
688
689 ! ----------------------------------------------------
690 subroutine mesh_partition_messages_debug(mesh, namespace)
691 type(mesh_t), intent(in) :: mesh
692 type(namespace_t), intent(in) :: namespace
693
694 integer :: ip
695 integer(int64) :: ipg
696 integer :: iunit ! For debug output to files.
697 character(len=6) :: filenum
698
699 if (.not. debug%info) return
700
702
703 call io_mkdir('debug/mesh_partition', namespace, parents=.true.)
704
705 ! Debug output. Write points of each partition in a different file.
706 write(filenum, '(i6.6)') mesh%mpi_grp%rank+1
707
708 ! without boundary
709 iunit = io_open('debug/mesh_partition/mesh_partition.'//filenum, &
710 namespace, action='write')
711 do ip = 1, mesh%np
712 ipg = mesh_local2global(mesh, ip)
713 write(iunit, '(i8,99f18.8)') ipg, mesh_x_global(mesh, ipg)
714 end do
715 call io_close(iunit)
716
717 ! with boundary included
718 iunit = io_open('debug/mesh_partition/mesh_partition_all.'//filenum, &
719 namespace, action='write')
720 do ip = 1, mesh%np_part
721 ipg = mesh_local2global(mesh, ip)
722 write(iunit, '(i8,99f18.8)') ipg, mesh_x_global(mesh, ipg)
723 end do
724 call io_close(iunit)
725
726 ! points from enlargement
727 if (mpi_world%is_root()) then
728 iunit = io_open('debug/mesh_partition/mesh_partition_boundary', &
729 namespace, action='write')
730 do ipg = mesh%np_global+1, mesh%np_part_global
731 write(iunit, '(i8,99f18.8)') ipg, mesh_x_global(mesh, ipg)
732 end do
733 call io_close(iunit)
734 end if
735
737
738 end subroutine mesh_partition_messages_debug
739
740end module mesh_partition_oct_m
741
742!! Local Variables:
743!! mode: f90
744!! coding: utf-8
745!! End:
subroutine metis_error_code(ierr)
type(debug_t), save, public debug
Definition: debug.F90:133
real(real64), parameter, public m_zero
Definition: global.F90:173
real(real64), parameter, public m_one
Definition: global.F90:174
This module implements the index, used for the mesh points.
Definition: index.F90:101
Definition: io.F90:93
subroutine, public io_close(iunit, grp)
Definition: io.F90:444
logical function, public io_file_exists(filename, msg)
Returns true if a file with name 'filename' exists and issues a reminder.
Definition: io.F90:542
subroutine, public io_mkdir(fname, namespace, parents)
Definition: io.F90:338
integer function, public io_open(file, namespace, action, status, form, position, die, recl, grp)
Definition: io.F90:379
This module defines the meshes, which are used in Octopus.
Definition: mesh.F90:97
subroutine, public mesh_global_index_to_coords(mesh, ipg, ix)
Given a global point index, this function returns the set of integer coordinates of the point.
Definition: mesh.F90:914
integer(int64) function, public mesh_global_index_from_coords(mesh, ix)
This function returns the true global index of the point for a given vector of integer coordinates.
Definition: mesh.F90:905
integer(int64) function, public mesh_local2global(mesh, ip)
This function returns the global mesh index for a given local index.
Definition: mesh.F90:948
real(real64) function, dimension(1:mesh%box%dim), public mesh_x_global(mesh, ipg)
Given a global point index, this function returns the coordinates of the point.
Definition: mesh.F90:803
integer, parameter hilbert
subroutine, public mesh_partition_from_parent(mesh, parent)
create a mesh partition from a given parent mesh
subroutine, public mesh_partition_write_info(mesh, iunit, namespace)
integer, parameter graph
subroutine, public mesh_partition_messages_debug(mesh, namespace)
integer, parameter parmetis
subroutine, public mesh_partition(mesh, namespace, space, lapl_stencil, vsize)
This routine converts the mesh given by grid points into a graph.
subroutine, public mesh_partition_dump(restart, mesh, vsize, ierr)
subroutine, public mesh_partition_load(restart, mesh, ierr)
subroutine, public messages_warning(no_lines, all_nodes, namespace)
Definition: messages.F90:501
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
Definition: messages.F90:138
subroutine, public messages_fatal(no_lines, only_root_writes, namespace)
Definition: messages.F90:386
subroutine, public messages_input_error(namespace, var, details, row, column)
Definition: messages.F90:671
subroutine, public messages_info(no_lines, iunit, debug_only, stress, all_nodes, namespace)
Definition: messages.F90:570
This module contains interfaces for METIS and PARMETIS routines.
Definition: metis.F90:97
integer, parameter metis_ok
Returned normally.
Definition: metis.F90:142
integer(c_int), protected metis_option_numbering
Definition: metis.F90:126
integer, parameter metis_error_memory
Returned due to insufficient memory.
Definition: metis.F90:142
integer(c_int), protected metis_noptions
Definition: metis.F90:107
integer, parameter metis_error_input
Returned due to erroneous inputs and/or options.
Definition: metis.F90:142
type(mpi_datatype), parameter mpi_metis_int
Definition: metis.F90:154
integer, parameter metis_error
Some other errors.
Definition: metis.F90:142
type(mpi_grp_t), public mpi_world
Definition: mpi.F90:317
subroutine, public partition_init(partition, np_global, mpi_grp)
initialize the partition table
Definition: partition.F90:172
subroutine, public partition_set(partition, part)
Definition: partition.F90:222
subroutine, public partition_dump(partition, dir, filename, ierr)
write the partition data
Definition: partition.F90:239
subroutine, public partition_end(partition)
Definition: partition.F90:209
subroutine, public partition_load(partition, dir, filename, ierr)
read the partition data
Definition: partition.F90:288
subroutine, public partition_get_local_size(partition, istart, np_local)
Definition: partition.F90:359
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
Definition: profiling.F90:122
This module defines stencils used in Octopus.
Definition: stencil.F90:114
subroutine, public stencil_end(this)
Definition: stencil.F90:194
subroutine, public stencil_copy(input, output)
Definition: stencil.F90:176
This module defines routines, generating operators for a star stencil.
subroutine, public stencil_star_get_lapl(this, dim, order)
Describes mesh distribution to nodes.
Definition: mesh.F90:165
int true(void)