Octopus
batch_ops.F90
Go to the documentation of this file.
1!! Copyright (C) 2008 X. Andrade
2!!
3!! This program is free software; you can redistribute it and/or modify
4!! it under the terms of the GNU General Public License as published by
5!! the Free Software Foundation; either version 2, or (at your option)
6!! any later version.
7!!
8!! This program is distributed in the hope that it will be useful,
9!! but WITHOUT ANY WARRANTY; without even the implied warranty of
10!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11!! GNU General Public License for more details.
12!!
13!! You should have received a copy of the GNU General Public License
14!! along with this program; if not, write to the Free Software
15!! Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16!! 02110-1301, USA.
17!!
18
19#include "global.h"
20
23module batch_ops_oct_m
24 use accel_oct_m
25 use batch_oct_m
26 use blas_oct_m
27 use debug_oct_m
28 use iso_c_binding
29 use global_oct_m
32 use math_oct_m
35 use types_oct_m
36
37 implicit none
38
39 private
40 public :: &
42 batch_axpy, &
44 batch_gemv, &
45 batch_scal, &
47 batch_xpay, &
65
67 interface batch_axpy
68 module procedure dbatch_axpy_const
69 module procedure zbatch_axpy_const
70 module procedure sbatch_axpy_const
71 module procedure cbatch_axpy_const
72 module procedure dbatch_axpy_vec
73 module procedure zbatch_axpy_vec
74 end interface batch_axpy
75
77 interface batch_scal
78 module procedure dbatch_scal_const
79 module procedure zbatch_scal_const
80 module procedure dbatch_scal_vec
81 module procedure zbatch_scal_vec
82 end interface batch_scal
83
85 interface batch_axpby
86 module procedure dbatch_axpby
87 module procedure zbatch_axpby
88 end interface batch_axpby
89
91 interface batch_gemv
92 module procedure dbatch_gemv
93 module procedure zbatch_gemv
94 end interface batch_gemv
97 interface batch_scal2v
98 module procedure dbatch_scal2v
99 module procedure zbatch_scal2v
100 end interface batch_scal2v
101
103 interface batch_xpay
104 module procedure dbatch_xpay_vec
105 module procedure zbatch_xpay_vec
106 module procedure dbatch_xpay_const
107 module procedure zbatch_xpay_const
108 end interface batch_xpay
109
110 interface batch_add_with_map
111 module procedure batch_add_with_map_cpu
112 module procedure batch_add_with_map_accel
113 end interface batch_add_with_map
114
115 interface batch_copy_with_map
116 module procedure batch_copy_with_map_cpu
117 module procedure batch_copy_with_map_accel
118 end interface batch_copy_with_map
119
134 interface batch_set_state
135 module procedure dbatch_set_state1
136 module procedure zbatch_set_state1
137 module procedure sbatch_set_state1
138 module procedure cbatch_set_state1
139 module procedure dbatch_set_state2
140 module procedure zbatch_set_state2
141 module procedure sbatch_set_state2
142 module procedure cbatch_set_state2
143 module procedure dbatch_set_state3
144 module procedure zbatch_set_state3
145 module procedure sbatch_set_state3
146 module procedure cbatch_set_state3
147 end interface batch_set_state
148
150 module procedure dbatch_get_state1
151 module procedure zbatch_get_state1
152 module procedure sbatch_get_state1
153 module procedure cbatch_get_state1
154 module procedure dbatch_get_state2
155 module procedure zbatch_get_state2
156 module procedure sbatch_get_state2
157 module procedure cbatch_get_state2
158 module procedure dbatch_get_state3
159 module procedure zbatch_get_state3
160 module procedure sbatch_get_state3
161 module procedure cbatch_get_state3
162 end interface batch_get_state
164 interface batch_get_points
165 module procedure dbatch_get_points
166 module procedure zbatch_get_points
167 module procedure batch_get_points_accel
168 end interface batch_get_points
170 interface batch_set_points
171 module procedure dbatch_set_points
172 module procedure zbatch_set_points
173 module procedure batch_set_points_accel
174 end interface batch_set_points
176 interface batch_pointwise_mul
177 module procedure batch_mul_cj
178 end interface batch_pointwise_mul
179
182 interface batch_mul_mf
183 module procedure dbatch_mul_mf
184 module procedure zbatch_mul_mf
185 end interface batch_mul_mf
186
188contains
189
190 !--------------------------------------------------------------
192 subroutine batch_set_zero(this, np, async)
193 class(batch_t), intent(inout) :: this
194 integer, optional, intent(in) :: np
196 logical, optional, intent(in) :: async
197
198 integer :: ist_linear, ist, ip, np_
199
200 push_sub(batch_set_zero)
201
202 assert(not_in_openmp())
203
204 call profiling_in("BATCH_SET_ZERO")
205
206 select case (this%status())
208 np_ = optional_default(np, int(this%pack_size(2), int32))
209 assert(np_ <= int(this%pack_size(2), int32))
210 call accel_set_buffer_to_zero(this%ff_device, this%type(), (int(this%pack_size(1), int32) * np_), async=async)
211
212 case (batch_packed)
213 np_ = optional_default(np, int(this%pack_size(2), int32))
214 assert(np_ <= int(this%pack_size(2), int32))
215 if (this%type() == type_float) then
216 !$omp parallel do private(ist) schedule(static)
217 do ip = 1, np_
218 !$omp simd
219 do ist = 1, int(this%pack_size(1), int32)
220 this%dff_pack(ist, ip) = m_zero
221 end do
222 end do
223 else if (this%type() == type_float_single) then
224 !$omp parallel do private(ist) schedule(static)
225 do ip = 1, np_
226 !$omp simd
227 do ist = 1, int(this%pack_size(1), int32)
228 this%sff_pack(ist, ip) = 0.0_real32
229 end do
230 end do
231 else if (this%type() == type_cmplx_single) then
232 !$omp parallel do private(ist) schedule(static)
233 do ip = 1, np_
234 !$omp simd
235 do ist = 1, int(this%pack_size(1), int32)
236 this%cff_pack(ist, ip) = cmplx(0.0_real32, 0.0_real32, real32)
237 end do
238 end do
239 else
240 !$omp parallel do private(ist) schedule(static)
241 do ip = 1, np_
242 !$omp simd
243 do ist = 1, int(this%pack_size(1), int32)
244 this%zff_pack(ist, ip) = m_z0
245 end do
246 end do
247 end if
249 case (batch_not_packed)
250 if (this%type() == type_float) then
251 np_ = optional_default(np, ubound(this%dff_linear, dim=1))
252 assert(np_ <= ubound(this%dff_linear, dim=1))
253 do ist_linear = 1, this%nst_linear
254 !$omp parallel do schedule(static)
255 do ip = 1, np_
256 this%dff_linear(ip, ist_linear) = m_zero
257 end do
258 end do
259 else if (this%type() == type_float_single) then
260 np_ = optional_default(np, ubound(this%sff_linear, dim=1))
261 assert(np_ <= ubound(this%sff_linear, dim=1))
262 do ist_linear = 1, this%nst_linear
263 !$omp parallel do schedule(static)
264 do ip = 1, np_
265 this%sff_linear(ip, ist_linear) = 0.0_real32
266 end do
267 end do
268 else if (this%type() == type_cmplx_single) then
269 np_ = optional_default(np, ubound(this%cff_linear, dim=1))
270 assert(np_ <= ubound(this%cff_linear, dim=1))
271 do ist_linear = 1, this%nst_linear
272 !$omp parallel do schedule(static)
273 do ip = 1, np_
274 this%cff_linear(ip, ist_linear) = cmplx(0.0_real32, 0.0_real32, real32)
275 end do
276 end do
277 else
278 np_ = optional_default(np, ubound(this%zff_linear, dim=1))
279 assert(np_ <= ubound(this%zff_linear, dim=1))
280 do ist_linear = 1, this%nst_linear
281 !$omp parallel do schedule(static)
282 do ip = 1, np_
283 this%zff_linear(ip, ist_linear) = m_z0
284 end do
285 end do
286 end if
287
288 case default
289 message(1) = "batch_set_zero: unknown batch status."
290 call messages_fatal(1)
291
292 end select
293
294 call profiling_out("BATCH_SET_ZERO")
295
296 pop_sub(batch_set_zero)
297 end subroutine batch_set_zero
298
299 ! --------------------------------------------------------------
301 !
302 subroutine batch_get_points_accel(this, sp, ep, psi, ldpsi1, ldpsi2)
303 class(batch_t), intent(in) :: this
304 integer, intent(in) :: sp
305 integer, intent(in) :: ep
306 type(accel_mem_t), intent(inout) :: psi
307 integer, intent(in) :: ldpsi1
308 integer, intent(in) :: ldpsi2
309
310 integer :: tsize, ii, it
311 type(accel_kernel_t), save :: kernel
312 integer, allocatable :: linear_to_ist(:), linear_to_idim(:)
313 type(accel_mem_t) :: buff_linear_to_ist, buff_linear_to_idim
314
315 push_sub(batch_get_points_accel)
316 call profiling_in("GET_POINTS")
317
318 select case (this%status())
320 call messages_not_implemented('batch_get_points_accel for non-GPU batches')
321
323
324 tsize = types_get_size(this%type())/types_get_size(type_float)
325 safe_allocate(linear_to_ist(1:this%nst_linear*tsize))
326 safe_allocate(linear_to_idim(1:this%nst_linear*tsize))
327 do ii = 1, this%nst_linear
328 do it = 1, tsize
329 linear_to_ist(tsize*(ii-1)+it) = tsize*(this%linear_to_ist(ii) - 1) + it - 1
330 linear_to_idim(tsize*(ii-1)+it) = this%linear_to_idim(ii) - 1
331 end do
332 end do
333
334 call accel_create_buffer(buff_linear_to_ist, accel_mem_read_only, type_integer, this%nst_linear*tsize)
335 call accel_write_buffer(buff_linear_to_ist, this%nst_linear*tsize, linear_to_ist)
336 call accel_create_buffer(buff_linear_to_idim, accel_mem_read_only, type_integer, this%nst_linear*tsize)
337 call accel_write_buffer(buff_linear_to_idim, this%nst_linear*tsize, linear_to_idim)
338
339 call accel_kernel_start_call(kernel, 'points.cu', 'get_points<double>')
340
341 call accel_set_kernel_arg(kernel, 0, sp)
342 call accel_set_kernel_arg(kernel, 1, ep)
343 call accel_set_kernel_arg(kernel, 2, buff_linear_to_ist)
344 call accel_set_kernel_arg(kernel, 3, buff_linear_to_idim)
345 call accel_set_kernel_arg(kernel, 4, this%nst_linear*tsize)
346 call accel_set_kernel_arg(kernel, 5, this%ff_device)
347 call accel_set_kernel_arg(kernel, 6, int(this%pack_size_real(1), int32))
348 call accel_set_kernel_arg(kernel, 7, psi)
349 call accel_set_kernel_arg(kernel, 8, ldpsi1*tsize)
350 call accel_set_kernel_arg(kernel, 9, ldpsi2)
351
352 call accel_kernel_run(kernel, (/1_int64, int(ep - sp + 1, int64)/), (/this%pack_size_real(1), 1_int64/))
353
354 call accel_free_buffer(buff_linear_to_ist)
355 call accel_free_buffer(buff_linear_to_idim)
356 safe_deallocate_a(linear_to_ist)
357 safe_deallocate_a(linear_to_idim)
358
359 end select
360
361 call profiling_out("GET_POINTS")
362
364 end subroutine batch_get_points_accel
365
366 ! --------------------------------------------------------------
368 !
369 subroutine batch_set_points_accel(this, sp, ep, psi, ldpsi1, ldpsi2)
370 class(batch_t), intent(inout) :: this
371 integer, intent(in) :: sp
372 integer, intent(in) :: ep
373 type(accel_mem_t), intent(in) :: psi
374 integer, intent(in) :: ldpsi1
375 integer, intent(in) :: ldpsi2
376
377 integer :: tsize, ii, it
378 type(accel_kernel_t), save :: kernel
379 integer, allocatable :: linear_to_ist(:), linear_to_idim(:)
380 type(accel_mem_t) :: buff_linear_to_ist, buff_linear_to_idim
381
382 push_sub(batch_set_points_accel)
383 call profiling_in("SET_POINTS")
384
385 select case (this%status())
387 call messages_not_implemented('batch_set_points_accel for non-GPU batches')
388
390
391 tsize = types_get_size(this%type())/types_get_size(type_float)
392 safe_allocate(linear_to_ist(1:this%nst_linear*tsize))
393 safe_allocate(linear_to_idim(1:this%nst_linear*tsize))
394 do ii = 1, this%nst_linear
395 do it = 1, tsize
396 linear_to_ist(tsize*(ii-1)+it) = tsize*(this%linear_to_ist(ii) - 1) + it - 1
397 linear_to_idim(tsize*(ii-1)+it) = this%linear_to_idim(ii) - 1
398 end do
399 end do
400
401 call accel_create_buffer(buff_linear_to_ist, accel_mem_read_only, type_integer, this%nst_linear*tsize)
402 call accel_write_buffer(buff_linear_to_ist, this%nst_linear*tsize, linear_to_ist)
403 call accel_create_buffer(buff_linear_to_idim, accel_mem_read_only, type_integer, this%nst_linear*tsize)
404 call accel_write_buffer(buff_linear_to_idim, this%nst_linear*tsize, linear_to_idim)
405
406 call accel_kernel_start_call(kernel, 'points.cu', 'set_points<double>')
407
408 call accel_set_kernel_arg(kernel, 0, sp)
409 call accel_set_kernel_arg(kernel, 1, ep)
410 call accel_set_kernel_arg(kernel, 2, buff_linear_to_ist)
411 call accel_set_kernel_arg(kernel, 3, buff_linear_to_idim)
412 call accel_set_kernel_arg(kernel, 4, this%nst_linear*tsize)
413 call accel_set_kernel_arg(kernel, 5, psi)
414 call accel_set_kernel_arg(kernel, 6, ldpsi1*tsize)
415 call accel_set_kernel_arg(kernel, 7, ldpsi2)
416 call accel_set_kernel_arg(kernel, 8, this%ff_device)
417 call accel_set_kernel_arg(kernel, 9, int(this%pack_size_real(1), int32))
418
419 call accel_kernel_run(kernel, (/1_int64, int(ep - sp + 1, int64)/), (/this%pack_size_real(1), 1_int64/))
420
421 call accel_free_buffer(buff_linear_to_ist)
422 call accel_free_buffer(buff_linear_to_idim)
423 safe_deallocate_a(linear_to_ist)
424 safe_deallocate_a(linear_to_idim)
425
426 end select
427
428 call profiling_out("SET_POINTS")
429
431 end subroutine batch_set_points_accel
432
433 ! -------------------------
437 !
438 integer pure function batch_points_block_size() result(block_size)
439
440 block_size = 61440
442 end function batch_points_block_size
443
444! -------------------------
447 subroutine batch_mul_cj(np, xx, yy, zz, conjugate_yy)
448 integer, intent(in) :: np
449 class(batch_t), intent(in) :: xx
450 class(batch_t), intent(in) :: yy
451 class(batch_t), intent(inout) :: zz
452 logical, optional, intent(in) :: conjugate_yy
453
454 integer :: ii, ip
455 logical :: conj_yy
456 integer(int64), dimension(3) :: gsizes, bsizes
457 type(accel_kernel_t), save, target :: kernel_zmul_conj
458 type(accel_kernel_t), save, target :: kernel_zmul
459 type(accel_kernel_t), save, target :: kernel_dmul
460 type(accel_kernel_t), pointer :: kernel
461 push_sub(batch_mul_cj)
462
463 call xx%check_compatibility_with(yy)
464 call xx%check_compatibility_with(zz)
465
466 conj_yy = optional_default(conjugate_yy, .true.)
467
468 select case (xx%status())
469 case (batch_not_packed)
470 if (xx%type() == type_cmplx) then
471 if (conj_yy) then
472 !$omp parallel private(ii, ip)
473 do ii = 1, xx%nst_linear
474 !$omp do schedule(static)
475 do ip = 1, np
476 zz%zff_linear(ip, ii) = xx%zff_linear(ip, ii)*conjg(yy%zff_linear(ip, ii))
477 end do
478 !$omp end do
479 end do
480 !$omp end parallel
481 else
482 !$omp parallel private(ii, ip)
483 do ii = 1, xx%nst_linear
484 !$omp do schedule(static)
485 do ip = 1, np
486 zz%zff_linear(ip, ii) = xx%zff_linear(ip, ii)*yy%zff_linear(ip, ii)
487 end do
488 !$omp end do
489 end do
490 !$omp end parallel
491 end if
492 else
493 !$omp parallel private(ii, ip)
494 do ii = 1, xx%nst_linear
495 !$omp do schedule(static)
496 do ip = 1, np
497 zz%dff_linear(ip, ii) = xx%dff_linear(ip, ii)*yy%dff_linear(ip, ii)
498 end do
499 !$omp end do
500 end do
501 !$omp end parallel
502 end if
503
504 case (batch_packed)
505 if (xx%type() == type_cmplx) then
506 if (conj_yy) then
507 !$omp parallel do private(ii)
508 do ip = 1, np
509 !$omp simd
510 do ii = 1, xx%nst_linear
511 zz%zff_pack(ii, ip) = xx%zff_pack(ii, ip)*conjg(yy%zff_pack(ii, ip))
512 end do
513 end do
514 else
515 !$omp parallel do private(ii)
516 do ip = 1, np
517 !$omp simd
518 do ii = 1, xx%nst_linear
519 zz%zff_pack(ii, ip) = xx%zff_pack(ii, ip)*yy%zff_pack(ii, ip)
520 end do
521 end do
522 end if
523 else
524 !$omp parallel do private(ii)
525 do ip = 1, np
526 !$omp simd
527 do ii = 1, xx%nst_linear
528 zz%dff_pack(ii, ip) = xx%dff_pack(ii, ip)*yy%dff_pack(ii, ip)
529 end do
530 end do
531 end if
532
533 case (batch_device_packed)
534 if (xx%type() == type_cmplx) then
535 if (conj_yy) then
536 kernel => kernel_zmul_conj
537 call accel_kernel_start_call(kernel, 'batch_mul.cu', 'mul_batch<complex<double>, Conjugation::Yes>')
538 else
539 kernel => kernel_zmul
540 call accel_kernel_start_call(kernel, 'batch_mul.cu', 'mul_batch<complex<double>, Conjugation::No>')
541 end if
542
543 call accel_set_kernel_arg(kernel, 0, np)
544 call accel_set_kernel_arg(kernel, 1, xx%ff_device)
545 call accel_set_kernel_arg(kernel, 2, log2(int(xx%pack_size(1), int32)))
546 call accel_set_kernel_arg(kernel, 3, yy%ff_device)
547 call accel_set_kernel_arg(kernel, 4, log2(int(yy%pack_size(1), int32)))
548 call accel_set_kernel_arg(kernel, 5, zz%ff_device)
549 call accel_set_kernel_arg(kernel, 6, log2(int(zz%pack_size(1), int32)))
550
551 call accel_grid_size_extend_dim(int(np, int64), xx%pack_size(1), gsizes, bsizes, kernel)
552 else
553 kernel => kernel_dmul
554 call accel_kernel_start_call(kernel, 'batch_mul.cu', 'mul_batch<double>')
555
556 call accel_set_kernel_arg(kernel, 0, np)
557 call accel_set_kernel_arg(kernel, 1, xx%ff_device)
558 call accel_set_kernel_arg(kernel, 2, log2(int(xx%pack_size_real(1), int32)))
559 call accel_set_kernel_arg(kernel, 3, yy%ff_device)
560 call accel_set_kernel_arg(kernel, 4, log2(int(yy%pack_size_real(1), int32)))
561 call accel_set_kernel_arg(kernel, 5, zz%ff_device)
562 call accel_set_kernel_arg(kernel, 6, log2(int(zz%pack_size_real(1), int32)))
563
564 call accel_grid_size_extend_dim(int(np, int64), xx%pack_size_real(1), gsizes, bsizes, kernel)
565 end if
566
567 call accel_kernel_run(kernel, gsizes, bsizes)
568 end select
569
570 pop_sub(batch_mul_cj)
571 end subroutine batch_mul_cj
572
573! -------------------------
574 subroutine batch_add_with_map_cpu(np, map, xx, yy, zz)
575 integer, intent(in) :: np
576 integer, intent(in) :: map(:)
577 class(batch_t), intent(in) :: xx
578 class(batch_t), intent(in) :: yy
579 class(batch_t), intent(inout) :: zz
580 type(accel_mem_t) :: buff_map
581
582 push_sub(batch_add_with_map_cpu)
583
584 if (xx%status() /= batch_device_packed) then
585 if (xx%type() == type_float) then
586 call dbatch_add_with_map(np, map, xx, yy, zz)
587 else
588 call zbatch_add_with_map(np, map, xx, yy, zz)
589 end if
590 else
591 ! copy map to GPU if not already there
592 call accel_create_buffer(buff_map, accel_mem_read_only, type_integer, np)
593 call accel_write_buffer(buff_map, np, map)
594 call batch_add_with_map_accel(np, buff_map, xx, yy, zz)
595 call accel_free_buffer(buff_map)
596 end if
597
599 end subroutine batch_add_with_map_cpu
600
601! -------------------------
602 subroutine batch_add_with_map_accel(np, map, xx, yy, zz)
603 integer, intent(in) :: np
604 class(accel_mem_t), intent(in) :: map
605 class(batch_t), intent(in) :: xx
606 class(batch_t), intent(in) :: yy
607 class(batch_t), intent(inout) :: zz
608
609 type(accel_kernel_t), save :: kernel
610 integer(int64), dimension(3) :: gsizes, bsizes
611
613
614 call accel_kernel_start_call(kernel, 'copy.cu', 'add_with_map<double>')
615
616 call accel_set_kernel_arg(kernel, 0, np)
617 call accel_set_kernel_arg(kernel, 1, map)
618 call accel_set_kernel_arg(kernel, 2, xx%ff_device)
619 call accel_set_kernel_arg(kernel, 3, log2(int(xx%pack_size_real(1), int32)))
620 call accel_set_kernel_arg(kernel, 4, yy%ff_device)
621 call accel_set_kernel_arg(kernel, 5, log2(int(yy%pack_size_real(1), int32)))
622 call accel_set_kernel_arg(kernel, 6, zz%ff_device)
623 call accel_set_kernel_arg(kernel, 7, log2(int(zz%pack_size_real(1), int32)))
624
625 ! Compute the grid (extend to another dimensions if the size of the problem is too big)
626 call accel_grid_size_extend_dim(int(np, int64), xx%pack_size_real(1), gsizes, bsizes, kernel)
627
628 call accel_kernel_run(kernel, gsizes, bsizes)
629
631 end subroutine batch_add_with_map_accel
632
633! -------------------------
634 subroutine batch_copy_with_map_cpu(np, map, xx, yy)
635 integer, intent(in) :: np
636 integer, intent(in) :: map(:)
637 class(batch_t), intent(in) :: xx
638 class(batch_t), intent(inout) :: yy
639 type(accel_mem_t) :: buff_map
640
642
643 if (xx%status() /= batch_device_packed) then
644 if (xx%type() == type_float) then
645 call dbatch_copy_with_map(np, map, xx, yy)
646 else
647 call zbatch_copy_with_map(np, map, xx, yy)
648 end if
649 else
650 ! copy map to GPU if not already there
651 call accel_create_buffer(buff_map, accel_mem_read_only, type_integer, np)
652 call accel_write_buffer(buff_map, np, map)
653 call batch_copy_with_map_accel(np, buff_map, xx, yy)
654 call accel_free_buffer(buff_map)
655 end if
656
658 end subroutine batch_copy_with_map_cpu
659
660! -------------------------
661 subroutine batch_copy_with_map_accel(np, map, xx, yy)
662 integer, intent(in) :: np
663 class(accel_mem_t), intent(in) :: map
664 class(batch_t), intent(in) :: xx
665 class(batch_t), intent(inout) :: yy
666
667 type(accel_kernel_t), save :: kernel
668 integer(int64), dimension(3) :: gsizes, bsizes
669
671
672 call accel_kernel_start_call(kernel, 'copy.cu', 'copy_with_map<double>')
673
674 ! execute only if map has at least one element
675 if (np > 0) then
676 call accel_set_kernel_arg(kernel, 0, np)
677 call accel_set_kernel_arg(kernel, 1, map)
678 call accel_set_kernel_arg(kernel, 2, xx%ff_device)
679 call accel_set_kernel_arg(kernel, 3, log2(int(xx%pack_size_real(1), int32)))
680 call accel_set_kernel_arg(kernel, 4, yy%ff_device)
681 call accel_set_kernel_arg(kernel, 5, log2(int(yy%pack_size_real(1), int32)))
682
683 ! Compute the grid (extend to another dimensions if the size of the problem is too big)
684 call accel_grid_size_extend_dim(int(np, int64), xx%pack_size_real(1), gsizes, bsizes, kernel)
685
686 call accel_kernel_run(kernel, gsizes, bsizes)
687 end if
688
690 end subroutine batch_copy_with_map_accel
691
692 ! -------------------------
696 !
697 subroutine batch_split_complex(np, xx, yy, zz)
698 integer, intent(in) :: np
699 class(batch_t), intent(in) :: xx
700 class(batch_t), intent(inout) :: yy
701 class(batch_t), intent(inout) :: zz
702
703 integer :: ist_linear, ip
704 type(accel_kernel_t), save :: kernel
705 integer(int64), dimension(3) :: gsizes, bsizes
707 push_sub(batch_split_complex)
708
709 assert(xx%type() == type_cmplx)
710 assert(yy%type() == type_float)
711 assert(zz%type() == type_float)
712 assert(xx%status() == yy%status())
713 assert(xx%status() == zz%status())
714 assert(np <= xx%pack_size(2))
715 assert(np <= yy%pack_size(2))
716 assert(np <= zz%pack_size(2))
717
718 select case (xx%status())
719 case (batch_not_packed)
720 do ist_linear = 1, xx%nst_linear
721 !$omp parallel do schedule(static)
722 do ip = 1, np
723 yy%dff_linear(ip, ist_linear) = real(xx%zff_linear(ip, ist_linear), real64)
724 zz%dff_linear(ip, ist_linear) = aimag(xx%zff_linear(ip, ist_linear))
725 end do
726 end do
727 case (batch_packed)
728 !$omp parallel do private(ist_linear) schedule(static)
729 do ip = 1, np
730 do ist_linear = 1, xx%nst_linear
731 yy%dff_pack(ist_linear, ip) = real(xx%zff_pack(ist_linear, ip), real64)
732 zz%dff_pack(ist_linear, ip) = aimag(xx%zff_pack(ist_linear, ip))
733 end do
734 end do
735 case (batch_device_packed)
736 call accel_kernel_start_call(kernel, 'split.cu', 'split_complex<double>')
737
738 call accel_set_kernel_arg(kernel, 0, np)
739 call accel_set_kernel_arg(kernel, 1, xx%ff_device)
740 call accel_set_kernel_arg(kernel, 2, log2(int(xx%pack_size(1), int32)))
741 call accel_set_kernel_arg(kernel, 3, yy%ff_device)
742 call accel_set_kernel_arg(kernel, 4, log2(int(yy%pack_size(1), int32)))
743 call accel_set_kernel_arg(kernel, 5, zz%ff_device)
744 call accel_set_kernel_arg(kernel, 6, log2(int(zz%pack_size(1), int32)))
745
746 ! Compute the grid (extend to another dimensions if the size of the problem is too big)
747 call accel_grid_size_extend_dim(int(np, int64), xx%pack_size(1), gsizes, bsizes, kernel)
748
749 call accel_kernel_run(kernel, gsizes, bsizes)
750 end select
751
752 pop_sub(batch_split_complex)
753 end subroutine batch_split_complex
754
755 ! --------------------------------------------------------------
763 subroutine batch_copy_precision(src, dst, np)
764 class(batch_t), intent(in) :: src
765 class(batch_t), intent(inout) :: dst
766 integer, intent(in) :: np
767
768 integer :: ist, ip
769 integer(int64) :: gsizes(3), bsizes(3)
770 logical :: is_cplx, to_single
771 character(len=47) :: kernel_name
775 type(accel_kernel_t), save, target :: kernels(1:2, 1:2)
776 type(accel_kernel_t), pointer :: kernel
777
778 push_sub(batch_copy_precision)
779
780 assert(src%nst_linear == dst%nst_linear)
781 assert(src%status() == dst%status())
782 assert(type_is_complex(src%type()) .eqv. type_is_complex(dst%type()))
783 assert(type_is_single(src%type()) .neqv. type_is_single(dst%type()))
784
785 is_cplx = type_is_complex(src%type())
786 to_single = type_is_single(dst%type())
787
788 select case (src%status())
789 case (batch_device_packed)
790 if (is_cplx) then
791 kernel_name = merge('precision_copy<complex<double>, complex<float>>', &
792 'precision_copy<complex<float>, complex<double>>', to_single)
793 else
794 kernel_name = merge('precision_copy<double, float>', 'precision_copy<float, double>', to_single)
795 end if
796 kernel => kernels(merge(2, 1, is_cplx), merge(2, 1, to_single))
797 call accel_kernel_start_call(kernel, 'precision_copy.cu', trim(kernel_name))
798 call accel_set_kernel_arg(kernel, 0, np)
799 call accel_set_kernel_arg(kernel, 1, src%nst_linear)
800 call accel_set_kernel_arg(kernel, 2, src%ff_device)
801 call accel_set_kernel_arg(kernel, 3, log2(int(src%pack_size(1), int32)))
802 call accel_set_kernel_arg(kernel, 4, dst%ff_device)
803 call accel_set_kernel_arg(kernel, 5, log2(int(dst%pack_size(1), int32)))
804 call accel_grid_size_extend_dim(int(np, int64), dst%pack_size(1), gsizes, bsizes, kernel)
805 call accel_kernel_run(kernel, gsizes, bsizes)
806 call accel_finish()
807
808 case (batch_packed)
809 if (to_single) then
810 if (is_cplx) then
811 !$omp parallel do private(ist) schedule(static)
812 do ip = 1, np
813 do ist = 1, dst%nst_linear
814 dst%cff_pack(ist, ip) = cmplx(src%zff_pack(ist, ip), kind=real32)
815 end do
816 end do
817 else
818 !$omp parallel do private(ist) schedule(static)
819 do ip = 1, np
820 do ist = 1, dst%nst_linear
821 dst%sff_pack(ist, ip) = real(src%dff_pack(ist, ip), kind=real32)
822 end do
823 end do
824 end if
825 else
826 if (is_cplx) then
827 !$omp parallel do private(ist) schedule(static)
828 do ip = 1, np
829 do ist = 1, dst%nst_linear
830 dst%zff_pack(ist, ip) = src%cff_pack(ist, ip)
831 end do
832 end do
833 else
834 !$omp parallel do private(ist) schedule(static)
835 do ip = 1, np
836 do ist = 1, dst%nst_linear
837 dst%dff_pack(ist, ip) = src%sff_pack(ist, ip)
838 end do
839 end do
840 end if
841 end if
842
843 case (batch_not_packed)
844 if (to_single) then
845 if (is_cplx) then
846 !$omp parallel do private(ip) schedule(static)
847 do ist = 1, dst%nst_linear
848 do ip = 1, np
849 dst%cff_linear(ip, ist) = cmplx(src%zff_linear(ip, ist), kind=real32)
850 end do
851 end do
852 else
853 !$omp parallel do private(ip) schedule(static)
854 do ist = 1, dst%nst_linear
855 do ip = 1, np
856 dst%sff_linear(ip, ist) = real(src%dff_linear(ip, ist), kind=real32)
857 end do
858 end do
859 end if
860 else
861 if (is_cplx) then
862 !$omp parallel do private(ip) schedule(static)
863 do ist = 1, dst%nst_linear
864 do ip = 1, np
865 dst%zff_linear(ip, ist) = src%cff_linear(ip, ist)
866 end do
867 end do
868 else
869 !$omp parallel do private(ip) schedule(static)
870 do ist = 1, dst%nst_linear
871 do ip = 1, np
872 dst%dff_linear(ip, ist) = src%sff_linear(ip, ist)
873 end do
874 end do
875 end if
876 end if
877
878 end select
879
880 pop_sub(batch_copy_precision)
881 end subroutine batch_copy_precision
882
883
884#undef SPECIALIZED
885#include "undef.F90"
886#include "real.F90"
887#include "batch_ops_inc.F90"
888
889#include "undef.F90"
890#include "complex.F90"
891#include "batch_ops_inc.F90"
892
893! Specialized kernels for nst_linear=2
894#define SPECIALIZED 2
895#include "undef.F90"
896#include "real.F90"
897#include "batch_ops_inc.F90"
898
899#include "undef.F90"
900#include "complex.F90"
901#include "batch_ops_inc.F90"
902#undef SPECIALIZED
903
904! Specialized kernels for nst_linear=3
905#define SPECIALIZED 3
906#include "undef.F90"
907#include "real.F90"
908#include "batch_ops_inc.F90"
909
910#include "undef.F90"
911#include "complex.F90"
912#include "batch_ops_inc.F90"
913#undef SPECIALIZED
914
915! Specialized kernels for nst_linear=4
916#define SPECIALIZED 4
917#include "undef.F90"
918#include "real.F90"
919#include "batch_ops_inc.F90"
920
921#include "undef.F90"
922#include "complex.F90"
923#include "batch_ops_inc.F90"
924#undef SPECIALIZED
925
926! Specialized kernels for nst_linear=6
927#define SPECIALIZED 6
928#include "undef.F90"
929#include "real.F90"
930#include "batch_ops_inc.F90"
931
932#include "undef.F90"
933#include "complex.F90"
934#include "batch_ops_inc.F90"
935#undef SPECIALIZED
936
937! Specialized kernels for nst_linear=8
938#define SPECIALIZED 8
939#include "undef.F90"
940#include "real.F90"
941#include "batch_ops_inc.F90"
942
943#include "undef.F90"
944#include "complex.F90"
945#include "batch_ops_inc.F90"
946#undef SPECIALIZED
947
948! Single-precision batch operations: batch_ops_inc.F90 skips everything that needs a
949! single-precision lalg wrapper and keeps the subset the exchange path uses.
950#include "undef.F90"
951#include "real32.F90"
952#include "batch_ops_inc.F90"
953
954#include "undef.F90"
955#include "complex32.F90"
956#include "batch_ops_inc.F90"
957
958end module batch_ops_oct_m
959
960!! Local Variables:
961!! mode: f90
962!! coding: utf-8
963!! End:
batchified version of the BLAS axpy routine:
Definition: batch_ops.F90:139
batchified linear combination:
Definition: batch_ops.F90:163
batchified multiplication by mesh function with optional conjugation:
Definition: batch_ops.F90:254
batchified scale with optional conjugation:
Definition: batch_ops.F90:169
scale a batch by a constant or vector
Definition: batch_ops.F90:149
There are several ways how to call batch_set_state and batch_get_state:
Definition: batch_ops.F90:206
batchified version of
Definition: batch_ops.F90:175
double log2(double __x) __attribute__((__nothrow__
subroutine, public accel_free_buffer(this, async)
Definition: accel.F90:967
subroutine, public accel_kernel_start_call(this, file_name, kernel_name, flags)
Definition: accel.F90:1842
integer, parameter, public accel_mem_read_only
Definition: accel.F90:162
This module implements batches of mesh functions.
Definition: batch.F90:112
integer, parameter, public batch_not_packed
functions are stored in CPU memory, unpacked order
Definition: batch.F90:293
integer, parameter, public batch_device_packed
functions are stored in device memory in packed order
Definition: batch.F90:293
integer, parameter, public batch_packed
functions are stored in CPU memory, in transposed (packed) order
Definition: batch.F90:293
This module implements common operations on batches of mesh functions.
Definition: batch_ops.F90:95
subroutine zbatch_get_state3(this, ii, np, psi, async)
Definition: batch_ops.F90:3506
subroutine dbatch_get_state3(this, ii, np, psi, async)
Definition: batch_ops.F90:1416
subroutine batch_copy_with_map_accel(np, map, xx, yy)
Definition: batch_ops.F90:734
subroutine zbatch_get_state1(this, ist, np, psi, async)
Write a get of state with np points from a batch.
Definition: batch_ops.F90:3361
subroutine, public zbatch_copy_with_map_to_array(np, map, xx, array)
Transfer a batch from the mesh to an array on the submesh (defined by a map)
Definition: batch_ops.F90:4752
subroutine, public dbatch_ax_function_py(np, aa, psi, yy)
This routine performs a set of axpy operations adding the same function psi to all functions of a bat...
Definition: batch_ops.F90:1701
subroutine dbatch_set_state1(this, ist, np, psi)
Write a single state with np points into a batch at position ist.
Definition: batch_ops.F90:1134
subroutine dbatch_axpy_const(np, aa, xx, yy)
This routine applies a 'pair-wise' axpy operation to all functions of the batches xx and yy,...
Definition: batch_ops.F90:1038
subroutine zbatch_axpby(np, aa, xx, bb, yy)
calculate yy(ist,:) = aa*xx(ist,:) + bb*yy(ist,:) for a batch
Definition: batch_ops.F90:4169
subroutine cbatch_axpy_const(np, aa, xx, yy)
This routine applies a 'pair-wise' axpy operation to all functions of the batches xx and yy,...
Definition: batch_ops.F90:9009
subroutine zbatch_get_points(this, sp, ep, psi)
copy a set of points into a mesh function
Definition: batch_ops.F90:4501
subroutine dbatch_xpay_vec(np, xx, aa, yy, a_start, a_full)
calculate yy(ist,:) = xx(ist,:) + aa(ist)*yy(ist,:) for a batch
Definition: batch_ops.F90:1964
subroutine dbatch_scal_vec(np, aa, xx, a_start, a_full)
Definition: batch_ops.F90:2889
subroutine dbatch_set_state2(this, index, np, psi)
Write a single state with np points into a batch at position defined by index.
Definition: batch_ops.F90:1246
subroutine dbatch_copy_with_map(np, map, xx, yy)
Definition: batch_ops.F90:2776
subroutine cbatch_get_state2(this, index, np, psi, async)
Definition: batch_ops.F90:9373
subroutine, public dbatch_copy_with_map_to_array(np, map, xx, array)
Transfer a batch from the mesh to an array on the submesh (defined by a map)
Definition: batch_ops.F90:2819
subroutine dbatch_scal2v(np, aa, xx, yy, conjugate_xx)
calculate yy(ist,:) = aa*CJ(xx(ist,:)) for a batch
Definition: batch_ops.F90:2261
subroutine dbatch_get_state2(this, index, np, psi, async)
Definition: batch_ops.F90:1398
subroutine dbatch_axpy_vec(np, aa, xx, yy, a_start, a_full)
A simple switch between specialized kernels and generic kernels.
Definition: batch_ops.F90:2865
subroutine dbatch_set_state3(this, ii, np, psi)
Write a set of state with np points into a batch.
Definition: batch_ops.F90:1264
subroutine zbatch_axpy_const(np, aa, xx, yy)
This routine applies a 'pair-wise' axpy operation to all functions of the batches xx and yy,...
Definition: batch_ops.F90:3131
subroutine sbatch_set_state3(this, ii, np, psi)
Write a set of state with np points into a batch.
Definition: batch_ops.F90:8745
subroutine zbatch_scal_const(np, aa, xx)
scale all functions in a batch by constant aa
Definition: batch_ops.F90:3864
subroutine zbatch_get_state2(this, index, np, psi, async)
Definition: batch_ops.F90:3488
subroutine zbatch_xpay_const(np, xx, aa, yy)
calculate yy(ist) = xx(ist) + aa*yy(ist) for a batch
Definition: batch_ops.F90:4138
subroutine, public batch_split_complex(np, xx, yy, zz)
extract the real and imaginary parts of a complex batch
Definition: batch_ops.F90:770
subroutine batch_add_with_map_accel(np, map, xx, yy, zz)
Definition: batch_ops.F90:675
subroutine dbatch_set_points(this, sp, ep, psi)
copy a set of points into a mesh function
Definition: batch_ops.F90:2649
subroutine dbatch_get_state1(this, ist, np, psi, async)
Write a get of state with np points from a batch.
Definition: batch_ops.F90:1284
subroutine zbatch_copy_with_map(np, map, xx, yy)
Definition: batch_ops.F90:4709
subroutine dbatch_mul_mf(np, ff, xx, yy, conjugate_xx)
calculate yy(ist,:) = ff(:) * CJ(xx(ist,:)) for a batch
Definition: batch_ops.F90:2410
subroutine, public batch_set_zero(this, np, async)
fill all mesh functions of the batch with zero
Definition: batch_ops.F90:265
subroutine, public zbatch_ax_function_py(np, aa, psi, yy)
This routine performs a set of axpy operations adding the same function psi to all functions of a bat...
Definition: batch_ops.F90:3774
subroutine zbatch_axpy_vec(np, aa, xx, yy, a_start, a_full)
A simple switch between specialized kernels and generic kernels.
Definition: batch_ops.F90:4798
subroutine cbatch_set_state1(this, ist, np, psi)
Write a single state with np points into a batch at position ist.
Definition: batch_ops.F90:9100
subroutine batch_set_points_accel(this, sp, ep, psi, ldpsi1, ldpsi2)
GPU version of batch_set_points.
Definition: batch_ops.F90:442
subroutine dbatch_xpay_const(np, xx, aa, yy)
calculate yy(ist) = xx(ist) + aa*yy(ist) for a batch
Definition: batch_ops.F90:2107
subroutine dbatch_axpby(np, aa, xx, bb, yy)
calculate yy(ist,:) = aa*xx(ist,:) + bb*yy(ist,:) for a batch
Definition: batch_ops.F90:2138
subroutine dbatch_get_points(this, sp, ep, psi)
copy a set of points into a mesh function
Definition: batch_ops.F90:2569
subroutine zbatch_scal2v(np, aa, xx, yy, conjugate_xx)
calculate yy(ist,:) = aa*CJ(xx(ist,:)) for a batch
Definition: batch_ops.F90:4259
integer pure function, public batch_points_block_size()
determine the device block size
Definition: batch_ops.F90:511
subroutine dbatch_add_with_map(np, map, xx, yy, zz)
Definition: batch_ops.F90:2734
subroutine zbatch_gemv(np, xx, aa, bb, yy, aa2, bb2, yy2)
Linear combination of batches: yy(ist) = bb(ist) yy(ist) + sum_k aa(k, ist) xx(k)(ist)
Definition: batch_ops.F90:4853
subroutine zbatch_mul_mf(np, ff, xx, yy, conjugate_xx)
calculate yy(ist,:) = ff(:) * CJ(xx(ist,:)) for a batch
Definition: batch_ops.F90:4376
subroutine sbatch_get_state1(this, ist, np, psi, async)
Write a get of state with np points from a batch.
Definition: batch_ops.F90:8765
subroutine sbatch_get_state2(this, index, np, psi, async)
Definition: batch_ops.F90:8879
subroutine zbatch_xpay_vec(np, xx, aa, yy, a_start, a_full)
calculate yy(ist,:) = xx(ist,:) + aa(ist)*yy(ist,:) for a batch
Definition: batch_ops.F90:4020
subroutine sbatch_set_state1(this, ist, np, psi)
Write a single state with np points into a batch at position ist.
Definition: batch_ops.F90:8606
subroutine, public batch_copy_precision(src, dst, np)
Copy the data of a batch to another batch of the same logical shape but the opposite precision (doubl...
Definition: batch_ops.F90:836
subroutine cbatch_get_state3(this, ii, np, psi, async)
Definition: batch_ops.F90:9391
subroutine batch_mul_cj(np, xx, yy, zz, conjugate_yy)
Point-wise multiply two batches with optional conjugation on yy: zz_i = xx_i * CJ(yy_i)
Definition: batch_ops.F90:520
subroutine sbatch_get_state3(this, ii, np, psi, async)
Definition: batch_ops.F90:8897
subroutine dbatch_gemv(np, xx, aa, bb, yy, aa2, bb2, yy2)
Linear combination of batches: yy(ist) = bb(ist) yy(ist) + sum_k aa(k, ist) xx(k)(ist)
Definition: batch_ops.F90:2920
subroutine sbatch_set_state2(this, index, np, psi)
Write a single state with np points into a batch at position defined by index.
Definition: batch_ops.F90:8727
subroutine zbatch_set_state1(this, ist, np, psi)
Write a single state with np points into a batch at position ist.
Definition: batch_ops.F90:3215
subroutine batch_add_with_map_cpu(np, map, xx, yy, zz)
Definition: batch_ops.F90:647
subroutine zbatch_scal_vec(np, aa, xx, a_start, a_full)
Definition: batch_ops.F90:4822
subroutine cbatch_set_state3(this, ii, np, psi)
Write a set of state with np points into a batch.
Definition: batch_ops.F90:9226
subroutine zbatch_set_state2(this, index, np, psi)
Write a single state with np points into a batch at position defined by index.
Definition: batch_ops.F90:3323
subroutine sbatch_axpy_const(np, aa, xx, yy)
This routine applies a 'pair-wise' axpy operation to all functions of the batches xx and yy,...
Definition: batch_ops.F90:8495
subroutine, public zbatch_axpy_function(np, aa, xx, psi, nst)
This routine performs a set of axpy operations for each function x of a batch (xx),...
Definition: batch_ops.F90:3664
subroutine zbatch_set_points(this, sp, ep, psi)
copy a set of points into a mesh function
Definition: batch_ops.F90:4593
subroutine zbatch_add_with_map(np, map, xx, yy, zz)
Definition: batch_ops.F90:4667
subroutine, public dbatch_axpy_function(np, aa, xx, psi, nst)
This routine performs a set of axpy operations for each function x of a batch (xx),...
Definition: batch_ops.F90:1591
subroutine batch_get_points_accel(this, sp, ep, psi, ldpsi1, ldpsi2)
GPU version of batch_get_points.
Definition: batch_ops.F90:375
subroutine cbatch_set_state2(this, index, np, psi)
Write a single state with np points into a batch at position defined by index.
Definition: batch_ops.F90:9208
subroutine zbatch_set_state3(this, ii, np, psi)
Write a set of state with np points into a batch.
Definition: batch_ops.F90:3341
subroutine cbatch_get_state1(this, ist, np, psi, async)
Write a get of state with np points from a batch.
Definition: batch_ops.F90:9246
subroutine batch_copy_with_map_cpu(np, map, xx, yy)
Definition: batch_ops.F90:707
subroutine dbatch_scal_const(np, aa, xx)
scale all functions in a batch by constant aa
Definition: batch_ops.F90:1791
This module contains interfaces for BLAS routines You should not use these routines directly....
Definition: blas.F90:97
real(real64), parameter, public m_zero
Definition: global.F90:173
logical pure function, public not_in_openmp()
Definition: global.F90:573
complex(real64), parameter, public m_z0
Definition: global.F90:183
This module is intended to contain "only mathematical" functions and procedures.
Definition: math.F90:94
subroutine, public messages_not_implemented(feature, namespace)
Definition: messages.F90:1048
character(len=256), dimension(max_lines), public message
to be output by fatal, warning
Definition: messages.F90:138
subroutine, public messages_fatal(no_lines, only_root_writes, namespace)
Definition: messages.F90:386
/* To use the profiling module, call profiling_in and profiling_out with the label of the region as a...
Definition: profiling.F90:122
subroutine, public profiling_out(label)
Increment out counter and sum up difference between entry and exit time.
Definition: profiling.F90:672
subroutine, public profiling_in(label, exclude)
Increment in counter and save entry time.
Definition: profiling.F90:604
type(type_t), parameter, public type_cmplx_single
Definition: types.F90:119
type(type_t), parameter, public type_float_single
Definition: types.F90:118
integer pure function, public types_get_size(this)
Definition: types.F90:134
type(type_t), parameter, public type_integer
Definition: types.F90:115
type(type_t), parameter, public type_float
Definition: types.F90:113
Class defining batches of mesh functions.
Definition: batch.F90:142
int true(void)