Octopus
operate_sym_inc.c
Go to the documentation of this file.
1/*
2 Copyright (C) 2006 X. Andrade
3 Copyright (C) 2025 N. Tancogne-Dejean
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
8 any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 02110-1301, USA.
19
20*/
21
22#ifdef ALIGNED
23#define LOAD VEC_LD
24#ifdef STREAM
25#define STORE VEC_ST
26#else
27#define STORE VEC_STA
28#endif
29#else
30#define LOAD VEC_LDU
31#define STORE VEC_STU
32#endif
33
34{
35 const ptrdiff_t npairs = opn[0];
36 const ptrdiff_t nri = opnri[0];
37#if DEPTH >= 16
38 const ptrdiff_t unroll16 = max1(16 * VEC_SIZE >> ldf);
39#endif
40#if DEPTH >= 8
41 const ptrdiff_t unroll8 = max1(8 * VEC_SIZE >> ldf);
42#endif
43#if DEPTH >= 4
44 const ptrdiff_t unroll4 = max1(4 * VEC_SIZE >> ldf);
45#endif
46#if DEPTH >= 2
47 const ptrdiff_t unroll2 = max1(2 * VEC_SIZE >> ldf);
48#endif
49#if DEPTH >= 1
50 const ptrdiff_t unroll1 = max1(1 * VEC_SIZE >> ldf);
51#endif
52
53 ptrdiff_t l, i, j;
54 const int *restrict index_pos;
55 const int *restrict index_neg;
56
57 // Hoisting wcenter
58 const double w0 = wcenter[0];
59
60 for (l = 0; l < nri; l++) {
61 index_pos = opri_pos + npairs * l;
62 index_neg = opri_neg + npairs * l;
63 i = rimap_inv[l];
64
65#if DEPTH >= 16
66 for (; i < (rimap_inv_max[l] - unroll16 + 1); i += unroll16) {
67 ptrdiff_t k;
68 for (k = 0; k < (1 << ldf); k += 16 * VEC_SIZE) {
69 register VEC_TYPE a0, a1, a2, a3;
70 register VEC_TYPE a4, a5, a6, a7;
71 register VEC_TYPE a8, a9, aa, ab;
72 register VEC_TYPE ac, ad, ae, af;
73
74 a0 = a1 = a2 = a3 = VEC_ZERO;
75 a4 = a5 = a6 = a7 = VEC_ZERO;
76 a8 = a9 = aa = ab = VEC_ZERO;
77 ac = ad = ae = af = VEC_ZERO;
78
79 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
80
81 for (j = 0; j < npairs; j++) {
82 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
83 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
84 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
85 register VEC_TYPE pos = LOAD(fi + indexj_pos);
86 register VEC_TYPE neg = LOAD(fi + indexj_neg);
87 a0 = VEC_FMA(wj, VEC_ADD(pos, neg), a0);
88 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
89 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
90 a1 = VEC_FMA(wj, VEC_ADD(pos, neg), a1);
91 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
92 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
93 a2 = VEC_FMA(wj, VEC_ADD(pos, neg), a2);
94 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
95 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
96 a3 = VEC_FMA(wj, VEC_ADD(pos, neg), a3);
97 pos = LOAD(fi + indexj_pos + 4 * VEC_SIZE);
98 neg = LOAD(fi + indexj_neg + 4 * VEC_SIZE);
99 a4 = VEC_FMA(wj, VEC_ADD(pos, neg), a4);
100 pos = LOAD(fi + indexj_pos + 5 * VEC_SIZE);
101 neg = LOAD(fi + indexj_neg + 5 * VEC_SIZE);
102 a5 = VEC_FMA(wj, VEC_ADD(pos, neg), a5);
103 pos = LOAD(fi + indexj_pos + 6 * VEC_SIZE);
104 neg = LOAD(fi + indexj_neg + 6 * VEC_SIZE);
105 a6 = VEC_FMA(wj, VEC_ADD(pos, neg), a6);
106 pos = LOAD(fi + indexj_pos + 7 * VEC_SIZE);
107 neg = LOAD(fi + indexj_neg + 7 * VEC_SIZE);
108 a7 = VEC_FMA(wj, VEC_ADD(pos, neg), a7);
109 pos = LOAD(fi + indexj_pos + 8 * VEC_SIZE);
110 neg = LOAD(fi + indexj_neg + 8 * VEC_SIZE);
111 a8 = VEC_FMA(wj, VEC_ADD(pos, neg), a8);
112 pos = LOAD(fi + indexj_pos + 9 * VEC_SIZE);
113 neg = LOAD(fi + indexj_neg + 9 * VEC_SIZE);
114 a9 = VEC_FMA(wj, VEC_ADD(pos, neg), a9);
115 pos = LOAD(fi + indexj_pos + 10 * VEC_SIZE);
116 neg = LOAD(fi + indexj_neg + 10 * VEC_SIZE);
117 aa = VEC_FMA(wj, VEC_ADD(pos, neg), aa);
118 pos = LOAD(fi + indexj_pos + 11 * VEC_SIZE);
119 neg = LOAD(fi + indexj_neg + 11 * VEC_SIZE);
120 ab = VEC_FMA(wj, VEC_ADD(pos, neg), ab);
121 pos = LOAD(fi + indexj_pos + 12 * VEC_SIZE);
122 neg = LOAD(fi + indexj_neg + 12 * VEC_SIZE);
123 ac = VEC_FMA(wj, VEC_ADD(pos, neg), ac);
124 pos = LOAD(fi + indexj_pos + 13 * VEC_SIZE);
125 neg = LOAD(fi + indexj_neg + 13 * VEC_SIZE);
126 ad = VEC_FMA(wj, VEC_ADD(pos, neg), ad);
127 pos = LOAD(fi + indexj_pos + 14 * VEC_SIZE);
128 neg = LOAD(fi + indexj_neg + 14 * VEC_SIZE);
129 ae = VEC_FMA(wj, VEC_ADD(pos, neg), ae);
130 pos = LOAD(fi + indexj_pos + 15 * VEC_SIZE);
131 neg = LOAD(fi + indexj_neg + 15 * VEC_SIZE);
132 af = VEC_FMA(wj, VEC_ADD(pos, neg), af);
133
134 }
135 const VEC_TYPE wj = VEC_SCAL(w0);
136 a0 = VEC_FMA(wj, LOAD(fi + base_ik), a0);
137 a1 = VEC_FMA(wj, LOAD(fi + base_ik + 1 * VEC_SIZE), a1);
138 a2 = VEC_FMA(wj, LOAD(fi + base_ik + 2 * VEC_SIZE), a2);
139 a3 = VEC_FMA(wj, LOAD(fi + base_ik + 3 * VEC_SIZE), a3);
140 a4 = VEC_FMA(wj, LOAD(fi + base_ik + 4 * VEC_SIZE), a4);
141 a5 = VEC_FMA(wj, LOAD(fi + base_ik + 5 * VEC_SIZE), a5);
142 a6 = VEC_FMA(wj, LOAD(fi + base_ik + 6 * VEC_SIZE), a6);
143 a7 = VEC_FMA(wj, LOAD(fi + base_ik + 7 * VEC_SIZE), a7);
144 a8 = VEC_FMA(wj, LOAD(fi + base_ik + 8 * VEC_SIZE), a8);
145 a9 = VEC_FMA(wj, LOAD(fi + base_ik + 9 * VEC_SIZE), a9);
146 aa = VEC_FMA(wj, LOAD(fi + base_ik + 10 * VEC_SIZE), aa);
147 ab = VEC_FMA(wj, LOAD(fi + base_ik + 11 * VEC_SIZE), ab);
148 ac = VEC_FMA(wj, LOAD(fi + base_ik + 12 * VEC_SIZE), ac);
149 ad = VEC_FMA(wj, LOAD(fi + base_ik + 13 * VEC_SIZE), ad);
150 ae = VEC_FMA(wj, LOAD(fi + base_ik + 14 * VEC_SIZE), ae);
151 af = VEC_FMA(wj, LOAD(fi + base_ik + 15 * VEC_SIZE), af);
152
153 STORE(fo + base_ik, a0);
154 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
155 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
156 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
157 STORE(fo + base_ik + 4 * VEC_SIZE, a4);
158 STORE(fo + base_ik + 5 * VEC_SIZE, a5);
159 STORE(fo + base_ik + 6 * VEC_SIZE, a6);
160 STORE(fo + base_ik + 7 * VEC_SIZE, a7);
161 STORE(fo + base_ik + 8 * VEC_SIZE, a8);
162 STORE(fo + base_ik + 9 * VEC_SIZE, a9);
163 STORE(fo + base_ik + 10 * VEC_SIZE, aa);
164 STORE(fo + base_ik + 11 * VEC_SIZE, ab);
165 STORE(fo + base_ik + 12 * VEC_SIZE, ac);
166 STORE(fo + base_ik + 13 * VEC_SIZE, ad);
167 STORE(fo + base_ik + 14 * VEC_SIZE, ae);
168 STORE(fo + base_ik + 15 * VEC_SIZE, af);
169 }
170 }
171#endif
172
173#if DEPTH >= 8
174 for (; i < (rimap_inv_max[l] - unroll8 + 1); i += unroll8) {
175 ptrdiff_t k;
176 for (k = 0; k < (1 << ldf); k += 8 * VEC_SIZE) {
177 register VEC_TYPE a0, a1, a2, a3;
178 register VEC_TYPE a4, a5, a6, a7;
179
180 a0 = a1 = a2 = a3 = VEC_ZERO;
181 a4 = a5 = a6 = a7 = VEC_ZERO;
182
183 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
184
185 for (j = 0; j < npairs; j++) {
186 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
187 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
188 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
189 register VEC_TYPE pos = LOAD(fi + indexj_pos);
190 register VEC_TYPE neg = LOAD(fi + indexj_neg);
191 a0 = VEC_FMA(wj, VEC_ADD(pos, neg), a0);
192 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
193 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
194 a1 = VEC_FMA(wj, VEC_ADD(pos, neg), a1);
195 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
196 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
197 a2 = VEC_FMA(wj, VEC_ADD(pos, neg), a2);
198 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
199 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
200 a3 = VEC_FMA(wj, VEC_ADD(pos, neg), a3);
201 pos = LOAD(fi + indexj_pos + 4 * VEC_SIZE);
202 neg = LOAD(fi + indexj_neg + 4 * VEC_SIZE);
203 a4 = VEC_FMA(wj, VEC_ADD(pos, neg), a4);
204 pos = LOAD(fi + indexj_pos + 5 * VEC_SIZE);
205 neg = LOAD(fi + indexj_neg + 5 * VEC_SIZE);
206 a5 = VEC_FMA(wj, VEC_ADD(pos, neg), a5);
207 pos = LOAD(fi + indexj_pos + 6 * VEC_SIZE);
208 neg = LOAD(fi + indexj_neg + 6 * VEC_SIZE);
209 a6 = VEC_FMA(wj, VEC_ADD(pos, neg), a6);
210 pos = LOAD(fi + indexj_pos + 7 * VEC_SIZE);
211 neg = LOAD(fi + indexj_neg + 7 * VEC_SIZE);
212 a7 = VEC_FMA(wj, VEC_ADD(pos, neg), a7);
213 }
214 const VEC_TYPE wj = VEC_SCAL(w0);
215 a0 = VEC_FMA(wj, LOAD(fi + base_ik), a0);
216 a1 = VEC_FMA(wj, LOAD(fi + base_ik + 1 * VEC_SIZE), a1);
217 a2 = VEC_FMA(wj, LOAD(fi + base_ik + 2 * VEC_SIZE), a2);
218 a3 = VEC_FMA(wj, LOAD(fi + base_ik + 3 * VEC_SIZE), a3);
219 a4 = VEC_FMA(wj, LOAD(fi + base_ik + 4 * VEC_SIZE), a4);
220 a5 = VEC_FMA(wj, LOAD(fi + base_ik + 5 * VEC_SIZE), a5);
221 a6 = VEC_FMA(wj, LOAD(fi + base_ik + 6 * VEC_SIZE), a6);
222 a7 = VEC_FMA(wj, LOAD(fi + base_ik + 7 * VEC_SIZE), a7);
223 STORE(fo + base_ik, a0);
224 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
225 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
226 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
227 STORE(fo + base_ik + 4 * VEC_SIZE, a4);
228 STORE(fo + base_ik + 5 * VEC_SIZE, a5);
229 STORE(fo + base_ik + 6 * VEC_SIZE, a6);
230 STORE(fo + base_ik + 7 * VEC_SIZE, a7);
231 }
232 }
233#endif
234
235#if DEPTH >= 4
236 for (; i < (rimap_inv_max[l] - unroll4 + 1); i += unroll4) {
237 ptrdiff_t k;
238 for (k = 0; k < (1 << ldf); k += 4 * VEC_SIZE) {
239 register VEC_TYPE a0, a1, a2, a3;
240
241 a0 = a1 = a2 = a3 = VEC_ZERO;
242
243 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
244
245 for (j = 0; j < npairs; j++) {
246 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
247 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
248 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
249 register VEC_TYPE pos = LOAD(fi + indexj_pos);
250 register VEC_TYPE neg = LOAD(fi + indexj_neg);
251 a0 = VEC_FMA(wj, VEC_ADD(pos, neg), a0);
252 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
253 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
254 a1 = VEC_FMA(wj, VEC_ADD(pos, neg), a1);
255 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
256 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
257 a2 = VEC_FMA(wj, VEC_ADD(pos, neg), a2);
258 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
259 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
260 a3 = VEC_FMA(wj, VEC_ADD(pos, neg), a3);
261 }
262 const VEC_TYPE wj = VEC_SCAL(w0);
263 a0 = VEC_FMA(wj, LOAD(fi + base_ik), a0);
264 a1 = VEC_FMA(wj, LOAD(fi + base_ik + 1 * VEC_SIZE), a1);
265 a2 = VEC_FMA(wj, LOAD(fi + base_ik + 2 * VEC_SIZE), a2);
266 a3 = VEC_FMA(wj, LOAD(fi + base_ik + 3 * VEC_SIZE), a3);
267 STORE(fo + base_ik, a0);
268 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
269 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
270 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
271 }
272 }
273#endif
274
275#if DEPTH >= 2
276 for (; i < (rimap_inv_max[l] - unroll2 + 1); i += unroll2) {
277 ptrdiff_t k;
278 for (k = 0; k < (1 << ldf); k += 2 * VEC_SIZE) {
279 register VEC_TYPE a0, a1;
280
281 a0 = a1 = VEC_ZERO;
282
283 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
284
285 for (j = 0; j < npairs; j++) {
286 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
287 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
288 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
289 register VEC_TYPE pos = LOAD(fi + indexj_pos);
290 register VEC_TYPE neg = LOAD(fi + indexj_neg);
291 a0 = VEC_FMA(wj, VEC_ADD(pos, neg), a0);
292 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
293 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
294 a1 = VEC_FMA(wj, VEC_ADD(pos, neg), a1);
295 }
296 const VEC_TYPE wj = VEC_SCAL(w0);
297 a0 = VEC_FMA(wj, LOAD(fi + base_ik), a0);
298 a1 = VEC_FMA(wj, LOAD(fi + base_ik + 1 * VEC_SIZE), a1);
299 STORE(fo + base_ik, a0);
300 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
301 }
302 }
303#endif
304
305#if DEPTH >= 1
306 for (; i < (rimap_inv_max[l] - unroll1 + 1); i += unroll1) {
307 ptrdiff_t k;
308 for (k = 0; k < (1 << ldf); k += VEC_SIZE) {
309 register VEC_TYPE a0 = VEC_ZERO;
310 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
311 for (j = 0; j < npairs; j++) {
312 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
313 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
314 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
315 register VEC_TYPE pos = LOAD(fi + indexj_pos);
316 register VEC_TYPE neg = LOAD(fi + indexj_neg);
317 a0 = VEC_FMA(wj, VEC_ADD(pos, neg), a0);
318 }
319 const VEC_TYPE wj = VEC_SCAL(w0);
320 a0 = VEC_FMA(wj, LOAD(fi + base_ik), a0);
321 STORE(fo + base_ik, a0);
322 }
323 }
324#endif
325
326#if VEC_SIZE > 1
327
328 const ptrdiff_t size = (ptrdiff_t) 1 << ldf;
329 double a;
330
331 for (; i < rimap_inv_max[l]; i++) {
332 for (ptrdiff_t k = 0; k < size; k++) {
333 a = 0.0;
334 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
335 for (j = 0; j < npairs; j++) {
336 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
337 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
338 const double sum = fi[indexj_pos] + fi[indexj_neg];
339 a += wpair[j] * sum;
340 }
341 fo[base_ik] = a + w0 * fi[base_ik];
342 }
343 }
344#endif
345
346 } /* l */
347
348 // this fence instruction is needed to ensure correctness when using
349 // non-temporal stores
350#if defined(ALIGNED) && defined(STREAM) && defined(FENCE)
351 FENCE;
352#endif
353}
354
355#undef LOAD
356#undef STORE
long int ptrdiff_t
Definition: operate.c:103
const int *restrict index_neg
const ptrdiff_t nri
const double w0
ptrdiff_t l
ptrdiff_t i
ptrdiff_t j
const int *restrict index_pos