Octopus
operate_antisym_inc.c
Go to the documentation of this file.
1/*
2 Copyright (C) 2006 X. Andrade
3 Copyright (C) 2025 N. Tancogne-Dejean
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2, or (at your option)
8 any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 02110-1301, USA.
19
20*/
21
22#ifdef ALIGNED
23#define LOAD VEC_LD
24#ifdef STREAM
25#define STORE VEC_ST
26#else
27#define STORE VEC_STA
28#endif
29#else
30#define LOAD VEC_LDU
31#define STORE VEC_STU
32#endif
33
35 const ptrdiff_t npairs = opn[0];
36 const ptrdiff_t nri = opnri[0];
37#if DEPTH >= 16
38 const ptrdiff_t unroll16 = max1(16 * VEC_SIZE >> ldf);
39#endif
40#if DEPTH >= 8
41 const ptrdiff_t unroll8 = max1(8 * VEC_SIZE >> ldf);
42#endif
43#if DEPTH >= 4
44 const ptrdiff_t unroll4 = max1(4 * VEC_SIZE >> ldf);
45#endif
46#if DEPTH >= 2
47 const ptrdiff_t unroll2 = max1(2 * VEC_SIZE >> ldf);
48#endif
49#if DEPTH >= 1
50 const ptrdiff_t unroll1 = max1(1 * VEC_SIZE >> ldf);
51#endif
52
53 ptrdiff_t l, i, j;
54 const int *restrict index_pos;
55 const int *restrict index_neg;
56
57 for (l = 0; l < nri; l++) {
58 index_pos = opri_pos + npairs * l;
59 index_neg = opri_neg + npairs * l;
60 i = rimap_inv[l];
61
62#if DEPTH >= 16
63 for (; i < (rimap_inv_max[l] - unroll16 + 1); i += unroll16) {
64 ptrdiff_t k;
65 for (k = 0; k < (1 << ldf); k += 16 * VEC_SIZE) {
66 register VEC_TYPE a0, a1, a2, a3;
67 register VEC_TYPE a4, a5, a6, a7;
68 register VEC_TYPE a8, a9, aa, ab;
69 register VEC_TYPE ac, ad, ae, af;
70
71 a0 = a1 = a2 = a3 = VEC_ZERO;
72 a4 = a5 = a6 = a7 = VEC_ZERO;
73 a8 = a9 = aa = ab = VEC_ZERO;
74 ac = ad = ae = af = VEC_ZERO;
75
76 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
77
78 for (j = 0; j < npairs; j++) {
79 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
80 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
81 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
82 register VEC_TYPE pos = LOAD(fi + indexj_pos);
83 register VEC_TYPE neg = LOAD(fi + indexj_neg);
84 a0 = VEC_FMA(wj, VEC_SUB(pos, neg), a0);
85 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
86 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
87 a1 = VEC_FMA(wj, VEC_SUB(pos, neg), a1);
88 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
89 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
90 a2 = VEC_FMA(wj, VEC_SUB(pos, neg), a2);
91 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
92 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
93 a3 = VEC_FMA(wj, VEC_SUB(pos, neg), a3);
94 pos = LOAD(fi + indexj_pos + 4 * VEC_SIZE);
95 neg = LOAD(fi + indexj_neg + 4 * VEC_SIZE);
96 a4 = VEC_FMA(wj, VEC_SUB(pos, neg), a4);
97 pos = LOAD(fi + indexj_pos + 5 * VEC_SIZE);
98 neg = LOAD(fi + indexj_neg + 5 * VEC_SIZE);
99 a5 = VEC_FMA(wj, VEC_SUB(pos, neg), a5);
100 pos = LOAD(fi + indexj_pos + 6 * VEC_SIZE);
101 neg = LOAD(fi + indexj_neg + 6 * VEC_SIZE);
102 a6 = VEC_FMA(wj, VEC_SUB(pos, neg), a6);
103 pos = LOAD(fi + indexj_pos + 7 * VEC_SIZE);
104 neg = LOAD(fi + indexj_neg + 7 * VEC_SIZE);
105 a7 = VEC_FMA(wj, VEC_SUB(pos, neg), a7);
106 pos = LOAD(fi + indexj_pos + 8 * VEC_SIZE);
107 neg = LOAD(fi + indexj_neg + 8 * VEC_SIZE);
108 a8 = VEC_FMA(wj, VEC_SUB(pos, neg), a8);
109 pos = LOAD(fi + indexj_pos + 9 * VEC_SIZE);
110 neg = LOAD(fi + indexj_neg + 9 * VEC_SIZE);
111 a9 = VEC_FMA(wj, VEC_SUB(pos, neg), a9);
112 pos = LOAD(fi + indexj_pos + 10 * VEC_SIZE);
113 neg = LOAD(fi + indexj_neg + 10 * VEC_SIZE);
114 aa = VEC_FMA(wj, VEC_SUB(pos, neg), aa);
115 pos = LOAD(fi + indexj_pos + 11 * VEC_SIZE);
116 neg = LOAD(fi + indexj_neg + 11 * VEC_SIZE);
117 ab = VEC_FMA(wj, VEC_SUB(pos, neg), ab);
118 pos = LOAD(fi + indexj_pos + 12 * VEC_SIZE);
119 neg = LOAD(fi + indexj_neg + 12 * VEC_SIZE);
120 ac = VEC_FMA(wj, VEC_SUB(pos, neg), ac);
121 pos = LOAD(fi + indexj_pos + 13 * VEC_SIZE);
122 neg = LOAD(fi + indexj_neg + 13 * VEC_SIZE);
123 ad = VEC_FMA(wj, VEC_SUB(pos, neg), ad);
124 pos = LOAD(fi + indexj_pos + 14 * VEC_SIZE);
125 neg = LOAD(fi + indexj_neg + 14 * VEC_SIZE);
126 ae = VEC_FMA(wj, VEC_SUB(pos, neg), ae);
127 pos = LOAD(fi + indexj_pos + 15 * VEC_SIZE);
128 neg = LOAD(fi + indexj_neg + 15 * VEC_SIZE);
129 af = VEC_FMA(wj, VEC_SUB(pos, neg), af);
130
131 }
132 STORE(fo + base_ik, a0);
133 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
134 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
135 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
136 STORE(fo + base_ik + 4 * VEC_SIZE, a4);
137 STORE(fo + base_ik + 5 * VEC_SIZE, a5);
138 STORE(fo + base_ik + 6 * VEC_SIZE, a6);
139 STORE(fo + base_ik + 7 * VEC_SIZE, a7);
140 STORE(fo + base_ik + 8 * VEC_SIZE, a8);
141 STORE(fo + base_ik + 9 * VEC_SIZE, a9);
142 STORE(fo + base_ik + 10 * VEC_SIZE, aa);
143 STORE(fo + base_ik + 11 * VEC_SIZE, ab);
144 STORE(fo + base_ik + 12 * VEC_SIZE, ac);
145 STORE(fo + base_ik + 13 * VEC_SIZE, ad);
146 STORE(fo + base_ik + 14 * VEC_SIZE, ae);
147 STORE(fo + base_ik + 15 * VEC_SIZE, af);
148 }
149 }
150#endif
151
152#if DEPTH >= 8
153 for (; i < (rimap_inv_max[l] - unroll8 + 1); i += unroll8) {
154 ptrdiff_t k;
155 for (k = 0; k < (1 << ldf); k += 8 * VEC_SIZE) {
156 register VEC_TYPE a0, a1, a2, a3;
157 register VEC_TYPE a4, a5, a6, a7;
158
159 a0 = a1 = a2 = a3 = VEC_ZERO;
160 a4 = a5 = a6 = a7 = VEC_ZERO;
161
162 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
163
164 for (j = 0; j < npairs; j++) {
165 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
166 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
167 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
168 register VEC_TYPE pos = LOAD(fi + indexj_pos);
169 register VEC_TYPE neg = LOAD(fi + indexj_neg);
170 a0 = VEC_FMA(wj, VEC_SUB(pos, neg), a0);
171 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
172 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
173 a1 = VEC_FMA(wj, VEC_SUB(pos, neg), a1);
174 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
175 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
176 a2 = VEC_FMA(wj, VEC_SUB(pos, neg), a2);
177 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
178 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
179 a3 = VEC_FMA(wj, VEC_SUB(pos, neg), a3);
180 pos = LOAD(fi + indexj_pos + 4 * VEC_SIZE);
181 neg = LOAD(fi + indexj_neg + 4 * VEC_SIZE);
182 a4 = VEC_FMA(wj, VEC_SUB(pos, neg), a4);
183 pos = LOAD(fi + indexj_pos + 5 * VEC_SIZE);
184 neg = LOAD(fi + indexj_neg + 5 * VEC_SIZE);
185 a5 = VEC_FMA(wj, VEC_SUB(pos, neg), a5);
186 pos = LOAD(fi + indexj_pos + 6 * VEC_SIZE);
187 neg = LOAD(fi + indexj_neg + 6 * VEC_SIZE);
188 a6 = VEC_FMA(wj, VEC_SUB(pos, neg), a6);
189 pos = LOAD(fi + indexj_pos + 7 * VEC_SIZE);
190 neg = LOAD(fi + indexj_neg + 7 * VEC_SIZE);
191 a7 = VEC_FMA(wj, VEC_SUB(pos, neg), a7);
192 }
193 STORE(fo + base_ik, a0);
194 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
195 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
196 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
197 STORE(fo + base_ik + 4 * VEC_SIZE, a4);
198 STORE(fo + base_ik + 5 * VEC_SIZE, a5);
199 STORE(fo + base_ik + 6 * VEC_SIZE, a6);
200 STORE(fo + base_ik + 7 * VEC_SIZE, a7);
201 }
202 }
203#endif
204
205#if DEPTH >= 4
206 for (; i < (rimap_inv_max[l] - unroll4 + 1); i += unroll4) {
207 ptrdiff_t k;
208 for (k = 0; k < (1 << ldf); k += 4 * VEC_SIZE) {
209 register VEC_TYPE a0, a1, a2, a3;
210
211 a0 = a1 = a2 = a3 = VEC_ZERO;
212
213 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
214
215 for (j = 0; j < npairs; j++) {
216 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
217 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
218 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
219 register VEC_TYPE pos = LOAD(fi + indexj_pos);
220 register VEC_TYPE neg = LOAD(fi + indexj_neg);
221 a0 = VEC_FMA(wj, VEC_SUB(pos, neg), a0);
222 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
223 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
224 a1 = VEC_FMA(wj, VEC_SUB(pos, neg), a1);
225 pos = LOAD(fi + indexj_pos + 2 * VEC_SIZE);
226 neg = LOAD(fi + indexj_neg + 2 * VEC_SIZE);
227 a2 = VEC_FMA(wj, VEC_SUB(pos, neg), a2);
228 pos = LOAD(fi + indexj_pos + 3 * VEC_SIZE);
229 neg = LOAD(fi + indexj_neg + 3 * VEC_SIZE);
230 a3 = VEC_FMA(wj, VEC_SUB(pos, neg), a3);
231 }
232 STORE(fo + base_ik, a0);
233 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
234 STORE(fo + base_ik + 2 * VEC_SIZE, a2);
235 STORE(fo + base_ik + 3 * VEC_SIZE, a3);
236 }
237 }
238#endif
239
240#if DEPTH >= 2
241 for (; i < (rimap_inv_max[l] - unroll2 + 1); i += unroll2) {
242 ptrdiff_t k;
243 for (k = 0; k < (1 << ldf); k += 2 * VEC_SIZE) {
244 register VEC_TYPE a0, a1;
245
246 a0 = a1 = VEC_ZERO;
247
248 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
249
250 for (j = 0; j < npairs; j++) {
251 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
252 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
253 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
254 register VEC_TYPE pos = LOAD(fi + indexj_pos);
255 register VEC_TYPE neg = LOAD(fi + indexj_neg);
256 a0 = VEC_FMA(wj, VEC_SUB(pos, neg), a0);
257 pos = LOAD(fi + indexj_pos + 1 * VEC_SIZE);
258 neg = LOAD(fi + indexj_neg + 1 * VEC_SIZE);
259 a1 = VEC_FMA(wj, VEC_SUB(pos, neg), a1);
260 }
261 STORE(fo + base_ik, a0);
262 STORE(fo + base_ik + 1 * VEC_SIZE, a1);
263 }
264 }
265#endif
266
267#if DEPTH >= 1
268 for (; i < (rimap_inv_max[l] - unroll1 + 1); i += unroll1) {
269 ptrdiff_t k;
270 for (k = 0; k < (1 << ldf); k += VEC_SIZE) {
271 register VEC_TYPE a0 = VEC_ZERO;
272 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
273 for (j = 0; j < npairs; j++) {
274 register VEC_TYPE wj = VEC_SCAL(wpair[j]);
275 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
276 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
277 register VEC_TYPE pos = LOAD(fi + indexj_pos);
278 register VEC_TYPE neg = LOAD(fi + indexj_neg);
279 a0 = VEC_FMA(wj, VEC_SUB(pos, neg), a0);
280 }
281 STORE(fo + base_ik, a0);
282 }
283 }
284#endif
285
286#if VEC_SIZE > 1
287
288 const ptrdiff_t size = (ptrdiff_t) 1 << ldf;
289 double a;
290
291 for (; i < rimap_inv_max[l]; i++) {
292 for (ptrdiff_t k = 0; k < size; k++) {
293 a = 0.0;
294 const ptrdiff_t base_ik = (ptrdiff_t) (i << ldf) + k;
295 for (j = 0; j < npairs; j++) {
296 const ptrdiff_t indexj_pos = index_pos[j] + base_ik;
297 const ptrdiff_t indexj_neg = index_neg[j] + base_ik;
298 const double diff = fi[indexj_pos] - fi[indexj_neg];
299 a += wpair[j] * diff;
300 }
301 fo[base_ik] = a;
302 }
303 }
304#endif
305
306 } /* l */
307
308 // this fence instruction is needed to ensure correctness when using
309 // non-temporal stores
310#if defined(ALIGNED) && defined(STREAM) && defined(FENCE)
311 FENCE;
312#endif
313}
314
315#undef LOAD
316#undef STORE
long int ptrdiff_t
Definition: operate.c:103
const int *restrict index_neg
const ptrdiff_t nri
ptrdiff_t l
ptrdiff_t i
ptrdiff_t j
const int *restrict index_pos