Octopus
operate_inc.c
Go to the documentation of this file.
1/*
2 Copyright (C) 2006 X. Andrade
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation; either version 2, or (at your option)
7 any later version.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17 02110-1301, USA.
18
19*/
20
21#ifdef ALIGNED
22#define LOAD VEC_LD
23#ifdef STREAM
24#define STORE VEC_ST
25#else
26#define STORE VEC_STA
27#endif
28#else
29#define LOAD VEC_LDU
30#define STORE VEC_STU
31#endif
33{
34 const ptrdiff_t n = opn[0];
35 const ptrdiff_t nri = opnri[0];
36#if DEPTH >= 16
37 const ptrdiff_t unroll16 = max1(16 * VEC_SIZE >> ldf);
38#endif
39#if DEPTH >= 8
40 const ptrdiff_t unroll8 = max1(8 * VEC_SIZE >> ldf);
41#endif
42#if DEPTH >= 4
43 const ptrdiff_t unroll4 = max1(4 * VEC_SIZE >> ldf);
44#endif
45#if DEPTH >= 2
46 const ptrdiff_t unroll2 = max1(2 * VEC_SIZE >> ldf);
47#endif
48#if DEPTH >= 1
49 const ptrdiff_t unroll1 = max1(1 * VEC_SIZE >> ldf);
50#endif
51
52 ptrdiff_t l, i, j;
53 const int *restrict index;
54
55 for (l = 0; l < nri; l++) {
56 index = opri + n * l;
57 i = rimap_inv[l];
58
59#if DEPTH >= 16
60 for (; i < (rimap_inv_max[l] - unroll16 + 1); i += unroll16) {
61 ptrdiff_t k;
62 for (k = 0; k < (1 << ldf); k += 16 * VEC_SIZE) {
63 register VEC_TYPE a0, a1, a2, a3;
64 register VEC_TYPE a4, a5, a6, a7;
65 register VEC_TYPE a8, a9, aa, ab;
66 register VEC_TYPE ac, ad, ae, af;
67
68 a0 = a1 = a2 = a3 = VEC_ZERO;
69 a4 = a5 = a6 = a7 = VEC_ZERO;
70 a8 = a9 = aa = ab = VEC_ZERO;
71 ac = ad = ae = af = VEC_ZERO;
72
73 for (j = 0; j < n; j++) {
74 register VEC_TYPE wj = VEC_SCAL(w[j]);
75 ptrdiff_t indexj = (index[j] + i) << ldf;
76 a0 = VEC_FMA(wj, LOAD(fi + indexj + k), a0);
77 a1 = VEC_FMA(wj, LOAD(fi + indexj + 1 * VEC_SIZE + k), a1);
78 a2 = VEC_FMA(wj, LOAD(fi + indexj + 2 * VEC_SIZE + k), a2);
79 a3 = VEC_FMA(wj, LOAD(fi + indexj + 3 * VEC_SIZE + k), a3);
80 a4 = VEC_FMA(wj, LOAD(fi + indexj + 4 * VEC_SIZE + k), a4);
81 a5 = VEC_FMA(wj, LOAD(fi + indexj + 5 * VEC_SIZE + k), a5);
82 a6 = VEC_FMA(wj, LOAD(fi + indexj + 6 * VEC_SIZE + k), a6);
83 a7 = VEC_FMA(wj, LOAD(fi + indexj + 7 * VEC_SIZE + k), a7);
84 a8 = VEC_FMA(wj, LOAD(fi + indexj + 8 * VEC_SIZE + k), a8);
85 a9 = VEC_FMA(wj, LOAD(fi + indexj + 9 * VEC_SIZE + k), a9);
86 aa = VEC_FMA(wj, LOAD(fi + indexj + 10 * VEC_SIZE + k), aa);
87 ab = VEC_FMA(wj, LOAD(fi + indexj + 11 * VEC_SIZE + k), ab);
88 ac = VEC_FMA(wj, LOAD(fi + indexj + 12 * VEC_SIZE + k), ac);
89 ad = VEC_FMA(wj, LOAD(fi + indexj + 13 * VEC_SIZE + k), ad);
90 ae = VEC_FMA(wj, LOAD(fi + indexj + 14 * VEC_SIZE + k), ae);
91 af = VEC_FMA(wj, LOAD(fi + indexj + 15 * VEC_SIZE + k), af);
92 }
93 STORE(fo + (i << ldf) + k, a0);
94 STORE(fo + (i << ldf) + 1 * VEC_SIZE + k, a1);
95 STORE(fo + (i << ldf) + 2 * VEC_SIZE + k, a2);
96 STORE(fo + (i << ldf) + 3 * VEC_SIZE + k, a3);
97 STORE(fo + (i << ldf) + 4 * VEC_SIZE + k, a4);
98 STORE(fo + (i << ldf) + 5 * VEC_SIZE + k, a5);
99 STORE(fo + (i << ldf) + 6 * VEC_SIZE + k, a6);
100 STORE(fo + (i << ldf) + 7 * VEC_SIZE + k, a7);
101 STORE(fo + (i << ldf) + 8 * VEC_SIZE + k, a8);
102 STORE(fo + (i << ldf) + 9 * VEC_SIZE + k, a9);
103 STORE(fo + (i << ldf) + 10 * VEC_SIZE + k, aa);
104 STORE(fo + (i << ldf) + 11 * VEC_SIZE + k, ab);
105 STORE(fo + (i << ldf) + 12 * VEC_SIZE + k, ac);
106 STORE(fo + (i << ldf) + 13 * VEC_SIZE + k, ad);
107 STORE(fo + (i << ldf) + 14 * VEC_SIZE + k, ae);
108 STORE(fo + (i << ldf) + 15 * VEC_SIZE + k, af);
109 }
110 }
111#endif
112
113#if DEPTH >= 8
114 for (; i < (rimap_inv_max[l] - unroll8 + 1); i += unroll8) {
115 ptrdiff_t k;
116 for (k = 0; k < (1 << ldf); k += 8 * VEC_SIZE) {
117 register VEC_TYPE a0, a1, a2, a3;
118 register VEC_TYPE a4, a5, a6, a7;
119
120 a0 = a1 = a2 = a3 = VEC_ZERO;
121 a4 = a5 = a6 = a7 = VEC_ZERO;
122
123 for (j = 0; j < n; j++) {
124 register VEC_TYPE wj = VEC_SCAL(w[j]);
125 ptrdiff_t indexj = (index[j] + i) << ldf;
126 a0 = VEC_FMA(wj, LOAD(fi + indexj + k), a0);
127 a1 = VEC_FMA(wj, LOAD(fi + indexj + 1 * VEC_SIZE + k), a1);
128 a2 = VEC_FMA(wj, LOAD(fi + indexj + 2 * VEC_SIZE + k), a2);
129 a3 = VEC_FMA(wj, LOAD(fi + indexj + 3 * VEC_SIZE + k), a3);
130 a4 = VEC_FMA(wj, LOAD(fi + indexj + 4 * VEC_SIZE + k), a4);
131 a5 = VEC_FMA(wj, LOAD(fi + indexj + 5 * VEC_SIZE + k), a5);
132 a6 = VEC_FMA(wj, LOAD(fi + indexj + 6 * VEC_SIZE + k), a6);
133 a7 = VEC_FMA(wj, LOAD(fi + indexj + 7 * VEC_SIZE + k), a7);
134 }
135 STORE(fo + (i << ldf) + k, a0);
136 STORE(fo + (i << ldf) + 1 * VEC_SIZE + k, a1);
137 STORE(fo + (i << ldf) + 2 * VEC_SIZE + k, a2);
138 STORE(fo + (i << ldf) + 3 * VEC_SIZE + k, a3);
139 STORE(fo + (i << ldf) + 4 * VEC_SIZE + k, a4);
140 STORE(fo + (i << ldf) + 5 * VEC_SIZE + k, a5);
141 STORE(fo + (i << ldf) + 6 * VEC_SIZE + k, a6);
142 STORE(fo + (i << ldf) + 7 * VEC_SIZE + k, a7);
143 }
144 }
145#endif
146
147#if DEPTH >= 4
148 for (; i < (rimap_inv_max[l] - unroll4 + 1); i += unroll4) {
149 ptrdiff_t k;
150 for (k = 0; k < (1 << ldf); k += 4 * VEC_SIZE) {
151 register VEC_TYPE a0, a1, a2, a3;
152
153 a0 = a1 = a2 = a3 = VEC_ZERO;
154
155 for (j = 0; j < n; j++) {
156 register VEC_TYPE wj = VEC_SCAL(w[j]);
157 ptrdiff_t indexj = (index[j] + i) << ldf;
158 a0 = VEC_FMA(wj, LOAD(fi + indexj + k), a0);
159 a1 = VEC_FMA(wj, LOAD(fi + indexj + 1 * VEC_SIZE + k), a1);
160 a2 = VEC_FMA(wj, LOAD(fi + indexj + 2 * VEC_SIZE + k), a2);
161 a3 = VEC_FMA(wj, LOAD(fi + indexj + 3 * VEC_SIZE + k), a3);
162 }
163 STORE(fo + (i << ldf) + k, a0);
164 STORE(fo + (i << ldf) + 1 * VEC_SIZE + k, a1);
165 STORE(fo + (i << ldf) + 2 * VEC_SIZE + k, a2);
166 STORE(fo + (i << ldf) + 3 * VEC_SIZE + k, a3);
167 }
168 }
169#endif
170
171#if DEPTH >= 2
172 for (; i < (rimap_inv_max[l] - unroll2 + 1); i += unroll2) {
173 ptrdiff_t k;
174 for (k = 0; k < (1 << ldf); k += 2 * VEC_SIZE) {
175 register VEC_TYPE a0, a1;
176
177 a0 = a1 = VEC_ZERO;
178
179 for (j = 0; j < n; j++) {
180 register VEC_TYPE wj = VEC_SCAL(w[j]);
181 ptrdiff_t indexj = (index[j] + i) << ldf;
182 a0 = VEC_FMA(wj, LOAD(fi + indexj + k), a0);
183 a1 = VEC_FMA(wj, LOAD(fi + indexj + 1 * VEC_SIZE + k), a1);
184 }
185 STORE(fo + (i << ldf) + k, a0);
186 STORE(fo + (i << ldf) + 1 * VEC_SIZE + k, a1);
187 }
188 }
189#endif
190
191#if DEPTH >= 1
192 for (; i < (rimap_inv_max[l] - unroll1 + 1); i += unroll1) {
193 ptrdiff_t k;
194 for (k = 0; k < (1 << ldf); k += VEC_SIZE) {
195 register VEC_TYPE a0 = VEC_ZERO;
196 for (j = 0; j < n; j++) {
197 register VEC_TYPE wj = VEC_SCAL(w[j]);
198 ptrdiff_t indexj = (index[j] + i) << ldf;
199 a0 = VEC_FMA(wj, LOAD(fi + indexj + k), a0);
200 }
201 STORE(fo + (i << ldf) + k, a0);
202 }
203 }
204#endif
205
206#if VEC_SIZE > 1
207 for (; i < rimap_inv_max[l]; i++) {
208 ptrdiff_t k;
209 for (k = 0; k < (1 << ldf); k++) {
210 double a = 0.0;
211 for (j = 0; j < n; j++)
212 a += w[j] * fi[((index[j] + i) << ldf) + k];
213 fo[(i << ldf) + k] = a;
214 }
215 }
216#endif
217
218 } /* l */
219
220 // this fence instruction is needed to ensure correctness when using
221 // non-temporal stores
222#if defined(ALIGNED) && defined(STREAM) && defined(FENCE)
223 FENCE;
224#endif
225}
226
227#undef LOAD
228#undef STORE
long int ptrdiff_t
Definition: operate.c:103
const ptrdiff_t nri
Definition: operate_inc.c:27
ptrdiff_t l
Definition: operate_inc.c:29
ptrdiff_t i
Definition: operate_inc.c:29
ptrdiff_t j
Definition: operate_inc.c:29
const int *restrict index
Definition: operate_inc.c:30