10 #ifndef EIGEN_PACKET_MATH_ALTIVEC_H 11 #define EIGEN_PACKET_MATH_ALTIVEC_H 17 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 18 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4 21 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD 22 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD 25 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD 26 #define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD 30 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 31 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 34 typedef __vector
float Packet4f;
35 typedef __vector
int Packet4i;
36 typedef __vector
unsigned int Packet4ui;
37 typedef __vector __bool
int Packet4bi;
38 typedef __vector
short int Packet8i;
39 typedef __vector
unsigned char Packet16uc;
44 #define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \ 45 Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X)) 47 #define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \ 48 Packet4i p4i_##NAME = vec_splat_s32(X) 50 #define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \ 51 Packet4f p4f_##NAME = pset1<Packet4f>(X) 53 #define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \ 54 Packet4i p4i_##NAME = pset1<Packet4i>(X) 56 #define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \ 57 Packet2d p2d_##NAME = pset1<Packet2d>(X) 59 #define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \ 60 Packet2l p2l_##NAME = pset1<Packet2l>(X) 62 #define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \ 63 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X)) 66 #define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride)) 70 static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0);
71 static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0);
72 static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1);
73 static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16);
74 static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1);
75 static Packet4f p4f_MZERO = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1);
77 static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0);
80 static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
81 static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
83 static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
84 static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
88 #define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0 90 #define _EIGEN_MASK_ALIGNMENT 0xfffffff0 93 #define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT) 98 static Packet16uc p16uc_FORWARD = vec_lvsl(0, (
float*)0);
100 static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
102 static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);
103 static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);
104 static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8);
106 static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
107 static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
108 static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);
109 static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);
110 static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8);
111 #endif // _BIG_ENDIAN 113 static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
114 static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
115 static Packet16uc p16uc_TRANSPOSE64_HI = p16uc_PSET64_HI + p16uc_HALF64_0_16;
116 static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16;
118 static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8);
121 static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8);
123 static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8);
124 #endif // _BIG_ENDIAN 126 #if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC 127 #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR); 129 #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" ); 132 template<>
struct packet_traits<float> : default_packet_traits
134 typedef Packet4f type;
135 typedef Packet4f half;
155 #if !EIGEN_COMP_CLANG 171 template<>
struct packet_traits<int> : default_packet_traits
173 typedef Packet4i type;
174 typedef Packet4i half;
190 template<>
struct unpacket_traits<Packet4f> {
typedef float type;
enum {size=4, alignment=
Aligned16};
typedef Packet4f half; };
191 template<>
struct unpacket_traits<Packet4i> {
typedef int type;
enum {size=4, alignment=
Aligned16};
typedef Packet4i half; };
193 inline std::ostream & operator <<(std::ostream & s,
const Packet16uc & v)
200 for (
int i=0; i< 16; i++)
201 s << (
int)vt.n[i] <<
", ";
205 inline std::ostream & operator <<(std::ostream & s,
const Packet4f & v)
212 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
216 inline std::ostream & operator <<(std::ostream & s,
const Packet4i & v)
223 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
227 inline std::ostream & operator <<(std::ostream & s,
const Packet4ui & v)
234 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
239 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from)
241 EIGEN_DEBUG_ALIGNED_LOAD
243 return vec_vsx_ld(0, from);
245 return vec_ld(0, from);
249 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from)
251 EIGEN_DEBUG_ALIGNED_LOAD
253 return vec_vsx_ld(0, from);
255 return vec_ld(0, from);
259 template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from)
261 EIGEN_DEBUG_ALIGNED_STORE
263 vec_vsx_st(from, 0, to);
269 template<> EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from)
271 EIGEN_DEBUG_ALIGNED_STORE
273 vec_vsx_st(from, 0, to);
279 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
280 Packet4f v = {from, from, from, from};
284 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
285 Packet4i v = {from, from, from, from};
288 template<> EIGEN_STRONG_INLINE
void 289 pbroadcast4<Packet4f>(
const float *a,
290 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
292 a3 = pload<Packet4f>(a);
293 a0 = vec_splat(a3, 0);
294 a1 = vec_splat(a3, 1);
295 a2 = vec_splat(a3, 2);
296 a3 = vec_splat(a3, 3);
298 template<> EIGEN_STRONG_INLINE
void 299 pbroadcast4<Packet4i>(
const int *a,
300 Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3)
302 a3 = pload<Packet4i>(a);
303 a0 = vec_splat(a3, 0);
304 a1 = vec_splat(a3, 1);
305 a2 = vec_splat(a3, 2);
306 a3 = vec_splat(a3, 3);
309 template<> EIGEN_DEVICE_FUNC
inline Packet4f pgather<float, Packet4f>(
const float* from,
Index stride)
311 float EIGEN_ALIGN16 af[4];
312 af[0] = from[0*stride];
313 af[1] = from[1*stride];
314 af[2] = from[2*stride];
315 af[3] = from[3*stride];
316 return pload<Packet4f>(af);
318 template<> EIGEN_DEVICE_FUNC
inline Packet4i pgather<int, Packet4i>(
const int* from,
Index stride)
320 int EIGEN_ALIGN16 ai[4];
321 ai[0] = from[0*stride];
322 ai[1] = from[1*stride];
323 ai[2] = from[2*stride];
324 ai[3] = from[3*stride];
325 return pload<Packet4i>(ai);
327 template<> EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet4f>(
float* to,
const Packet4f& from,
Index stride)
329 float EIGEN_ALIGN16 af[4];
330 pstore<float>(af, from);
331 to[0*stride] = af[0];
332 to[1*stride] = af[1];
333 to[2*stride] = af[2];
334 to[3*stride] = af[3];
336 template<> EIGEN_DEVICE_FUNC
inline void pscatter<int, Packet4i>(
int* to,
const Packet4i& from,
Index stride)
338 int EIGEN_ALIGN16 ai[4];
339 pstore<int>((
int *)ai, from);
340 to[0*stride] = ai[0];
341 to[1*stride] = ai[1];
342 to[2*stride] = ai[2];
343 to[3*stride] = ai[3];
346 template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
return pset1<Packet4f>(a) + p4f_COUNTDOWN; }
347 template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int& a) {
return pset1<Packet4i>(a) + p4i_COUNTDOWN; }
349 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return a + b; }
350 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a + b; }
352 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return a - b; }
353 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a - b; }
355 template<> EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
return p4f_ZERO - a; }
356 template<> EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
return p4i_ZERO - a; }
358 template<> EIGEN_STRONG_INLINE Packet4f pconj(
const Packet4f& a) {
return a; }
359 template<> EIGEN_STRONG_INLINE Packet4i pconj(
const Packet4i& a) {
return a; }
361 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_madd(a,b, p4f_MZERO); }
362 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a * b; }
364 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b)
366 #ifndef __VSX__ // VSX actually provides a div instruction 367 Packet4f t, y_0, y_1;
373 t = vec_nmsub(y_0, b, p4f_ONE);
374 y_1 = vec_madd(y_0, t, y_0);
376 return vec_madd(a, y_1, p4f_MZERO);
378 return vec_div(a, b);
382 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& )
383 { eigen_assert(
false &&
"packet integer division are not supported by AltiVec");
384 return pset1<Packet4i>(0);
388 template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
return vec_madd(a,b,c); }
389 template<> EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
return a*b + c; }
391 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b)
395 __asm__ (
"xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa" (ret) :
"wa" (a),
"wa" (b));
398 return vec_min(a, b);
401 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_min(a, b); }
403 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b)
407 __asm__ (
"xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa" (ret) :
"wa" (a),
"wa" (b));
410 return vec_max(a, b);
413 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_max(a, b); }
415 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_and(a, b); }
416 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_and(a, b); }
418 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_or(a, b); }
419 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_or(a, b); }
421 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_xor(a, b); }
422 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_xor(a, b); }
424 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_and(a, vec_nor(b, b)); }
425 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_and(a, vec_nor(b, b)); }
427 template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(
const Packet4f& a) {
return vec_round(a); }
428 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
return vec_ceil(a); }
429 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
return vec_floor(a); }
432 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from)
434 EIGEN_DEBUG_ALIGNED_LOAD
437 MSQ = vec_ld(0, (
unsigned char *)from);
438 LSQ = vec_ld(15, (
unsigned char *)from);
439 mask = vec_lvsl(0, from);
440 return (Packet4f) vec_perm(MSQ, LSQ, mask);
443 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from)
445 EIGEN_DEBUG_ALIGNED_LOAD
449 MSQ = vec_ld(0, (
unsigned char *)from);
450 LSQ = vec_ld(15, (
unsigned char *)from);
451 mask = vec_lvsl(0, from);
452 return (Packet4i) vec_perm(MSQ, LSQ, mask);
456 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from)
458 EIGEN_DEBUG_UNALIGNED_LOAD
459 return (Packet4i) vec_vsx_ld((
long)from & 15, (
const int*) _EIGEN_ALIGNED_PTR(from));
461 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from)
463 EIGEN_DEBUG_UNALIGNED_LOAD
464 return (Packet4f) vec_vsx_ld((
long)from & 15, (
const float*) _EIGEN_ALIGNED_PTR(from));
468 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from)
471 if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from);
472 else p = ploadu<Packet4f>(from);
473 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
475 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from)
478 if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from);
479 else p = ploadu<Packet4i>(from);
480 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
484 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from)
486 EIGEN_DEBUG_UNALIGNED_STORE
489 Packet16uc MSQ, LSQ, edges;
490 Packet16uc edgeAlign, align;
492 MSQ = vec_ld(0, (
unsigned char *)to);
493 LSQ = vec_ld(15, (
unsigned char *)to);
494 edgeAlign = vec_lvsl(0, to);
495 edges=vec_perm(LSQ,MSQ,edgeAlign);
496 align = vec_lvsr( 0, to );
497 MSQ = vec_perm(edges,(Packet16uc)from,align);
498 LSQ = vec_perm((Packet16uc)from,edges,align);
499 vec_st( LSQ, 15, (
unsigned char *)to );
500 vec_st( MSQ, 0, (
unsigned char *)to );
502 template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from)
504 EIGEN_DEBUG_UNALIGNED_STORE
507 Packet16uc MSQ, LSQ, edges;
508 Packet16uc edgeAlign, align;
510 MSQ = vec_ld(0, (
unsigned char *)to);
511 LSQ = vec_ld(15, (
unsigned char *)to);
512 edgeAlign = vec_lvsl(0, to);
513 edges=vec_perm(LSQ, MSQ, edgeAlign);
514 align = vec_lvsr( 0, to );
515 MSQ = vec_perm(edges, (Packet16uc) from, align);
516 LSQ = vec_perm((Packet16uc) from, edges, align);
517 vec_st( LSQ, 15, (
unsigned char *)to );
518 vec_st( MSQ, 0, (
unsigned char *)to );
522 template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from)
524 EIGEN_DEBUG_ALIGNED_STORE
525 vec_vsx_st(from, (
long)to & 15, (
int*) _EIGEN_ALIGNED_PTR(to));
527 template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from)
529 EIGEN_DEBUG_ALIGNED_STORE
530 vec_vsx_st(from, (
long)to & 15, (
float*) _EIGEN_ALIGNED_PTR(to));
534 template<> EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) { EIGEN_PPC_PREFETCH(addr); }
535 template<> EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) { EIGEN_PPC_PREFETCH(addr); }
537 template<> EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
float EIGEN_ALIGN16 x; vec_ste(a, 0, &x);
return x; }
538 template<> EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
int EIGEN_ALIGN16 x; vec_ste(a, 0, &x);
return x; }
540 template<> EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a)
542 return reinterpret_cast<Packet4f
>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
544 template<> EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a)
546 return reinterpret_cast<Packet4i
>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32)); }
548 template<> EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
return vec_abs(a); }
549 template<> EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
return vec_abs(a); }
551 template<> EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a)
554 b = vec_sld(a, a, 8);
556 b = vec_sld(sum, sum, 4);
561 template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(
const Packet4f* vecs)
563 Packet4f v[4], sum[4];
568 v[0] = vec_mergeh(vecs[0], vecs[2]);
569 v[1] = vec_mergel(vecs[0], vecs[2]);
570 v[2] = vec_mergeh(vecs[1], vecs[3]);
571 v[3] = vec_mergel(vecs[1], vecs[3]);
573 sum[0] = vec_mergeh(v[0], v[2]);
574 sum[1] = vec_mergel(v[0], v[2]);
575 sum[2] = vec_mergeh(v[1], v[3]);
576 sum[3] = vec_mergel(v[1], v[3]);
580 sum[0] = sum[0] + sum[1];
582 sum[1] = sum[2] + sum[3];
584 sum[0] = sum[0] + sum[1];
589 template<> EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a)
592 sum = vec_sums(a, p4i_ZERO);
594 sum = vec_sld(sum, p4i_ZERO, 12);
596 sum = vec_sld(p4i_ZERO, sum, 4);
601 template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(
const Packet4i* vecs)
603 Packet4i v[4], sum[4];
608 v[0] = vec_mergeh(vecs[0], vecs[2]);
609 v[1] = vec_mergel(vecs[0], vecs[2]);
610 v[2] = vec_mergeh(vecs[1], vecs[3]);
611 v[3] = vec_mergel(vecs[1], vecs[3]);
613 sum[0] = vec_mergeh(v[0], v[2]);
614 sum[1] = vec_mergel(v[0], v[2]);
615 sum[2] = vec_mergeh(v[1], v[3]);
616 sum[3] = vec_mergel(v[1], v[3]);
620 sum[0] = sum[0] + sum[1];
622 sum[1] = sum[2] + sum[3];
624 sum[0] = sum[0] + sum[1];
631 template<> EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a)
634 prod = pmul(a, vec_sld(a, a, 8));
635 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
638 template<> EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a)
640 EIGEN_ALIGN16
int aux[4];
642 return aux[0] * aux[1] * aux[2] * aux[3];
646 template<> EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a)
649 b = vec_min(a, vec_sld(a, a, 8));
650 res = vec_min(b, vec_sld(b, b, 4));
654 template<> EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a)
657 b = vec_min(a, vec_sld(a, a, 8));
658 res = vec_min(b, vec_sld(b, b, 4));
663 template<> EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a)
666 b = vec_max(a, vec_sld(a, a, 8));
667 res = vec_max(b, vec_sld(b, b, 4));
671 template<> EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a)
674 b = vec_max(a, vec_sld(a, a, 8));
675 res = vec_max(b, vec_sld(b, b, 4));
680 struct palign_impl<Offset,Packet4f>
682 static EIGEN_STRONG_INLINE
void run(Packet4f& first,
const Packet4f& second)
685 switch (Offset % 4) {
687 first = vec_sld(first, second, 4);
break;
689 first = vec_sld(first, second, 8);
break;
691 first = vec_sld(first, second, 12);
break;
694 switch (Offset % 4) {
696 first = vec_sld(second, first, 12);
break;
698 first = vec_sld(second, first, 8);
break;
700 first = vec_sld(second, first, 4);
break;
707 struct palign_impl<Offset,Packet4i>
709 static EIGEN_STRONG_INLINE
void run(Packet4i& first,
const Packet4i& second)
712 switch (Offset % 4) {
714 first = vec_sld(first, second, 4);
break;
716 first = vec_sld(first, second, 8);
break;
718 first = vec_sld(first, second, 12);
break;
721 switch (Offset % 4) {
723 first = vec_sld(second, first, 12);
break;
725 first = vec_sld(second, first, 8);
break;
727 first = vec_sld(second, first, 4);
break;
733 EIGEN_DEVICE_FUNC
inline void 734 ptranspose(PacketBlock<Packet4f,4>& kernel) {
735 Packet4f t0, t1, t2, t3;
736 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
737 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
738 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
739 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
740 kernel.packet[0] = vec_mergeh(t0, t2);
741 kernel.packet[1] = vec_mergel(t0, t2);
742 kernel.packet[2] = vec_mergeh(t1, t3);
743 kernel.packet[3] = vec_mergel(t1, t3);
746 EIGEN_DEVICE_FUNC
inline void 747 ptranspose(PacketBlock<Packet4i,4>& kernel) {
748 Packet4i t0, t1, t2, t3;
749 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
750 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
751 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
752 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
753 kernel.packet[0] = vec_mergeh(t0, t2);
754 kernel.packet[1] = vec_mergel(t0, t2);
755 kernel.packet[2] = vec_mergeh(t1, t3);
756 kernel.packet[3] = vec_mergel(t1, t3);
759 template<> EIGEN_STRONG_INLINE Packet4i pblend(
const Selector<4>& ifPacket,
const Packet4i& thenPacket,
const Packet4i& elsePacket) {
760 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
761 Packet4ui mask =
reinterpret_cast<Packet4ui
>(vec_cmpeq(reinterpret_cast<Packet4ui>(select), reinterpret_cast<Packet4ui>(p4i_ONE)));
762 return vec_sel(elsePacket, thenPacket, mask);
765 template<> EIGEN_STRONG_INLINE Packet4f pblend(
const Selector<4>& ifPacket,
const Packet4f& thenPacket,
const Packet4f& elsePacket) {
766 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
767 Packet4ui mask =
reinterpret_cast<Packet4ui
>(vec_cmpeq(reinterpret_cast<Packet4ui>(select), reinterpret_cast<Packet4ui>(p4i_ONE)));
768 return vec_sel(elsePacket, thenPacket, mask);
774 typedef __vector
double Packet2d;
775 typedef __vector
unsigned long long Packet2ul;
776 typedef __vector
long long Packet2l;
778 typedef Packet2ul Packet2bl;
780 typedef __vector __bool
long Packet2bl;
783 static Packet2l p2l_ONE = { 1, 1 };
784 static Packet2l p2l_ZERO =
reinterpret_cast<Packet2l
>(p4i_ZERO);
785 static Packet2d p2d_ONE = { 1.0, 1.0 };
786 static Packet2d p2d_ZERO =
reinterpret_cast<Packet2d
>(p4f_ZERO);
787 static Packet2d p2d_MZERO = { -0.0, -0.0 };
790 static Packet2d p2d_COUNTDOWN =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(p2d_ZERO), reinterpret_cast<Packet4f>(p2d_ONE), 8));
792 static Packet2d p2d_COUNTDOWN =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(p2d_ONE), reinterpret_cast<Packet4f>(p2d_ZERO), 8));
795 template<
int index> Packet2d vec_splat_dbl(Packet2d& a);
797 template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<0>(Packet2d& a)
799 return reinterpret_cast<Packet2d
>(vec_perm(a, a, p16uc_PSET64_HI));
802 template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<1>(Packet2d& a)
804 return reinterpret_cast<Packet2d
>(vec_perm(a, a, p16uc_PSET64_LO));
807 template<>
struct packet_traits<double> : default_packet_traits
809 typedef Packet2d type;
810 typedef Packet2d half;
838 template<>
struct unpacket_traits<Packet2d> {
typedef double type;
enum {size=2, alignment=
Aligned16};
typedef Packet2d half; };
840 inline std::ostream & operator <<(std::ostream & s,
const Packet2l & v)
847 s << vt.n[0] <<
", " << vt.n[1];
851 inline std::ostream & operator <<(std::ostream & s,
const Packet2d & v)
858 s << vt.n[0] <<
", " << vt.n[1];
863 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from)
865 EIGEN_DEBUG_ALIGNED_LOAD
867 return vec_vsx_ld(0, from);
869 return vec_ld(0, from);
873 template<> EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from)
875 EIGEN_DEBUG_ALIGNED_STORE
877 vec_vsx_st(from, 0, to);
883 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
884 Packet2d v = {from, from};
888 template<> EIGEN_STRONG_INLINE
void 889 pbroadcast4<Packet2d>(
const double *a,
890 Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
892 a1 = pload<Packet2d>(a);
893 a0 = vec_splat_dbl<0>(a1);
894 a1 = vec_splat_dbl<1>(a1);
895 a3 = pload<Packet2d>(a+2);
896 a2 = vec_splat_dbl<0>(a3);
897 a3 = vec_splat_dbl<1>(a3);
900 template<> EIGEN_DEVICE_FUNC
inline Packet2d pgather<double, Packet2d>(
const double* from,
Index stride)
902 double EIGEN_ALIGN16 af[2];
903 af[0] = from[0*stride];
904 af[1] = from[1*stride];
905 return pload<Packet2d>(af);
907 template<> EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet2d>(
double* to,
const Packet2d& from,
Index stride)
909 double EIGEN_ALIGN16 af[2];
910 pstore<double>(af, from);
911 to[0*stride] = af[0];
912 to[1*stride] = af[1];
915 template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
return pset1<Packet2d>(a) + p2d_COUNTDOWN; }
917 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return a + b; }
919 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return a - b; }
921 template<> EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
return p2d_ZERO - a; }
923 template<> EIGEN_STRONG_INLINE Packet2d pconj(
const Packet2d& a) {
return a; }
925 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_madd(a,b,p2d_MZERO); }
926 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_div(a,b); }
929 template<> EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
return vec_madd(a, b, c); }
931 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b)
934 __asm__ (
"xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa" (ret) :
"wa" (a),
"wa" (b));
938 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b)
941 __asm__ (
"xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa" (ret) :
"wa" (a),
"wa" (b));
945 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_and(a, b); }
947 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_or(a, b); }
949 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_xor(a, b); }
951 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_and(a, vec_nor(b, b)); }
953 template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(
const Packet2d& a) {
return vec_round(a); }
954 template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
return vec_ceil(a); }
955 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
return vec_floor(a); }
957 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from)
959 EIGEN_DEBUG_ALIGNED_LOAD
960 return (Packet2d) vec_vsx_ld((
long)from & 15, (
const double*) _EIGEN_ALIGNED_PTR(from));
963 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from)
966 if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from);
967 else p = ploadu<Packet2d>(from);
968 return vec_splat_dbl<0>(p);
971 template<> EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from)
973 EIGEN_DEBUG_ALIGNED_STORE
974 vec_vsx_st((Packet4f)from, (
long)to & 15, (
float*) _EIGEN_ALIGNED_PTR(to));
977 template<> EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) { EIGEN_PPC_PREFETCH(addr); }
979 template<> EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
double EIGEN_ALIGN16 x[2]; pstore<double>(x, a);
return x[0]; }
981 template<> EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a)
983 return reinterpret_cast<Packet2d
>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
985 template<> EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
return vec_abs(a); }
987 template<> EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a)
990 b =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(a), reinterpret_cast<Packet4f>(a), 8));
992 return pfirst<Packet2d>(sum);
995 template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(
const Packet2d* vecs)
998 v[0] = vecs[0] +
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(vecs[0]), reinterpret_cast<Packet4f>(vecs[0]), 8));
999 v[1] = vecs[1] +
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(vecs[1]), reinterpret_cast<Packet4f>(vecs[1]), 8));
1002 sum =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(v[0]), reinterpret_cast<Packet4f>(v[1]), 8));
1004 sum =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4f>(v[1]), reinterpret_cast<Packet4f>(v[0]), 8));
1011 template<> EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a)
1013 return pfirst(pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1017 template<> EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a)
1019 return pfirst(pmin(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1023 template<> EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a)
1025 return pfirst(pmax(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1028 template<
int Offset>
1029 struct palign_impl<Offset,Packet2d>
1031 static EIGEN_STRONG_INLINE
void run(Packet2d& first,
const Packet2d& second)
1035 first =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4ui>(first), reinterpret_cast<Packet4ui>(second), 8));
1037 first =
reinterpret_cast<Packet2d
>(vec_sld(reinterpret_cast<Packet4ui>(second), reinterpret_cast<Packet4ui>(first), 8));
1042 EIGEN_DEVICE_FUNC
inline void 1043 ptranspose(PacketBlock<Packet2d,2>& kernel) {
1045 t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
1046 t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
1047 kernel.packet[0] = t0;
1048 kernel.packet[1] = t1;
1051 template<> EIGEN_STRONG_INLINE Packet2d pblend(
const Selector<2>& ifPacket,
const Packet2d& thenPacket,
const Packet2d& elsePacket) {
1052 Packet2l select = { ifPacket.select[0], ifPacket.select[1] };
1053 Packet2bl mask =
reinterpret_cast<Packet2bl
>( vec_cmpeq(reinterpret_cast<Packet2d>(select), reinterpret_cast<Packet2d>(p2l_ONE)) );
1054 return vec_sel(elsePacket, thenPacket, mask);
1061 #endif // EIGEN_PACKET_MATH_ALTIVEC_H Definition: Constants.h:230
Namespace containing all symbols from the Eigen library.
Definition: Core:306
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33
Definition: Eigen_Colamd.h:50