Skip to content

Commit 67574aa

Browse files
KamathForAIXfacebook-github-bot
KamathForAIX
authored and
facebook-github-bot
committed
Fix the endianness issue in AIX while running the benchmark. (facebookresearch#3345)
Summary: This pull request is for issue facebookresearch#3330. This patch makes sure that packed code arrays are in big endian format. Kindly let us know if we need any changes or if we can have a better approach. Pull Request resolved: facebookresearch#3345 Reviewed By: junjieqi Differential Revision: D55957630 Pulled By: mdouze fbshipit-source-id: f728f9563f6b942af9d8899b54662d7ceb811206
1 parent b2e91f6 commit 67574aa

File tree

6 files changed

+219
-47
lines changed

6 files changed

+219
-47
lines changed

‎contrib/vecs_io.py

+3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# This source code is licensed under the MIT license found in the
44
# LICENSE file in the root directory of this source tree.
55

6+
import sys
67
import numpy as np
78

89
"""
@@ -13,6 +14,8 @@
1314

1415
def ivecs_read(fname):
1516
a = np.fromfile(fname, dtype='int32')
17+
if sys.byteorder == 'big':
18+
a.byteswap(inplace=True)
1619
d = a[0]
1720
return a.reshape(-1, d + 1)[:, 1:].copy()
1821

‎faiss/cppcontrib/detail/UintReader.h

+95-17
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#pragma once
99

10+
#include <faiss/impl/platform_macros.h>
1011
#include <cstdint>
1112

1213
namespace faiss {
@@ -31,7 +32,11 @@ struct Uint8Reader {
3132
if (N_ELEMENTS > CPOS + 3) {
3233
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
3334
codes + ELEMENT_TO_READ * 4);
35+
#ifdef FAISS_BIG_ENDIAN
36+
return (code32) >> 24;
37+
#else
3438
return (code32 & 0x000000FF);
39+
#endif
3540
} else {
3641
return codes[CPOS];
3742
}
@@ -40,7 +45,11 @@ struct Uint8Reader {
4045
if (N_ELEMENTS > CPOS + 2) {
4146
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
4247
codes + ELEMENT_TO_READ * 4);
48+
#ifdef FAISS_BIG_ENDIAN
49+
return (code32 & 0x00FF0000) >> 16;
50+
#else
4351
return (code32 & 0x0000FF00) >> 8;
52+
#endif
4453
} else {
4554
return codes[CPOS];
4655
}
@@ -49,7 +58,11 @@ struct Uint8Reader {
4958
if (N_ELEMENTS > CPOS + 1) {
5059
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
5160
codes + ELEMENT_TO_READ * 4);
61+
#ifdef FAISS_BIG_ENDIAN
62+
return (code32 & 0x0000FF00) >> 8;
63+
#else
5264
return (code32 & 0x00FF0000) >> 16;
65+
#endif
5366
} else {
5467
return codes[CPOS];
5568
}
@@ -58,7 +71,11 @@ struct Uint8Reader {
5871
if (N_ELEMENTS > CPOS) {
5972
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
6073
codes + ELEMENT_TO_READ * 4);
74+
#ifdef FAISS_BIG_ENDIAN
75+
return (code32 & 0x000000FF);
76+
#else
6177
return (code32) >> 24;
78+
#endif
6279
} else {
6380
return codes[CPOS];
6481
}
@@ -87,40 +104,61 @@ struct Uint10Reader {
87104
switch (SUB_ELEMENT) {
88105
case 0: {
89106
if (N_ELEMENTS > CPOS + 2) {
90-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
107+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
91108
codes + ELEMENT_TO_READ * 5);
109+
#ifdef FAISS_BIG_ENDIAN
110+
code32 = Swap4Bytes(code32);
111+
#endif
92112
return (code32 & 0b0000001111111111);
93113
} else {
94-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
114+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
95115
codes + ELEMENT_TO_READ * 5 + 0);
116+
#ifdef FAISS_BIG_ENDIAN
117+
code16 = Swap2Bytes(code16);
118+
#endif
96119
return (code16 & 0b0000001111111111);
97120
}
98121
}
99122
case 1: {
100123
if (N_ELEMENTS > CPOS + 1) {
101-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
124+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
102125
codes + ELEMENT_TO_READ * 5);
126+
#ifdef FAISS_BIG_ENDIAN
127+
code32 = Swap4Bytes(code32);
128+
#endif
103129
return (code32 & 0b000011111111110000000000) >> 10;
104130
} else {
105-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
131+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
106132
codes + ELEMENT_TO_READ * 5 + 1);
133+
#ifdef FAISS_BIG_ENDIAN
134+
code16 = Swap2Bytes(code16);
135+
#endif
107136
return (code16 & 0b0000111111111100) >> 2;
108137
}
109138
}
110139
case 2: {
111140
if (N_ELEMENTS > CPOS) {
112-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
141+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
113142
codes + ELEMENT_TO_READ * 5);
143+
#ifdef FAISS_BIG_ENDIAN
144+
code32 = Swap4Bytes(code32);
145+
#endif
114146
return (code32 & 0b00111111111100000000000000000000) >> 20;
115147
} else {
116-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
148+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
117149
codes + ELEMENT_TO_READ * 5 + 2);
150+
#ifdef FAISS_BIG_ENDIAN
151+
code16 = Swap2Bytes(code16);
152+
#endif
118153
return (code16 & 0b0011111111110000) >> 4;
119154
}
120155
}
121156
case 3: {
122-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
157+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
123158
codes + ELEMENT_TO_READ * 5 + 3);
159+
#ifdef FAISS_BIG_ENDIAN
160+
code16 = Swap2Bytes(code16);
161+
#endif
124162
return (code16 & 0b1111111111000000) >> 6;
125163
}
126164
}
@@ -147,45 +185,69 @@ struct Uint12Reader {
147185
switch (SUB_ELEMENT) {
148186
case 0: {
149187
if (N_ELEMENTS > CPOS + 2) {
150-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
188+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
151189
codes + ELEMENT_TO_READ * 6);
190+
#ifdef FAISS_BIG_ENDIAN
191+
code32 = Swap4Bytes(code32);
192+
#endif
152193
return (code32 & 0b0000111111111111);
153194
} else {
154-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
195+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
155196
codes + ELEMENT_TO_READ * 6 + 0);
197+
#ifdef FAISS_BIG_ENDIAN
198+
code16 = Swap2Bytes(code16);
199+
#endif
156200
return (code16 & 0b0000111111111111);
157201
}
158202
}
159203
case 1: {
160204
if (N_ELEMENTS > CPOS + 1) {
161-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
205+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
162206
codes + ELEMENT_TO_READ * 6);
207+
#ifdef FAISS_BIG_ENDIAN
208+
code32 = Swap4Bytes(code32);
209+
#endif
163210
return (code32 & 0b111111111111000000000000) >> 12;
164211
} else {
165-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
212+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
166213
codes + ELEMENT_TO_READ * 6 + 1);
214+
#ifdef FAISS_BIG_ENDIAN
215+
code16 = Swap2Bytes(code16);
216+
#endif
167217
return (code16 & 0b1111111111110000) >> 4;
168218
}
169219
}
170220
case 2: {
171221
if (N_ELEMENTS > CPOS + 1) {
172-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
222+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
173223
codes + ELEMENT_TO_READ * 6 + 2);
224+
#ifdef FAISS_BIG_ENDIAN
225+
code32 = Swap4Bytes(code32);
226+
#endif
174227
return (code32 & 0b000011111111111100000000) >> 8;
175228
} else {
176-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
229+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
177230
codes + ELEMENT_TO_READ * 6 + 3);
231+
#ifdef FAISS_BIG_ENDIAN
232+
code16 = Swap2Bytes(code16);
233+
#endif
178234
return (code16 & 0b0000111111111111);
179235
}
180236
}
181237
case 3: {
182238
if (N_ELEMENTS > CPOS) {
183-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
239+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
184240
codes + ELEMENT_TO_READ * 6 + 2);
241+
#ifdef FAISS_BIG_ENDIAN
242+
code32 = Swap4Bytes(code32);
243+
#endif
185244
return (code32 & 0b11111111111100000000000000000000) >> 20;
186245
} else {
187-
const uint16_t code16 = *reinterpret_cast<const uint16_t*>(
246+
uint16_t code16 = *reinterpret_cast<const uint16_t*>(
188247
codes + ELEMENT_TO_READ * 6 + 4);
248+
#ifdef FAISS_BIG_ENDIAN
249+
code16 = Swap2Bytes(code16);
250+
#endif
189251
return (code16 & 0b1111111111110000) >> 4;
190252
}
191253
}
@@ -208,23 +270,39 @@ struct Uint16Reader {
208270
switch (SUB_ELEMENT) {
209271
case 0: {
210272
if (N_ELEMENTS > CPOS + 1) {
211-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
273+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
212274
codes + ELEMENT_TO_READ * 4);
275+
#ifdef FAISS_BIG_ENDIAN
276+
code32 = Swap4Bytes(code32);
277+
#endif
213278
return (code32 & 0x0000FFFF);
214279
} else {
215280
const uint16_t* const __restrict codesFp16 =
216281
reinterpret_cast<const uint16_t*>(codes);
282+
#ifdef FAISS_BIG_ENDIAN
283+
uint16_t rt = codesFp16[CPOS];
284+
rt = Swap2Bytes(rt);
285+
return rt;
286+
#endif
217287
return codesFp16[CPOS];
218288
}
219289
}
220290
case 1: {
221291
if (N_ELEMENTS > CPOS) {
222-
const uint32_t code32 = *reinterpret_cast<const uint32_t*>(
292+
uint32_t code32 = *reinterpret_cast<const uint32_t*>(
223293
codes + ELEMENT_TO_READ * 4);
294+
#ifdef FAISS_BIG_ENDIAN
295+
code32 = Swap4Bytes(code32);
296+
#endif
224297
return code32 >> 16;
225298
} else {
226299
const uint16_t* const __restrict codesFp16 =
227300
reinterpret_cast<const uint16_t*>(codes);
301+
#ifdef FAISS_BIG_ENDIAN
302+
uint16_t rt = codesFp16[CPOS];
303+
rt = Swap2Bytes(rt);
304+
return rt;
305+
#endif
228306
return codesFp16[CPOS];
229307
}
230308
}

0 commit comments

Comments
 (0)