12#pragma GCC diagnostic push
13#pragma GCC diagnostic ignored "-Wold-style-cast"
15char cnpy::BigEndianTest()
18 return (((
char*) & x)[0]) ?
'<' :
'>';
21char cnpy::map_type(
const std::type_info& t)
23 if (t ==
typeid(
float) )
28 if (t ==
typeid(
double) )
33 if (t ==
typeid(
long double) )
38 if (t ==
typeid(
int) )
43 if (t ==
typeid(
char) )
48 if (t ==
typeid(
short) )
53 if (t ==
typeid(
long) )
58 if (t ==
typeid(
long long) )
63 if (t ==
typeid(
unsigned char) )
68 if (t ==
typeid(
unsigned short) )
73 if (t ==
typeid(
unsigned long) )
78 if (t ==
typeid(
unsigned long long) )
83 if (t ==
typeid(
unsigned int) )
88 if (t ==
typeid(
bool) )
93 if (t ==
typeid(std::complex<float>) )
98 if (t ==
typeid(std::complex<double>) )
103 if (t ==
typeid(std::complex<long double>) )
113template<> std::vector<char>& cnpy::operator+=(std::vector<char>& lhs,
114 const std::string rhs)
116 lhs.insert(lhs.end(), rhs.begin(), rhs.end());
120template<> std::vector<char>& cnpy::operator+=(std::vector<char>& lhs,
124 size_t len = strlen(rhs);
127 for (
size_t byte = 0;
byte < len;
byte++)
129 lhs.push_back(rhs[
byte]);
135void cnpy::parse_npy_header(
unsigned char* buffer,
size_t& word_size,
136 std::vector<size_t>& shape,
bool& fortran_order, std::string& number_type)
139 uint8_t major_version = *
reinterpret_cast<uint8_t*
>(buffer + 6);
140 uint8_t minor_version = *
reinterpret_cast<uint8_t*
>(buffer + 7);
141 uint16_t header_len = *
reinterpret_cast<uint16_t*
>(buffer + 8);
142 std::string header(
reinterpret_cast<char*
>(buffer + 9), header_len);
145 loc1 = header.find(
"fortran_order") + 16;
146 fortran_order = (header.substr(loc1, 4) ==
"True" ?
true :
false);
148 loc1 = header.find(
"(");
149 loc2 = header.find(
")");
150 std::regex num_regex(
"[0-9][0-9]*");
153 std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1);
155 while (std::regex_search(str_shape, sm, num_regex))
157 shape.push_back(std::stoi(sm[0].str()));
158 str_shape = sm.suffix().str();
164 loc1 = header.find(
"descr") + 9;
165 bool littleEndian = (header[loc1] ==
'<' || header[loc1] ==
'|' ? true :
false);
166 assert(littleEndian);
169 number_type = header.substr(loc1 + 1, 2);
170 std::string str_ws = header.substr(loc1 + 2);
171 loc2 = str_ws.find(
"'");
172 word_size = atoi(str_ws.substr(0, loc2).c_str());
175void cnpy::parse_npy_header(FILE* fp,
size_t& word_size,
176 std::vector<size_t>& shape,
bool& fortran_order, std::string& number_type)
179 size_t res = fread(buffer,
sizeof(
char), 11, fp);
183 throw std::runtime_error(
"parse_npy_header: failed fread");
186 std::string header = fgets(buffer, 256, fp);
187 assert(header[header.size() - 1] ==
'\n');
190 loc1 = header.find(
"fortran_order");
192 if (loc1 == std::string::npos)
194 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: 'fortran_order'");
198 fortran_order = (header.substr(loc1, 4) ==
"True" ?
true :
false);
200 loc1 = header.find(
"(");
201 loc2 = header.find(
")");
203 if (loc1 == std::string::npos || loc2 == std::string::npos)
205 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: '(' or ')'");
208 std::regex num_regex(
"[0-9][0-9]*");
211 std::string str_shape = header.substr(loc1 + 1, loc2 - loc1 - 1);
213 while (std::regex_search(str_shape, sm, num_regex))
215 shape.push_back(std::stoi(sm[0].str()));
216 str_shape = sm.suffix().str();
222 loc1 = header.find(
"descr");
224 if (loc1 == std::string::npos)
226 throw std::runtime_error(
"parse_npy_header: failed to find header keyword: 'descr'");
230 bool littleEndian = (header[loc1] ==
'<' || header[loc1] ==
'|' ? true :
false);
231 assert(littleEndian);
234 std::string str_ws = header.substr(loc1 + 2);
235 number_type = header.substr(loc1 + 1, 2);
236 loc2 = str_ws.find(
"'");
237 word_size = atoi(str_ws.substr(0, loc2).c_str());
240void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs,
241 size_t& global_header_size,
size_t& global_header_offset)
243 std::vector<char> footer(22);
244 fseek(fp, -22, SEEK_END);
245 size_t res = fread(& footer[0],
sizeof(
char), 22, fp);
249 throw std::runtime_error(
"parse_zip_footer: failed fread");
252 uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
253 disk_no = * (uint16_t*) & footer[4];
254 disk_start = * (uint16_t*) & footer[6];
255 nrecs_on_disk = * (uint16_t*) & footer[8];
256 nrecs = * (uint16_t*) & footer[10];
257 global_header_size = * (uint32_t*) & footer[12];
258 global_header_offset = * (uint32_t*) & footer[16];
259 comment_len = * (uint16_t*) & footer[20];
260 assert(disk_no == 0);
261 assert(disk_start == 0);
262 assert(nrecs_on_disk == nrecs);
263 assert(comment_len == 0);
268 std::vector<size_t> shape;
271 std::string number_type;
272 cnpy::parse_npy_header(fp, word_size, shape, fortran_order, number_type);
274 size_t nread = fread(arr.data<
char>(), 1, arr.num_bytes(), fp);
276 if (nread != arr.num_bytes())
278 throw std::runtime_error(
"load_the_npy_file: failed fread");
286 FILE* fp = fopen(fname.c_str(),
"rb");
287 return load_the_npy_file(fp);
290void parse_zip64_sizes(
const std::vector<char>& extra_field,
291 uint32_t& compr_bytes, uint32_t& uncompr_bytes)
293 if (extra_field.size() >= 4 && (compr_bytes == 0xFFFFFFFF
294 || uncompr_bytes == 0xFFFFFFFF))
296 uint16_t extra_id = *
reinterpret_cast<const uint16_t*
>(&extra_field[0]);
297 uint16_t extra_size = *
reinterpret_cast<const uint16_t*
>(&extra_field[2]);
299 if (extra_id == 0x0001 && extra_size >= 16)
303 if (uncompr_bytes == 0xFFFFFFFF)
305 uncompr_bytes =
static_cast<uint32_t
>(*
reinterpret_cast<const uint64_t*
>
306 (&extra_field[offset]));
310 if (compr_bytes == 0xFFFFFFFF)
312 compr_bytes =
static_cast<uint32_t
>(*
reinterpret_cast<const uint64_t*
>
313 (&extra_field[offset]));
320 uint32_t uncompr_bytes)
322 std::vector<unsigned char> buffer_compr(compr_bytes);
323 std::vector<unsigned char> buffer_uncompr(uncompr_bytes);
324 size_t nread = fread(& buffer_compr[0], 1, compr_bytes, fp);
326 if (nread != compr_bytes)
328 throw std::runtime_error(
"load_the_npy_file: failed fread");
333 d_stream.zalloc = Z_NULL;
334 d_stream.zfree = Z_NULL;
335 d_stream.opaque = Z_NULL;
336 d_stream.avail_in = 0;
337 d_stream.next_in = Z_NULL;
338 err = inflateInit2(& d_stream, -MAX_WBITS);
339 d_stream.avail_in = compr_bytes;
340 d_stream.next_in = & buffer_compr[0];
341 d_stream.avail_out = uncompr_bytes;
342 d_stream.next_out = & buffer_uncompr[0];
343 err = inflate(& d_stream, Z_FINISH);
344 err = inflateEnd(& d_stream);
345 std::vector<size_t> shape;
348 std::string number_type;
349 cnpy::parse_npy_header(& buffer_uncompr[0], word_size, shape, fortran_order,
351 cnpy::NpyArray array(shape, word_size, fortran_order, number_type);
352 size_t offset = uncompr_bytes - array.num_bytes();
353 memcpy(array.data<
unsigned char>(), & buffer_uncompr[0] + offset,
358cnpy::npz_t cnpy::npz_load(std::string fname)
360 FILE* fp = fopen(fname.c_str(),
"rb");
361 std::cerr <<
"Line: 350 of file cnpy.C" << endl;
365 throw std::runtime_error(
"npz_load: Error! Unable to open file " + fname +
"!");
372 std::vector<char> local_header(30);
373 size_t headerres = fread(&local_header[0],
sizeof(
char), 30, fp);
374 std::cerr << headerres << endl;
378 throw std::runtime_error(
"npz_load: failed fread");
382 if (local_header[2] != 0x03 || local_header[3] != 0x04)
388 uint16_t name_len = *(uint16_t*) &local_header[26];
389 std::string varname(name_len,
' ');
390 size_t vname_res = fread(&varname[0],
sizeof(
char), name_len, fp);
392 if (vname_res != name_len)
394 throw std::runtime_error(
"npz_load: failed fread");
398 varname.erase(varname.end() - 4, varname.end());
400 uint16_t extra_field_len = *(uint16_t*) &local_header[28];
401 std::vector<char> extra_field(extra_field_len);
403 if (extra_field_len > 0)
405 size_t efield_res = fread(&extra_field[0],
sizeof(
char), extra_field_len, fp);
407 if (efield_res != extra_field_len)
409 throw std::runtime_error(
"npz_load: failed fread");
413 uint16_t compr_method = *
reinterpret_cast<uint16_t*
>(&local_header[0] + 8);
414 uint32_t compr_bytes = *
reinterpret_cast<uint32_t*
>(&local_header[0] + 18);
415 uint32_t uncompr_bytes = *
reinterpret_cast<uint32_t*
>(&local_header[0] + 22);
417 parse_zip64_sizes(extra_field, compr_bytes, uncompr_bytes);
419 if (compr_method == 0)
421 arrays[varname] = load_the_npy_file(fp);
425 arrays[varname] = load_the_npz_array(fp, compr_bytes, uncompr_bytes);
433cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname)
435 FILE* fp = fopen(fname.c_str(),
"rb");
439 throw std::runtime_error(
"npz_load: Unable to open file " + fname);
444 std::vector<char> local_header(30);
445 size_t header_res = fread(&local_header[0],
sizeof(
char), 30, fp);
447 if (header_res != 30)
449 throw std::runtime_error(
"npz_load: failed fread");
453 if (local_header[2] != 0x03 || local_header[3] != 0x04)
459 uint16_t name_len = *(uint16_t*) &local_header[26];
460 std::string vname(name_len,
' ');
461 size_t vname_res = fread(&vname[0],
sizeof(
char), name_len, fp);
463 if (vname_res != name_len)
465 throw std::runtime_error(
"npz_load: failed fread");
468 vname.erase(vname.end() - 4, vname.end());
470 uint16_t extra_field_len = *(uint16_t*) &local_header[28];
471 std::vector<char> extra_field(extra_field_len);
473 if (extra_field_len > 0)
475 size_t efield_res = fread(&extra_field[0],
sizeof(
char), extra_field_len, fp);
477 if (efield_res != extra_field_len)
479 throw std::runtime_error(
"npz_load: failed fread");
483 uint16_t compr_method = *
reinterpret_cast<uint16_t*
>(&local_header[0] + 8);
484 uint32_t compr_bytes = *
reinterpret_cast<uint32_t*
>(&local_header[0] + 18);
485 uint32_t uncompr_bytes = *
reinterpret_cast<uint32_t*
>(&local_header[0] + 22);
487 parse_zip64_sizes(extra_field, compr_bytes, uncompr_bytes);
489 if (vname == varname)
491 NpyArray array = (compr_method == 0) ? load_the_npy_file(
492 fp) : load_the_npz_array(fp, compr_bytes, uncompr_bytes);
499 fseek(fp, compr_bytes, SEEK_CUR);
505 throw std::runtime_error(
"npz_load: Variable name " + varname +
" not found in "
512 FILE* fp = fopen(fname.c_str(),
"rb");
516 throw std::runtime_error(
"npy_load: Unable to open file " + fname);
519 NpyArray arr = load_the_npy_file(fp);
524template<
class typeNumber,
int dim>
525void cnpy::save(
const Eigen::Matrix<typeNumber, Eigen::Dynamic, dim>&
526 mat,
const std::string fname)
528 std::vector<typeNumber> matvec(mat.rows() * mat.cols());
529 std::vector<size_t> shape = {(
unsigned int) mat.rows(), (
unsigned int)mat.cols()};
531 for (
int i = 0; i < mat.rows(); ++i)
533 for (
int j = 0; j < mat.cols(); ++j)
535 matvec[i * mat.cols() + j] = mat(i, j);
539 npy_save(fname, matvec.data(), shape);
542template<
class typeNumber>
543void cnpy::save(
const Eigen::Tensor<typeNumber, 3>&
544 tens,
const std::string fname)
546 typename Eigen::Tensor<typeNumber, 3>::Dimensions dim = tens.dimensions();
549 for (
int k = 0; k < dim.size(); k++)
554 std::vector<typeNumber> matvec(tot);
555 std::vector<size_t> shape = {(
unsigned int) dim[0], (
unsigned int) dim[1], (
unsigned int) dim[2]};
557 for (
int i = 0; i < dim[0]; ++i)
559 for (
int j = 0; j < dim[1]; ++j)
561 for (
int k = 0; k < dim[2]; ++k)
563 matvec[i * dim[2] * dim[1] + j * dim[2] + k] = tens(i, j, k);
568 npy_save(fname, matvec.data(), shape);
571template<
class typeNumber,
int dim>
572Eigen::Matrix<typeNumber, Eigen::Dynamic, dim> cnpy::load(
573 Eigen::Matrix<typeNumber, Eigen::Dynamic, dim>& mat,
574 const std::string fname)
576 NpyArray arr = npy_load(fname);
577 assert(arr.shape.size() == 2);
578 mat.resize(arr.shape[0], arr.shape[1]);
579 std::vector<typeNumber> data = arr.as_vec<typeNumber>();
581 if (!arr.fortran_order)
583 for (
size_t i = 0; i < arr.shape[0]; ++i)
585 for (
size_t j = 0; j < arr.shape[1]; ++j)
587 mat(i, j) = data[arr.shape[1] * i + j];
593 for (
size_t i = 0; i < arr.shape[0]; ++i)
595 for (
size_t j = 0; j < arr.shape[1]; ++j)
597 mat(i, j) = data[arr.shape[0] * j + i];
606template<
typename typeNumber>
607Eigen::Tensor<typeNumber, 3> cnpy::load(Eigen::Tensor<typeNumber, 3>& tens,
608 const std::string fname)
610 NpyArray arr = npy_load(fname);
611 assert(arr.shape.size() == 3);
612 tens.resize(
static_cast<long>(arr.shape[0]),
static_cast<long>(arr.shape[1]),
613 static_cast<long>(arr.shape[2]));
614 std::vector<typeNumber> data = arr.as_vec<typeNumber>();
616 if (!arr.fortran_order)
618 for (
size_t i = 0; i < arr.shape[0]; ++i)
620 for (
size_t j = 0; j < arr.shape[1]; ++j)
622 for (
size_t k = 0; k < arr.shape[2]; ++k)
624 tens(i, j, k) = data[arr.shape[1] * arr.shape[2] * i + j *
632 for (
size_t i = 0; i < arr.shape[0]; ++i)
634 for (
size_t j = 0; j < arr.shape[1]; ++j)
636 for (
size_t k = 0; k < arr.shape[2]; ++k)
638 tens(i, j, k) = (typeNumber) data[arr.shape[0] * arr.shape[1] * k + j *
650Eigen::SparseMatrix<T> cnpy::load(Eigen::SparseMatrix<T>& smatrix,
651 const std::string fname)
653 auto d1 = cnpy::npz_load(fname,
"data");
654 auto d2 = cnpy::npz_load(fname,
"indices");
655 auto d3 = cnpy::npz_load(fname,
"indptr");
656 auto d4 = cnpy::npz_load(fname,
"shape");
657 std::vector<T> data = d1.as_vec<T>();
658 int32_t* indices =
reinterpret_cast<int32_t*
>(d2.data<int32_t>());
659 int32_t* indptr =
reinterpret_cast<int32_t*
>(d3.data<int32_t>());
660 int* shape =
reinterpret_cast<int*
>(d4.data<
int>());
662 M_Assert(d4.shape[0] == 2,
"Method works only with 2-D matrices");
663 rows = (int) shape[0];
664 cols = (int) d3.shape[0] - 1;
665 int nel = d1.shape[0];
666 smatrix.resize(rows, cols);
667 smatrix.reserve(nel);
668 typedef Eigen::Triplet<T> Trip;
669 std::vector<Trip> tripletList;
671 for (
int i = 0; i < cols; ++i)
673 for (
int j = indptr[i]; j < indptr[i + 1]; j++)
675 tripletList.push_back(Trip(indices[j], i, (T) data[j]));
679 smatrix.setFromTriplets(tripletList.begin(), tripletList.end());
684void cnpy::save(
const Eigen::SparseMatrix<T>& mat,
const std::string fname)
686 std::vector<size_t> shape1 = std::vector<size_t> {(
unsigned long) (mat.nonZeros())};
687 std::vector<size_t> shape2 = std::vector<size_t> {(
unsigned long) (mat.outerSize() + 1)};
688 std::vector<size_t> shape3 = std::vector<size_t> {2};
689 std::vector<size_t> shape4 = std::vector<size_t> {256};
690 cnpy::npz_save(fname,
"data", mat.valuePtr(), shape1,
"w");
691 cnpy::npz_save(fname,
"indices", mat.innerIndexPtr(), shape1,
"a");
692 cnpy::npz_save(fname,
"indptr", mat.outerIndexPtr(), shape2,
"a");
693 int64_t t1 = mat.rows();
694 int64_t t2 = mat.cols();
695 int64_t* shape =
new int64_t[2];
696 shape[0] = mat.rows();
697 shape[1] = mat.cols();
698 cnpy::npz_save(fname,
"shape", shape, shape3,
"a");
700 cnpy::npz_save(fname,
"format", & myVar2, shape4,
"a");
703template void cnpy::save(
const Eigen::MatrixXi& mat,
const std::string fname);
704template Eigen::MatrixXi cnpy::load(Eigen::MatrixXi& mat,
705 const std::string fname);
706template void cnpy::save(
const Eigen::MatrixXd& mat,
const std::string fname);
707template Eigen::MatrixXd cnpy::load(Eigen::MatrixXd& mat,
708 const std::string fname);
709template void cnpy::save(
const Eigen::MatrixXf& mat,
const std::string fname);
710template Eigen::MatrixXf cnpy::load(Eigen::MatrixXf& mat,
711 const std::string fname);
712template void cnpy::save(
const Eigen::MatrixXcd& mat,
const std::string fname);
713template Eigen::MatrixXcd cnpy::load(Eigen::MatrixXcd& mat,
714 const std::string fname);
715template void cnpy::save(
const Eigen::VectorXi& mat,
const std::string fname);
716template Eigen::VectorXi cnpy::load(Eigen::VectorXi& mat,
717 const std::string fname);
718template void cnpy::save(
const Eigen::VectorXd& mat,
const std::string fname);
719template Eigen::VectorXd cnpy::load(Eigen::VectorXd& mat,
720 const std::string fname);
721template void cnpy::save(
const Eigen::VectorXf& mat,
const std::string fname);
722template Eigen::VectorXf cnpy::load(Eigen::VectorXf& mat,
723 const std::string fname);
724template void cnpy::save(
const Eigen::VectorXcd& mat,
const std::string fname);
725template Eigen::VectorXcd cnpy::load(Eigen::VectorXcd& mat,
726 const std::string fname);
727template void cnpy::save(
const Eigen::SparseMatrix<double>& mat,
728 const std::string fname);
729template Eigen::SparseMatrix<double> cnpy::load(Eigen::SparseMatrix<double>&
730 smatrix,
const std::string fname);
732template void cnpy::save(
const Eigen::Tensor<int, 3>& mat,
733 const std::string fname);
734template Eigen::Tensor<int, 3> cnpy::load(Eigen::Tensor<int, 3>& tens,
735 const std::string fname);
736template void cnpy::save(
const Eigen::Tensor<double, 3>& mat,
737 const std::string fname);
738template Eigen::Tensor<double, 3> cnpy::load(Eigen::Tensor<double, 3>& tens,
739 const std::string fname);
740template void cnpy::save(
const Eigen::Tensor<float, 3>& mat,
741 const std::string fname);
742template Eigen::Tensor<float, 3> cnpy::load(Eigen::Tensor<float, 3>& tens,
743 const std::string fname);
744template void cnpy::save(
const Eigen::Tensor<std::complex<double>, 3>& mat,
745 const std::string fname);
746template Eigen::Tensor<std::complex<double>, 3> cnpy::load(
747 Eigen::Tensor<std::complex<double>, 3>& tens,
748 const std::string fname);
749#pragma GCC diagnostic pop
Header file of the ITHACAstream class, it contains the implementation of several methods for input ou...