From 59bff1c49ad87068bdee63f3b398411d196f7784 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Tue, 11 Jun 2019 16:16:27 -0600 Subject: [PATCH 01/10] add float64 for dataset creation --- main.c | 2 +- test_attributes.c | 11 ++++++++++- z5wrapper.cc | 32 +++++++++++++++++++++++++++++++- z5wrapper.h | 6 +++++- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/main.c b/main.c index 2d438bb..8da8a92 100644 --- a/main.c +++ b/main.c @@ -34,7 +34,7 @@ int main() { char* arrayName = "test_c.z5"; int cusezlib = 1; int level = 1; - z5CreateFloatDataset(arrayName, ndim, shape, chunks, cusezlib, level); + z5CreateFloat32Dataset(arrayName, ndim, shape, chunks, cusezlib, level); z5WriteFloatSubarray(arrayName, data1, ndim, chunks, offset); z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset); for (int i = 0; i < chunks[0]; i++){ diff --git a/test_attributes.c b/test_attributes.c index 06d220c..793e05d 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -11,6 +11,7 @@ void my_create_dataset(char* arrayName) float data1[10][10][10]; float data2[10][10][10]; float rdata[10][10][10]; + double ddata[10][10][10]; long long int idata1[10][10][10]; long long int idata2[10][10][10]; long long int irdata[10][10][10]; @@ -34,7 +35,9 @@ void my_create_dataset(char* arrayName) //char* arrayName = "test_c.z5"; int cusezlib = 1; int level = 1; - z5CreateFloatDataset(arrayName, ndim, shape, chunks, cusezlib, level); + z5CreateFloat32Dataset(arrayName, ndim, shape, chunks, cusezlib, level); + + z5WriteFloatSubarray(arrayName, data1, ndim, chunks, offset); z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset); for (int i = 0; i < chunks[0]; i++){ @@ -92,6 +95,12 @@ void my_create_dataset(char* arrayName) z5readAttributesWithKeys(arrayName, keysinput, 2); printf("after assert\n"); + + printf("testing different array type\n"); + char* float64arrayName = "new_file/group1/float64variables"; + z5CreateFloat64Dataset(float64arrayName, ndim, shape, chunks, cusezlib, level); + + z5WriteFloat64Subarray(float64arrayName, ddata, ndim, chunks, offset); } void test_create_file() diff --git a/z5wrapper.cc b/z5wrapper.cc index ce536e7..633f422 100644 --- a/z5wrapper.cc +++ b/z5wrapper.cc @@ -25,7 +25,7 @@ namespace z5 { handle::Group cGroup(path_s); createGroup(cGroup, asZarr); } - void z5CreateFloatDataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + void z5CreateFloat32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); std::vector dtype({"float32"}); std::vector shape_v(shape, shape + ndim); @@ -44,6 +44,25 @@ namespace z5 { } + void z5CreateFloat64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + std::vector dtype({"float64"}); + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata floatMeta(types::Datatype::float64,shape_v,chunks_v,asZarr); + if (cuseZlib) { + floatMeta.compressor = types::zlib; + floatMeta.compressionOptions["useZlib"] = true; + floatMeta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, floatMeta); + + } + void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); std::vector dtype({"long long int"}); @@ -81,6 +100,17 @@ namespace z5 { multiarray::writeSubarray(ds,adp_array,offset_v.begin()); } + void z5WriteFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + xt::xarray adp_array=xt::adapt(array,size,xt::no_ownership(),shape_v); + std::vector offset_v(offset,offset + ndim); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds =openDataset(path_s); diff --git a/z5wrapper.h b/z5wrapper.h index 27367f9..47241d6 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -26,10 +26,14 @@ namespace z5 { void z5CreateGroup(char* path); - void z5CreateFloatDataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5CreateFloat32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + + void z5CreateFloat64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); void z5WriteFloatSubarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5WriteFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadFloatSubarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset); void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); From 0a5cd2ea033ddcbf21902e2ade3965024ace4ab3 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Tue, 11 Jun 2019 16:31:09 -0600 Subject: [PATCH 02/10] fixing float32 --- main.c | 4 ++-- test_attributes.c | 4 ++-- z5wrapper.cc | 2 +- z5wrapper.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/main.c b/main.c index 8da8a92..41f97ff 100644 --- a/main.c +++ b/main.c @@ -35,7 +35,7 @@ int main() { int cusezlib = 1; int level = 1; z5CreateFloat32Dataset(arrayName, ndim, shape, chunks, cusezlib, level); - z5WriteFloatSubarray(arrayName, data1, ndim, chunks, offset); + z5WriteFloat32Subarray(arrayName, data1, ndim, chunks, offset); z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset); for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++) @@ -43,7 +43,7 @@ int main() { assert(data1[i][j][k] == rdata[i][j][k]); printf("data1 = %f\n",data1[i][0][0]); } - z5WriteFloatSubarray("test_c.z5", data2, ndim, chunks, offset2); + z5WriteFloat32Subarray("test_c.z5", data2, ndim, chunks, offset2); z5ReadFloatSubarray("test_c.z5", rdata, ndim, chunks, offset2); printf("after read float\n"); for (int i = 0; i < chunks[0]; i++){ diff --git a/test_attributes.c b/test_attributes.c index 793e05d..d694bee 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -38,7 +38,7 @@ void my_create_dataset(char* arrayName) z5CreateFloat32Dataset(arrayName, ndim, shape, chunks, cusezlib, level); - z5WriteFloatSubarray(arrayName, data1, ndim, chunks, offset); + z5WriteFloat32Subarray(arrayName, data1, ndim, chunks, offset); z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset); for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++) @@ -46,7 +46,7 @@ void my_create_dataset(char* arrayName) assert(data1[i][j][k] == rdata[i][j][k]); printf("data1 = %f\n",data1[i][0][0]); } - z5WriteFloatSubarray(arrayName, data2, ndim, chunks, offset2); + z5WriteFloat32Subarray(arrayName, data2, ndim, chunks, offset2); z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset2); printf("after read float\n"); for (int i = 0; i < chunks[0]; i++){ diff --git a/z5wrapper.cc b/z5wrapper.cc index 633f422..59fcb93 100644 --- a/z5wrapper.cc +++ b/z5wrapper.cc @@ -88,7 +88,7 @@ namespace z5 { //std::cout<<"long long int "< Date: Tue, 11 Jun 2019 18:16:38 -0600 Subject: [PATCH 03/10] fixing float 64 and add tests --- main.c | 4 +-- test_attributes.c | 20 ++++++++++---- z5wrapper.cc | 70 +++++++++++++++++++++++++++++------------------ z5wrapper.h | 13 ++++----- 4 files changed, 67 insertions(+), 40 deletions(-) diff --git a/main.c b/main.c index 41f97ff..9724f27 100644 --- a/main.c +++ b/main.c @@ -36,7 +36,7 @@ int main() { int level = 1; z5CreateFloat32Dataset(arrayName, ndim, shape, chunks, cusezlib, level); z5WriteFloat32Subarray(arrayName, data1, ndim, chunks, offset); - z5ReadFloatSubarray(arrayName, rdata, ndim, chunks, offset); + z5ReadFloat32Subarray(arrayName, rdata, ndim, chunks, offset); for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++) for (int k = 0; k shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + xt::xarray adp_array=xt::adapt(array,size,xt::no_ownership(),shape_v); + std::vector offset_v(offset,offset + ndim); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadFloat32Subarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + void z5CreateFloat64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); std::vector dtype({"float64"}); @@ -63,6 +90,21 @@ namespace z5 { } + void z5ReadFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); std::vector dtype({"long long int"}); @@ -88,18 +130,6 @@ namespace z5 { //std::cout<<"long long int "< shape_v(shape,shape + ndim); - for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) - size=size*(*i); - xt::xarray adp_array=xt::adapt(array,size,xt::no_ownership(),shape_v); - std::vector offset_v(offset,offset + ndim); - multiarray::writeSubarray(ds,adp_array,offset_v.begin()); - } - void z5WriteFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds =openDataset(path_s); @@ -111,6 +141,7 @@ namespace z5 { std::vector offset_v(offset,offset + ndim); multiarray::writeSubarray(ds,adp_array,offset_v.begin()); } + void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds =openDataset(path_s); @@ -126,20 +157,6 @@ namespace z5 { multiarray::writeSubarray(ds,adp_array,offset_v.begin()); } - void z5ReadFloatSubarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset) { - std::string path_s(path); - auto ds = openDataset(path_s); - using vec_type = std::vector; - size_t size = 1; - std::vector shape_v(shape,shape + ndim); - for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) - size*=(*i); - using shape_type = std::vector; - shape_type s(shape,shape+ndim); - std::vector offset_v(offset,offset + ndim); - auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); - multiarray::readSubarray(ds,adp_array,offset_v.begin()); - } void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds = openDataset(path_s); @@ -154,6 +171,7 @@ namespace z5 { auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); multiarray::readSubarray(ds,adp_array,offset_v.begin()); } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); diff --git a/z5wrapper.h b/z5wrapper.h index ac9b4e1..dd9760b 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -26,20 +26,19 @@ namespace z5 { void z5CreateGroup(char* path); + // float 32 void z5CreateFloat32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); - - void z5CreateFloat64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); - void z5WriteFloat32Subarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadFloat32Subarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset); + // float 64 / double + void z5CreateFloat64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); void z5WriteFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset); - void z5ReadFloatSubarray(char *path, float *array, unsigned int ndim, size_t *shape, size_t *offset); - + // int64 void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); - void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); - void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); size_t z5GetFileSize(char *path); From a546cda4383295cb83517c59fa99939ce217bc5a Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Tue, 11 Jun 2019 19:01:53 -0600 Subject: [PATCH 04/10] add int8 data type --- test_attributes.c | 28 ++++++++++------ z5wrapper.cc | 85 +++++++++++++++++++++++++++++++++++------------ z5wrapper.h | 5 +++ 3 files changed, 86 insertions(+), 32 deletions(-) diff --git a/test_attributes.c b/test_attributes.c index 31b7a4d..f70f9c4 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -16,36 +16,29 @@ void my_create_dataset(char* arrayName) long long int idata1[10][10][10]; long long int idata2[10][10][10]; long long int irdata[10][10][10]; + int8_t i8data[10][10][10]; + int8_t ri8data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ for (int k = 0; k shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + xt::xarray adp_array=xt::adapt(array,size,xt::no_ownership(),shape_v); + std::vector offset_v(offset,offset + ndim); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + void z5ReadFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds = openDataset(path_s); @@ -107,69 +119,98 @@ namespace z5 { void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); - std::vector dtype({"long long int"}); std::vector shape_v(shape, shape + ndim); std::vector chunks_v(chunks, chunks + ndim); bool asZarr = true; - + DatasetMetadata int64Meta(types::Datatype::int64,shape_v,chunks_v,asZarr); if (cuseZlib) { - int64Meta.compressor = types::zlib; - int64Meta.compressionOptions["useZlib"] = true; - int64Meta.compressionOptions["level"] = level; + int64Meta.compressor = types::zlib; + int64Meta.compressionOptions["useZlib"] = true; + int64Meta.compressionOptions["level"] = level; } handle::Dataset handle_(path_s); handle_.createDir(); writeMetadata(handle_, int64Meta); - long long int idata1; - int64_t data1; - long int data2; - //std::cout<<"int64 "<; size_t size = 1; std::vector shape_v(shape,shape + ndim); for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) size=size*(*i); - xt::xarray adp_array=xt::adapt(array,size,xt::no_ownership(),shape_v); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); std::vector offset_v(offset,offset + ndim); - multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); } - void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { + void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); - auto ds =openDataset(path_s); + auto ds = openDataset(path_s); using vec_type = std::vector; size_t size = 1; std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + + void z5CreateInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata int8Meta(types::Datatype::int8,shape_v,chunks_v,asZarr); + if (cuseZlib) { + int8Meta.compressor = types::zlib; + int8Meta.compressionOptions["useZlib"] = true; + int8Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, int8Meta); + } + + void z5WriteInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) size=size*(*i); using shape_type = std::vector; shape_type s(shape,shape+ndim); std::vector offset_v(offset,offset + ndim); auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); - multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); } - void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { + void z5ReadInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset) { std::string path_s(path); auto ds = openDataset(path_s); - using vec_type = std::vector; + using vec_type = std::vector; size_t size = 1; - std::vector shape_v(shape,shape + ndim); + std::vector shape_v(shape,shape + ndim); for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) size*=(*i); using shape_type = std::vector; shape_type s(shape,shape+ndim); std::vector offset_v(offset,offset + ndim); auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); - multiarray::readSubarray(ds,adp_array,offset_v.begin()); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); } size_t z5GetFileSize(char *path){ diff --git a/z5wrapper.h b/z5wrapper.h index dd9760b..740254d 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -36,6 +36,11 @@ namespace z5 { void z5WriteFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset); void z5ReadFloat64Subarray(char *path, double *array, unsigned int ndim, size_t *shape, size_t *offset); + // int8_t + void z5CreateInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset); + // int64 void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); From 68acd354523d26549e86bfba4cd236e812b36e61 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Tue, 11 Jun 2019 19:09:24 -0600 Subject: [PATCH 05/10] add int16 datatype for dataset --- test_attributes.c | 17 +++++++++++++++++ z5wrapper.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++++ z5wrapper.h | 5 +++++ 3 files changed, 70 insertions(+) diff --git a/test_attributes.c b/test_attributes.c index f70f9c4..b8fce32 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -18,6 +18,8 @@ void my_create_dataset(char* arrayName) long long int irdata[10][10][10]; int8_t i8data[10][10][10]; int8_t ri8data[10][10][10]; + int16_t i16data[10][10][10]; + int16_t ri16data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -27,6 +29,7 @@ void my_create_dataset(char* arrayName) data2[i][j][k]=rand()%1000+1; idata2[i][j][k]=rand()%1000+1; i8data[i][j][k]=rand()%1000+1; + i16data[i][j][k]=rand()%1000+1; } } } @@ -106,6 +109,7 @@ void my_create_dataset(char* arrayName) } printf("===successfully read Float64Dataset===\n"); + char* int8arrayName = "new_file/group1/int8variables"; z5CreateInt8Dataset(int8arrayName, ndim, shape, chunks, cusezlib, level); printf("===successfully creat Int8Dataset===\n"); @@ -119,6 +123,19 @@ void my_create_dataset(char* arrayName) } printf("===successfully read Int8Dataset===\n"); + char* int16arrayName = "new_file/group1/int16variables"; + z5CreateInt16Dataset(int16arrayName, ndim, shape, chunks, cusezlib, level); + printf("===successfully creat Int16Dataset===\n"); + z5WriteInt16Subarray(int16arrayName, i16data, ndim, chunks, offset); + printf("===successfully write Int16Dataset===\n"); + z5ReadInt16Subarray(int16arrayName, ri16data, ndim, chunks, offset); + for (int i = 0; i < chunks[0]; i++){ + for (int j = 0; j < chunks[1]; j++) + for (int k = 0; k(ds,adp_array,offset_v.begin()); } + void z5CreateInt16Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata int16Meta(types::Datatype::int16,shape_v,chunks_v,asZarr); + if (cuseZlib) { + int16Meta.compressor = types::zlib; + int16Meta.compressionOptions["useZlib"] = true; + int16Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, int16Meta); + } + + void z5WriteInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); diff --git a/z5wrapper.h b/z5wrapper.h index 740254d..3173057 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -41,6 +41,11 @@ namespace z5 { void z5WriteInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset); void z5ReadInt8Subarray(char *path, int8_t *array, unsigned int ndim, size_t *shape, size_t *offset); + // int16_t + void z5CreateInt16Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset); + // int64 void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); From 084ab3eb254ec86d185278677cb0e8b078700c47 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Wed, 12 Jun 2019 09:17:05 -0600 Subject: [PATCH 06/10] add int32 type for dataset --- test_attributes.c | 16 ++++++++++++++++ z5wrapper.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++++ z5wrapper.h | 5 +++++ 3 files changed, 69 insertions(+) diff --git a/test_attributes.c b/test_attributes.c index b8fce32..6b05c38 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -20,6 +20,8 @@ void my_create_dataset(char* arrayName) int8_t ri8data[10][10][10]; int16_t i16data[10][10][10]; int16_t ri16data[10][10][10]; + int32_t i32data[10][10][10]; + int32_t ri32data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -30,6 +32,7 @@ void my_create_dataset(char* arrayName) idata2[i][j][k]=rand()%1000+1; i8data[i][j][k]=rand()%1000+1; i16data[i][j][k]=rand()%1000+1; + i32data[10][10][10]; } } } @@ -136,6 +139,19 @@ void my_create_dataset(char* arrayName) } printf("===successfully read Int16Dataset===\n"); + char* int32arrayName = "new_file/group1/int32variables"; + z5CreateInt32Dataset(int32arrayName, ndim, shape, chunks, cusezlib, level); + printf("===successfully creat Int32Dataset===\n"); + z5WriteInt32Subarray(int32arrayName, i32data, ndim, chunks, offset); + printf("===successfully write Int32Dataset===\n"); + z5ReadInt32Subarray(int32arrayName, ri32data, ndim, chunks, offset); + for (int i = 0; i < chunks[0]; i++){ + for (int j = 0; j < chunks[1]; j++) + for (int k = 0; k(ds,adp_array,offset_v.begin()); } + void z5CreateInt32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata int32Meta(types::Datatype::int32,shape_v,chunks_v,asZarr); + if (cuseZlib) { + int32Meta.compressor = types::zlib; + int32Meta.compressionOptions["useZlib"] = true; + int32Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, int32Meta); + } + + void z5WriteInt32Subarray(char *path, int32_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadInt32Subarray(char *path, int32_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); diff --git a/z5wrapper.h b/z5wrapper.h index 3173057..c3d0042 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -46,6 +46,11 @@ namespace z5 { void z5WriteInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset); void z5ReadInt16Subarray(char *path, int16_t *array, unsigned int ndim, size_t *shape, size_t *offset); + // int32_t + void z5CreateInt32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteInt32Subarray(char *path, int32_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadInt32Subarray(char *path, int32_t *array, unsigned int ndim, size_t *shape, size_t *offset); + // int64 void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); From 88291638230e8f573179bda722b0db470281528c Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Wed, 12 Jun 2019 09:35:55 -0600 Subject: [PATCH 07/10] add uint8 datatype fore dataset --- test_attributes.c | 44 +++++++++++++++++++++++++++++++++++++++++- z5wrapper.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++ z5wrapper.h | 22 +++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/test_attributes.c b/test_attributes.c index 6b05c38..c829adf 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -22,6 +22,8 @@ void my_create_dataset(char* arrayName) int16_t ri16data[10][10][10]; int32_t i32data[10][10][10]; int32_t ri32data[10][10][10]; + uint8_t ui8data[10][10][10]; + uint8_t rui8data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -32,7 +34,8 @@ void my_create_dataset(char* arrayName) idata2[i][j][k]=rand()%1000+1; i8data[i][j][k]=rand()%1000+1; i16data[i][j][k]=rand()%1000+1; - i32data[10][10][10]; + i32data[i][j][k]=rand()%1000+1; + ui8data[i][j][k]=rand()%1000+1; } } } @@ -152,6 +155,45 @@ void my_create_dataset(char* arrayName) } printf("===successfully read Int32Dataset===\n"); + char* uint8arrayName = "new_file/group1/uint8variables"; + z5CreateUInt8Dataset(uint8arrayName, ndim, shape, chunks, cusezlib, level); + printf("===successfully creat UInt8Dataset===\n"); + z5WriteUInt8Subarray(uint8arrayName, ui8data, ndim, chunks, offset); + printf("===successfully write UInt8Dataset===\n"); + z5ReadUInt8Subarray(uint8arrayName, rui8data, ndim, chunks, offset); + for (int i = 0; i < chunks[0]; i++){ + for (int j = 0; j < chunks[1]; j++) + for (int k = 0; k(ds,adp_array,offset_v.begin()); } + + void z5CreateUInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata uint8Meta(types::Datatype::uint8,shape_v,chunks_v,asZarr); + if (cuseZlib) { + uint8Meta.compressor = types::zlib; + uint8Meta.compressionOptions["useZlib"] = true; + uint8Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, uint8Meta); + } + + void z5WriteUInt8Subarray(char *path, uint8_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadUInt8Subarray(char *path, uint8_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); diff --git a/z5wrapper.h b/z5wrapper.h index c3d0042..7ef6029 100644 --- a/z5wrapper.h +++ b/z5wrapper.h @@ -18,6 +18,8 @@ #include "z5/types/types.hxx" #endif +#include + #ifdef __cplusplus namespace z5 { extern "C" { @@ -56,6 +58,26 @@ namespace z5 { void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset); + // uint8_t + void z5CreateUInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteUInt8Subarray(char *path, uint8_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadUInt8Subarray(char *path, uint8_t *array, unsigned int ndim, size_t *shape, size_t *offset); + + // uint16_t + void z5CreateUInt16Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteUInt16Subarray(char *path, uint16_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadUInt16Subarray(char *path, uint16_t *array, unsigned int ndim, size_t *shape, size_t *offset); + + // uint32_t + void z5CreateUInt32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteUInt32Subarray(char *path, uint32_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadUInt32Subarray(char *path, uint32_t *array, unsigned int ndim, size_t *shape, size_t *offset); + + // uint64 + void z5CreateUInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level); + void z5WriteUInt64Subarray(char *path, uint64_t *array, unsigned int ndim, size_t *shape, size_t *offset); + void z5ReadUInt64Subarray(char *path, uint64_t *array, unsigned int ndim, size_t *shape, size_t *offset); + size_t z5GetFileSize(char *path); void z5Delete(char *path ); From d808ed01e2cfba5efc5a595a0a015171f26937ba Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Wed, 12 Jun 2019 09:43:43 -0600 Subject: [PATCH 08/10] add unit16 datatype --- test_attributes.c | 27 ++++++++++++++------------ z5wrapper.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/test_attributes.c b/test_attributes.c index c829adf..4512935 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -24,6 +24,8 @@ void my_create_dataset(char* arrayName) int32_t ri32data[10][10][10]; uint8_t ui8data[10][10][10]; uint8_t rui8data[10][10][10]; + uint16_t ui16data[10][10][10]; + uint16_t rui16data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -36,6 +38,7 @@ void my_create_dataset(char* arrayName) i16data[i][j][k]=rand()%1000+1; i32data[i][j][k]=rand()%1000+1; ui8data[i][j][k]=rand()%1000+1; + ui16data[i][j][k]=rand()%1000+1; } } } @@ -168,18 +171,18 @@ void my_create_dataset(char* arrayName) } printf("===successfully read UInt8Dataset===\n"); -// char* int16arrayName = "new_file/group1/int16variables"; -// z5CreateInt16Dataset(int16arrayName, ndim, shape, chunks, cusezlib, level); -// printf("===successfully creat Int16Dataset===\n"); -// z5WriteInt16Subarray(int16arrayName, i16data, ndim, chunks, offset); -// printf("===successfully write Int16Dataset===\n"); -// z5ReadInt16Subarray(int16arrayName, ri16data, ndim, chunks, offset); -// for (int i = 0; i < chunks[0]; i++){ -// for (int j = 0; j < chunks[1]; j++) -// for (int k = 0; k(ds,adp_array,offset_v.begin()); } + void z5CreateUInt16Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata uint16Meta(types::Datatype::uint16,shape_v,chunks_v,asZarr); + if (cuseZlib) { + uint16Meta.compressor = types::zlib; + uint16Meta.compressionOptions["useZlib"] = true; + uint16Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, uint16Meta); + } + + void z5WriteUInt16Subarray(char *path, uint16_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadUInt16Subarray(char *path, uint16_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); From 9b7b3aa50c0bf927682b07eb09d7064d53849db5 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Wed, 12 Jun 2019 09:53:53 -0600 Subject: [PATCH 09/10] add unit32 datatype --- test_attributes.c | 29 +++++----- z5wrapper.cc | 143 ++++++++++++++++++++++++++++++---------------- 2 files changed, 111 insertions(+), 61 deletions(-) diff --git a/test_attributes.c b/test_attributes.c index 4512935..689d5d1 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -26,6 +26,8 @@ void my_create_dataset(char* arrayName) uint8_t rui8data[10][10][10]; uint16_t ui16data[10][10][10]; uint16_t rui16data[10][10][10]; + uint32_t ui32data[10][10][10]; + uint32_t rui32data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -39,6 +41,7 @@ void my_create_dataset(char* arrayName) i32data[i][j][k]=rand()%1000+1; ui8data[i][j][k]=rand()%1000+1; ui16data[i][j][k]=rand()%1000+1; + ui32data[i][j][k]=rand()%1000+1; } } } @@ -183,19 +186,19 @@ void my_create_dataset(char* arrayName) assert(ui16data[i][j][k] == rui16data[i][j][k]); } printf("===successfully read UInt16Dataset===\n"); -// -// char* int32arrayName = "new_file/group1/int32variables"; -// z5CreateInt32Dataset(int32arrayName, ndim, shape, chunks, cusezlib, level); -// printf("===successfully creat Int32Dataset===\n"); -// z5WriteInt32Subarray(int32arrayName, i32data, ndim, chunks, offset); -// printf("===successfully write Int32Dataset===\n"); -// z5ReadInt32Subarray(int32arrayName, ri32data, ndim, chunks, offset); -// for (int i = 0; i < chunks[0]; i++){ -// for (int j = 0; j < chunks[1]; j++) -// for (int k = 0; k(ds,adp_array,offset_v.begin()); } - void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { - std::string path_s(path); - - std::vector shape_v(shape, shape + ndim); - std::vector chunks_v(chunks, chunks + ndim); - bool asZarr = true; - - DatasetMetadata int64Meta(types::Datatype::int64,shape_v,chunks_v,asZarr); - if (cuseZlib) { - int64Meta.compressor = types::zlib; - int64Meta.compressionOptions["useZlib"] = true; - int64Meta.compressionOptions["level"] = level; - } - handle::Dataset handle_(path_s); - handle_.createDir(); - writeMetadata(handle_, int64Meta); - } - - void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { - std::string path_s(path); - auto ds =openDataset(path_s); - using vec_type = std::vector; - size_t size = 1; - std::vector shape_v(shape,shape + ndim); - for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) - size=size*(*i); - using shape_type = std::vector; - shape_type s(shape,shape+ndim); - std::vector offset_v(offset,offset + ndim); - auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); - multiarray::writeSubarray(ds,adp_array,offset_v.begin()); - } - - void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { - std::string path_s(path); - auto ds = openDataset(path_s); - using vec_type = std::vector; - size_t size = 1; - std::vector shape_v(shape,shape + ndim); - for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) - size*=(*i); - using shape_type = std::vector; - shape_type s(shape,shape+ndim); - std::vector offset_v(offset,offset + ndim); - auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); - multiarray::readSubarray(ds,adp_array,offset_v.begin()); - } - void z5CreateInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); @@ -309,6 +261,53 @@ namespace z5 { multiarray::readSubarray(ds,adp_array,offset_v.begin()); } + void z5CreateInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata int64Meta(types::Datatype::int64,shape_v,chunks_v,asZarr); + if (cuseZlib) { + int64Meta.compressor = types::zlib; + int64Meta.compressionOptions["useZlib"] = true; + int64Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, int64Meta); + } + + void z5WriteInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadInt64Subarray(char *path, long long int *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } void z5CreateUInt8Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { std::string path_s(path); @@ -406,6 +405,54 @@ namespace z5 { multiarray::readSubarray(ds,adp_array,offset_v.begin()); } + void z5CreateUInt32Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata uint32Meta(types::Datatype::uint32,shape_v,chunks_v,asZarr); + if (cuseZlib) { + uint32Meta.compressor = types::zlib; + uint32Meta.compressionOptions["useZlib"] = true; + uint32Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, uint32Meta); + } + + void z5WriteUInt32Subarray(char *path, uint32_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadUInt32Subarray(char *path, uint32_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary); From 7e1c08f2770fee9c41c7695bf25175aedf13fb83 Mon Sep 17 00:00:00 2001 From: Weile Wei Date: Wed, 12 Jun 2019 10:00:34 -0600 Subject: [PATCH 10/10] add uint64 datatype --- test_attributes.c | 19 +++++++++++++++++++ z5wrapper.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/test_attributes.c b/test_attributes.c index 689d5d1..4db2bfc 100644 --- a/test_attributes.c +++ b/test_attributes.c @@ -22,12 +22,16 @@ void my_create_dataset(char* arrayName) int16_t ri16data[10][10][10]; int32_t i32data[10][10][10]; int32_t ri32data[10][10][10]; + int64_t i64data[10][10][10]; + int64_t ri64data[10][10][10]; uint8_t ui8data[10][10][10]; uint8_t rui8data[10][10][10]; uint16_t ui16data[10][10][10]; uint16_t rui16data[10][10][10]; uint32_t ui32data[10][10][10]; uint32_t rui32data[10][10][10]; + uint64_t ui64data[10][10][10]; + uint64_t rui64data[10][10][10]; unsigned int ndim = 3; for (int i = 0; i < chunks[0]; i++){ for (int j = 0; j < chunks[1]; j++){ @@ -39,9 +43,11 @@ void my_create_dataset(char* arrayName) i8data[i][j][k]=rand()%1000+1; i16data[i][j][k]=rand()%1000+1; i32data[i][j][k]=rand()%1000+1; + i64data[i][j][k]=rand()%1000+1; ui8data[i][j][k]=rand()%1000+1; ui16data[i][j][k]=rand()%1000+1; ui32data[i][j][k]=rand()%1000+1; + ui64data[i][j][k]=rand()%1000+1; } } } @@ -200,6 +206,19 @@ void my_create_dataset(char* arrayName) } printf("===successfully read UInt32Dataset===\n"); + char* uint64arrayName = "new_file/group1/uint64variables"; + z5CreateUInt64Dataset(uint64arrayName, ndim, shape, chunks, cusezlib, level); + printf("===successfully creat UInt64Dataset===\n"); + z5WriteUInt64Subarray(uint64arrayName, ui64data, ndim, chunks, offset); + printf("===successfully write UInt64Dataset===\n"); + z5ReadUInt64Subarray(uint64arrayName, rui64data, ndim, chunks, offset); + for (int i = 0; i < chunks[0]; i++){ + for (int j = 0; j < chunks[1]; j++) + for (int k = 0; k(ds,adp_array,offset_v.begin()); } + void z5CreateUInt64Dataset(char *path, unsigned int ndim, size_t *shape, size_t *chunks, int cuseZlib, int level) { + std::string path_s(path); + + std::vector shape_v(shape, shape + ndim); + std::vector chunks_v(chunks, chunks + ndim); + bool asZarr = true; + + DatasetMetadata uint64Meta(types::Datatype::uint64,shape_v,chunks_v,asZarr); + if (cuseZlib) { + uint64Meta.compressor = types::zlib; + uint64Meta.compressionOptions["useZlib"] = true; + uint64Meta.compressionOptions["level"] = level; + } + handle::Dataset handle_(path_s); + handle_.createDir(); + writeMetadata(handle_, uint64Meta); + } + + void z5WriteUInt64Subarray(char *path, uint64_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds =openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size=size*(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::writeSubarray(ds,adp_array,offset_v.begin()); + } + + void z5ReadUInt64Subarray(char *path, uint64_t *array, unsigned int ndim, size_t *shape, size_t *offset) { + std::string path_s(path); + auto ds = openDataset(path_s); + using vec_type = std::vector; + size_t size = 1; + std::vector shape_v(shape,shape + ndim); + for (std::vector::const_iterator i = shape_v.begin(); i != shape_v.end(); ++i) + size*=(*i); + using shape_type = std::vector; + shape_type s(shape,shape+ndim); + std::vector offset_v(offset,offset + ndim); + auto adp_array=xt::adapt(array,size,xt::no_ownership(),s); + multiarray::readSubarray(ds,adp_array,offset_v.begin()); + } + size_t z5GetFileSize(char *path){ std::string path_s(path); fs::ifstream file(path_s, std::ios::binary);