Skip to content

Commit 8881fc1

Browse files
FrancescAltedFrancescAlted
committed
First attempt at diasbling btune by default
1 parent e4a1055 commit 8881fc1

3 files changed

Lines changed: 131 additions & 6 deletions

File tree

‎examples/example_load_copy.c‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright INAOS GmbH, Thalwil, 2019.
3+
* Copyright Francesc Alted, 2019.
4+
*
5+
* All rights reserved.
6+
*
7+
* This software is the confidential and proprietary information of INAOS GmbH
8+
* and Francesc Alted ("Confidential Information"). You shall not disclose such Confidential
9+
* Information and shall use it only in accordance with the terms of the license agreement.
10+
*
11+
*/
12+
13+
#include <libiarray/iarray.h>
14+
#include "iarray_private.h"
15+
16+
17+
18+
int main(void) {
19+
20+
iarray_init();
21+
22+
blosc2_remove_urlpath("example_load_copy.iarr");
23+
24+
ina_stopwatch_t *w;
25+
double elapsed_sec = 0;
26+
INA_STOPWATCH_NEW(-1, -1, &w);
27+
28+
iarray_context_t *ctx;
29+
iarray_config_t cfg = IARRAY_CONFIG_DEFAULTS;
30+
// Activating btune will override compression_level and compression_codec defaults later on
31+
// cfg.btune = true;
32+
iarray_context_new(&cfg, &ctx);
33+
34+
int64_t shape[] = {1024 * 1024};
35+
int8_t ndim = 1;
36+
37+
iarray_dtshape_t dtshape;
38+
dtshape.dtype = IARRAY_DATA_TYPE_FLOAT;
39+
int8_t itemsize = sizeof(double);
40+
dtshape.ndim = ndim;
41+
42+
int64_t nelem = 1;
43+
for (int i = 0; i < ndim; ++i) {
44+
dtshape.shape[i] = shape[i];
45+
nelem *= shape[i];
46+
}
47+
48+
int32_t xchunkshape[] = {512 * 1024};
49+
int32_t xblockshape[] = {128 * 1024};
50+
51+
iarray_storage_t xstorage;
52+
xstorage.contiguous = false;
53+
xstorage.urlpath = NULL;
54+
for (int i = 0; i < ndim; ++i) {
55+
xstorage.chunkshape[i] = xchunkshape[i];
56+
xstorage.blockshape[i] = xblockshape[i];
57+
}
58+
59+
iarray_container_t *x;
60+
IARRAY_RETURN_IF_FAILED(iarray_linspace(ctx, &dtshape, -10, 10, &xstorage, 0, &x));
61+
62+
float cratio = (float)x->catarr->sc->nbytes / (float)x->catarr->sc->cbytes;
63+
printf("orig cratio: %.3f\n", cratio);
64+
printf("orig contiguous: %d\n", x->storage->contiguous);
65+
66+
iarray_storage_t xstorage2;
67+
xstorage2.contiguous = true;
68+
xstorage2.urlpath = NULL;
69+
for (int i = 0; i < ndim; ++i) {
70+
xstorage2.chunkshape[i] = xchunkshape[i] + 1;
71+
xstorage2.blockshape[i] = xblockshape[i] + 1;
72+
}
73+
74+
iarray_container_t *out;
75+
IARRAY_RETURN_IF_FAILED(iarray_copy(ctx, x, false, &xstorage2, 0, &out));
76+
77+
IARRAY_RETURN_IF_FAILED(iarray_container_save(ctx, out, "example_load_copy.iarr"));
78+
79+
cratio = (float)out->catarr->sc->nbytes / (float)out->catarr->sc->cbytes;
80+
printf("copy cratio: %.3f\n", cratio);
81+
printf("copy contiguous: %d\n", out->storage->contiguous);
82+
83+
iarray_container_free(ctx, &x);
84+
// Check whether ctx affects the load process (it does not look like it)
85+
ctx->cfg->compression_level = 0;
86+
ctx->cfg->compression_codec = IARRAY_COMPRESSION_ZLIB;
87+
88+
INA_STOPWATCH_START(w);
89+
IARRAY_RETURN_IF_FAILED(iarray_container_load(ctx, "example_load_copy.iarr", &x));
90+
INA_STOPWATCH_STOP(w);
91+
INA_MUST_SUCCEED(ina_stopwatch_duration(w, &elapsed_sec));
92+
printf("Time: %.4f s\n", elapsed_sec);
93+
94+
cratio = (float)x->catarr->sc->nbytes / (float)x->catarr->sc->cbytes;
95+
printf("load cratio: %.3f\n", cratio);
96+
printf("load contiguous: %d\n", x->storage->contiguous);
97+
98+
iarray_container_free(ctx, &x);
99+
ctx->cfg->compression_level = 0;
100+
ctx->cfg->compression_codec = IARRAY_COMPRESSION_ZLIB;
101+
102+
INA_STOPWATCH_START(w);
103+
IARRAY_RETURN_IF_FAILED(iarray_container_load(ctx, "example_load_copy.iarr", &x));
104+
INA_STOPWATCH_STOP(w);
105+
INA_MUST_SUCCEED(ina_stopwatch_duration(w, &elapsed_sec));
106+
printf("Time: %.4f s\n", elapsed_sec);
107+
108+
cratio = (float)x->catarr->sc->nbytes / (float)x->catarr->sc->cbytes;
109+
printf("load2 cratio: %.3f\n", cratio);
110+
printf("load2 contiguous: %d\n", x->storage->contiguous);
111+
112+
int64_t buflen = nelem * itemsize;
113+
uint8_t *buf = malloc(buflen);
114+
115+
IARRAY_RETURN_IF_FAILED(iarray_to_buffer(ctx, x, buf, buflen));
116+
117+
free(buf);
118+
iarray_container_free(ctx, &x);
119+
iarray_context_free(&ctx);
120+
121+
INA_STOPWATCH_FREE(&w);
122+
return 0;
123+
}

‎include/libiarray/iarray.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ static const iarray_config_t IARRAY_CONFIG_DEFAULTS = {
275275
.eval_method = IARRAY_EVAL_METHOD_ITERCHUNK,
276276
.max_num_threads = 1,
277277
.fp_mantissa_bits = 0,
278-
.btune = true,
278+
.btune = false,
279279
};
280280

281281
static const iarray_config_t IARRAY_CONFIG_NO_COMPRESSION = {
@@ -287,7 +287,8 @@ static const iarray_config_t IARRAY_CONFIG_NO_COMPRESSION = {
287287
.filter_flags = 0,
288288
.eval_method = 0,
289289
.max_num_threads = 1,
290-
.fp_mantissa_bits = 0
290+
.fp_mantissa_bits = 0,
291+
.btune = false,
291292
};
292293

293294
INA_API(ina_rc_t) iarray_init(void);

tests/test_constructor_buffer_btune.c renamed to tests/test_btune_favor.c

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ test_btune_favor(iarray_config_t *cfg, iarray_data_type_t dtype, size_t type_siz
8484
blosc2_remove_urlpath(xstore.urlpath);
8585

8686
iarray_context_free(&ctx);
87-
87+
8888
return INA_SUCCESS;
8989
}
9090

@@ -98,6 +98,7 @@ INA_TEST_SETUP(btune_favor)
9898
iarray_init();
9999
data->cbytes = 0;
100100
data->cfg = IARRAY_CONFIG_DEFAULTS;
101+
data->cfg.btune = true; // activate btune (deactivated by default)
101102
}
102103

103104
INA_TEST_TEARDOWN(btune_favor)
@@ -114,9 +115,9 @@ INA_TEST_FIXTURE(btune_favor, cratio)
114115
int64_t shape[] = {367, 333};
115116
int64_t cshape[] = {70, 91};
116117
int64_t bshape[] = {12, 25};
117-
118+
118119
data->cfg.compression_favor = IARRAY_COMPRESSION_FAVOR_CRATIO;
119-
120+
120121
INA_TEST_ASSERT_SUCCEED(test_btune_favor(&data->cfg, dtype, type_size, ndim, shape, cshape,
121122
bshape, &data->cbytes, false, NULL));
122123
}
@@ -139,7 +140,7 @@ INA_TEST_FIXTURE(btune_favor, balance)
139140

140141
INA_TEST_FIXTURE(btune_favor, speed)
141142
{
142-
iarray_data_type_t dtype = IARRAY_DATA_TYPE_DOUBLE;
143+
iarray_data_type_t dtype = IARRAY_DATA_TYPE_FLOAT;
143144
size_t type_size = sizeof(double);
144145

145146
int8_t ndim = 2;

0 commit comments

Comments
 (0)