Skip to content

Commit

Permalink
Switch mean over to internal accumulation state
Browse files Browse the repository at this point in the history
  • Loading branch information
@e-n-f
e-n-f committed Jan 30, 2025
1 parent 9c299ba commit bc6b5f3
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 16 deletions.
18 changes: 6 additions & 12 deletions attribute.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,19 +120,13 @@ static void preserve_attribute1(attribute_op const &op, std::string const &key,
}

case op_mean: {
auto state = attribute_accum_state.find(key);
if (state == attribute_accum_state.end()) {
accum_state s;
s.sum = full_values[i].to_double() + val.to_double();
s.count = 2;
attribute_accum_state.insert(std::pair<std::string, accum_state>(key, s));

full_values[i] = (s.sum / s.count);
size_t count = full_values[i].get_count();
if (count <= 1) {
full_values[i].set_double_count((full_values[i].to_double() + val.to_double()) / 2, 2);
} else {
state->second.sum += val.to_double();
state->second.count += 1;

full_values[i] = (state->second.sum / state->second.count);
double sum = full_values[i].to_double() * count + val.to_double();
count++;
full_values[i].set_double_count(sum / count, count);
}
return;
}
Expand Down
2 changes: 1 addition & 1 deletion mvt.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ struct mvt_value {
this->count = 0;
}

void set_double_accum(double v, size_t c) {
void set_double_count(double v, size_t c) {
this->type = mvt_double;
this->numeric_value.double_value = v;
this->count = c;
Expand Down
2 changes: 1 addition & 1 deletion serial.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ struct serial_val {
}
}

void set_double_accum(double v, size_t c) {
void set_double_count(double v, size_t c) {
type = mvt_double;
s = milo::dtoa_milo(v) + '\0' + std::to_string(c);
}
Expand Down
4 changes: 2 additions & 2 deletions ...ax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"description": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles",
"format": "pbf",
"generator_options": "./tippecanoe -q -a@ -f -o tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles -z5 -Ethesum:sum -Etheproduct:product -Ethemax:max -Ethemin:min -Ethemean:mean -Etheconcat:concat -Ethecomma:comma tests/accumulate/in.json",
"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.773809523809529,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}",
"json": "{\"vector_layers\":[{\"id\":\"in\",\"description\":\"\",\"minzoom\":0,\"maxzoom\":5,\"fields\":{\"thecomma\":\"Mixed\",\"theconcat\":\"Mixed\",\"themax\":\"Number\",\"themean\":\"Number\",\"themin\":\"Number\",\"theproduct\":\"Number\",\"thesum\":\"Number\"}}],\"tilestats\":{\"layerCount\":1,\"layers\":[{\"layer\":\"in\",\"count\":100,\"geometry\":\"Point\",\"attributeCount\":7,\"attributes\":[{\"attribute\":\"thecomma\",\"count\":152,\"type\":\"mixed\",\"values\":[1,\"1,8,44,6\",10,100,11,12,13,14,15,16,\"16,82\",17,\"17,62\",18,\"18,39\",\"18,39,15\",19,2,\"2,16,82\",20,21,22,23,24,\"24,83\",25,26,\"26,89,25\",\"26,89,25,55,91,33,17,62,69,81,7,93,31\",27,28,29,3,\"3,14\",\"3,14,78,28\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",\"3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,73,33\",30,31,32,33,34,35,\"35,79\",\"35,79,1,31,70,64,8,44,6,52,61,36,87,96,58\",36,37,38,\"38,49\",\"38,49,43,54,71,9,32\",39,4,40,41,42,43,\"43,54\",44,45,46,47,48,49,\"49,1,31,70,64,8,44,6\",\"49,72,86,46,76,48,18,39,15,80,85,24,83,10\",5,\"5,12\",50,51,52,53,54,55,\"55,91\",\"55,91,33\",56,57,\"57,37\",\"57,37,97\",58,59,6,60,61,\"61,36\",62,63,64,65,\"65,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,96\",66,\"66,22\",\"66,22,63,95,73,86,19\",67,\"67,38,11,2,16,82\",\"67,65\",\"67,74,65,5,12,94,84,51,38\",68,69,\"69,81\"],\"min\":1,\"max\":100},{\"attribute\":\"theconcat\",\"count\":152,\"type\":\"mixed\",\"values\":[1,10,100,11,12,13,14,15,16,\"1682\",17,\"1762\",18,\"1839\",\"183915\",\"18446\",19,2,20,21,\"21682\",22,23,24,\"2483\",25,26,\"268925\",\"2689255591331762698179331\",27,28,29,3,30,31,\"314\",\"3147828\",\"31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658\",\"3147828677465512948451573797539299902113387333\",32,33,34,35,\"3579\",\"357913170648446526136879658\",36,37,38,\"3849\",\"3849435471932\",39,4,40,41,42,43,\"4354\",44,45,46,47,48,49,\"4913170648446\",\"4972864676481839158085248310\",5,50,51,\"512\",52,53,54,55,\"5591\",\"559133\",56,57,\"5737\",\"573797\",58,59,6,60,61,\"6136\",62,63,64,65,\"6538494354719327523722959421006622639573861996\",66,\"6622\",\"66226395738619\",67,\"67381121682\",\"6765\",\"67746551294845138\",68,69,\"6981\"],\"min\":1,\"max\":100},{\"attribute\":\"themax\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"themean\",\"count\":134,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,14.75,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,28.5,29,3,30,30.75,31,32,33,33.333333333333339,34,34.125,35,36,37,37.333333333333339,38,39,39.5,4,40,41,42,42.285714285714288,43,43.5,44,45,46,46.666666666666667,47,47.333333333333339,48,48.5,48.53333333333333,49,5,50,50.5,50.77380952380951,51,52,52.214285714285718,52.23076923076923,52.333333333333339,53,53.25,53.5,54,54.44444444444444,55,55.69565217391305,55.75,56,57,58,59,59.333333333333339,59.666666666666667,6,60,60.4,60.57142857142857,61,62,62.5,63,63.666666666666667,64,64.66666666666667,65,66,67,68,69,69.33333333333333,7,70],\"min\":1,\"max\":100},{\"attribute\":\"themin\",\"count\":100,\"type\":\"number\",\"values\":[1,10,100,11,12,13,14,15,16,17,18,19,2,20,21,22,23,24,25,26,27,28,29,3,30,31,32,33,34,35,36,37,38,39,4,40,41,42,43,44,45,46,47,48,49,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,68,69,7,70,71,72,73,74,75,76,77,78,79,8,80,81,82,83,84,85,86,87,88,89,9,90,91,92,93,94,95,96,97,98,99],\"min\":1,\"max\":100},{\"attribute\":\"theproduct\",\"count\":150,\"type\":\"number\",\"values\":[1,1.1358965693283625e+21,1.5864380887725077e+38,1.6098219495357537e+132,10,100,1036586822040,10530,1054,11,119282,12,12157085491200,13,1312,1320551424,14,14372413440,1452,15,16,16336199880,165165,1693200,17,1725,18,1862,19,1992,2,20,20448,204573,2088,21,2109,2112,2196,22,23,2322,24,2432430,25,26,2624,27,2765,28,288600,29,295891195017600,3,30,300656,31,32,33,34,35,35840,36,3638439,37,38,39,39123,3982783094784,4,4.486244263022014e+22,40,402696,41,42,43,4355,44,45,46,47,48,49,5,5.103920456870841e+38,50,5005,51,517406400,52,53,54,55,5589,56,57,57850,58,59,6],\"min\":1,\"max\":1.6098219495357537e+132},{\"attribute\":\"thesum\",\"count\":136,\"type\":\"number\",\"values\":[1,10,100,101,107,11,112,114,12,123,1278,1281,13,132,14,140,142,146,15,150,156,157,16,17,178,179,18,19,191,2,20,202,208,21,216,22,223,229,23,24,25,250,26,27,273,28,29,296,3,30,302,31,32,33,34,35,352,36,37,38,388,39,4,40,408,41,42,424,4265,43,44,45,46,462,47,48,49,490,5,50,51,52,53,54,55,56,57,58,59,6,60,61,62,63,64,65,66,67,679,68],\"min\":1,\"max\":4265}]}]}}",
"maxzoom": "5",
"minzoom": "0",
"name": "tests/accumulate/out/-z5_-Ethesum%3asum_-Etheproduct%3aproduct_-Ethemax%3amax_-Ethemin%3amin_-Ethemean%3amean_-Etheconcat%3aconcat_-Ethecomma%3acomma.json.check.mbtiles",
Expand All @@ -15,7 +15,7 @@
}, "features": [
{ "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [
{ "type": "FeatureCollection", "properties": { "layer": "in", "version": 2, "extent": 4096 }, "features": [
{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.773809523809529, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } }
{ "type": "Feature", "properties": { "thesum": 4265, "themax": 100, "themin": 1, "theproduct": 1.6098219495357537e+132, "themean": 50.77380952380951, "theconcat": "31478286774655129484515737975392999021133849435471932752372295942100662263957386191121682268925559133176269817934676481839158085248310357913170648446526136879658", "thecomma": "3,14,78,28,67,74,65,5,12,94,84,51,57,37,97,53,92,99,90,21,13,38,49,43,54,71,9,32,75,23,72,29,59,42,100,66,22,63,95,73,86,19,11,2,16,82,26,89,25,55,91,33,17,62,69,81,7,93,46,76,48,18,39,15,80,85,24,83,10,35,79,1,31,70,64,8,44,6,52,61,36,87,96,58" }, "geometry": { "type": "Point", "coordinates": [ -147.392578, 84.778533 ] } }
] }
] }
,
Expand Down
5 changes: 5 additions & 0 deletions tile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2370,6 +2370,11 @@ long long write_tile(decompressor *geoms, std::atomic<long long> *geompos_in, ch
if (p.need_tilestats.size() > 0) {
for (size_t j = 0; j < p.full_keys.size(); j++) {
if (p.need_tilestats.count(*p.full_keys[j]) > 0) {
// remove accumulation state
size_t found = p.full_values[j].s.find('\0');
if (found != std::string::npos) {
p.full_values[j].s = p.full_values[j].s.substr(0, found);
}
add_tilestats(layername, z, layermaps, tiling_seg, layer_unmaps, *p.full_keys[j], p.full_values[j]);
}
}
Expand Down

0 comments on commit bc6b5f3

Please sign in to comment.