Skip to content

Commit 2a0a319

Browse files
missinglinkmissinglink
committed
refactor(tags): support semi-colon delimited name values
1 parent fd8a752 commit 2a0a319

File tree

5 files changed

+153
-26
lines changed

5 files changed

+153
-26
lines changed

‎stream/tag_mapper.js

+31-24
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
const _ = require('lodash');
88
const through = require('through2');
99
const peliasLogger = require('pelias-logger').get('openstreetmap');
10+
const parseSemicolonDelimitedValues = require('../util/parseSemicolonDelimitedValues');
1011

1112
const LOCALIZED_NAME_KEYS = require('../config/localized_name_keys');
1213
const NAME_SCHEMA = require('../schema/name_osm');
@@ -37,29 +38,34 @@ module.exports = function(){
3738
// @ref: http://wiki.openstreetmap.org/wiki/Namespace#Language_code_suffix
3839
const langCode = getNameSuffix( key );
3940
if( langCode ){
40-
const langValue = trim( value );
41-
if( langValue ){
42-
doc.setName( langCode, langValue );
43-
}
41+
const langValues = parseSemicolonDelimitedValues( value );
42+
langValues.forEach(( langValue, i ) => {
43+
if ( i === 0 ) {
44+
doc.setName( langCode, langValue );
45+
} else {
46+
doc.setNameAlias( langCode, langValue );
47+
}
48+
});
4449
}
4550

4651
// Map name data from our name mapping schema
4752
else if( _.has(NAME_SCHEMA, key) ){
48-
const nameValue = trim( value );
49-
if( nameValue ){
50-
if( 'name' === key ){
51-
doc.setName( NAME_SCHEMA[key], nameValue );
52-
} else if ( 'default' === NAME_SCHEMA[key] ) {
53-
doc.setNameAlias( NAME_SCHEMA[key], nameValue );
54-
} else {
55-
doc.setName( NAME_SCHEMA[key], nameValue );
53+
const nameValues = parseSemicolonDelimitedValues( cleanString( value ) );
54+
nameValues.forEach(( nameValue, i ) => {
55+
// For the primary name key 'name', ensure it is the first value
56+
if( 'name' === key && i === 0 ){
57+
doc.setName(NAME_SCHEMA[key], nameValue);
58+
return;
5659
}
57-
}
60+
61+
// Otherwise set as an alias
62+
doc.setNameAlias( NAME_SCHEMA[key], nameValue );
63+
});
5864
}
5965

6066
// Map address data from our address mapping schema
6167
else if( _.has(ADDRESS_SCHEMA, key) ){
62-
const addrValue = trim( value );
68+
const addrValue = cleanString( value );
6369
if( addrValue ){
6470
const label = ADDRESS_SCHEMA[key];
6571
doc.setAddress(label, normalizeAddressField(label, addrValue));
@@ -71,16 +77,17 @@ module.exports = function(){
7177
// other names which we could use as the default.
7278
if( !doc.getName('default') ){
7379

74-
const defaultName =
75-
_.get(tags, 'official_name') ||
76-
_.get(tags, 'int_name') ||
77-
_.get(tags, 'nat_name') ||
78-
_.get(tags, 'reg_name') ||
79-
doc.getName('en');
80+
const defaultName = [
81+
...parseSemicolonDelimitedValues(_.get(tags, 'official_name')),
82+
...parseSemicolonDelimitedValues(_.get(tags, 'int_name')),
83+
...parseSemicolonDelimitedValues(_.get(tags, 'nat_name')),
84+
...parseSemicolonDelimitedValues(_.get(tags, 'reg_name')),
85+
...parseSemicolonDelimitedValues(doc.getName('en'))
86+
].filter(Boolean);
8087

8188
// use one of the preferred name tags listed above
82-
if ( defaultName ){
83-
doc.setName('default', defaultName);
89+
if ( defaultName.length ){
90+
doc.setName('default', defaultName[0]);
8491
}
8592

8693
// else try to use an available two-letter language name tag
@@ -101,7 +108,7 @@ module.exports = function(){
101108
// Import airport codes as aliases
102109
if( tags.hasOwnProperty('aerodrome') || tags.hasOwnProperty('aeroway') ){
103110
if( tags.hasOwnProperty('iata') ){
104-
const iata = trim( tags.iata );
111+
const iata = cleanString( tags.iata );
105112
if( iata ){
106113
doc.setNameAlias( 'default', iata );
107114
doc.setNameAlias( 'default', `${iata} Airport` );
@@ -127,7 +134,7 @@ module.exports = function(){
127134
};
128135

129136
// Clean string of leading/trailing junk chars
130-
function trim( str ){
137+
function cleanString( str ){
131138
return _.trim( str, '#$%^*<>-=_{};:",./?\t\n\' ' );
132139
}
133140

‎test/run.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ var tests = [
1717
require('./stream/pbf'),
1818
require('./stream/stats'),
1919
require('./stream/tag_mapper'),
20-
require('./stream/addresses_without_street')
20+
require('./stream/addresses_without_street'),
21+
require('./util/parseSemicolonDelimitedValues')
2122
];
2223

2324
tests.map(function(t) {

‎test/stream/tag_mapper.js

+78
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,84 @@ module.exports.tests.osm_names = function(test, common) {
110110
}));
111111
stream.write(doc);
112112
});
113+
114+
test('maps - name aliases - multiple alt_names', function(t) {
115+
var doc = new Document('a','b',1);
116+
doc.setMeta('tags', {
117+
loc_name: 'loc_name',
118+
nat_name: 'nat_name',
119+
int_name: 'int_name',
120+
name: 'name ;name2; name3',
121+
alt_name: 'alt_name;alt_name2;alt_name3',
122+
official_name: 'official_name',
123+
old_name: 'old_name',
124+
reg_name: 'reg_name',
125+
short_name: 'short_name',
126+
sorting_name: 'sorting_name'
127+
});
128+
var stream = mapper();
129+
stream.pipe( through.obj( function( doc, enc, next ){
130+
t.equal(doc.getName('default'), 'name', 'correctly mapped');
131+
t.deepEqual(doc.getNameAliases('default'), [
132+
'loc_name',
133+
'name2','name3',
134+
'alt_name','alt_name2','alt_name3',
135+
'short_name'
136+
], 'correctly mapped');
137+
138+
t.end(); // test will fail if not called (or called twice).
139+
next();
140+
}));
141+
142+
stream.write(doc);
143+
});
144+
145+
test('maps - semi-colon delimited names', function(t) {
146+
var doc = new Document('a','b',1);
147+
doc.setMeta('tags', {
148+
name: 'name ;name2; name3',
149+
'name:de': 'ding ;ding2; ding3',
150+
});
151+
var stream = mapper();
152+
stream.pipe( through.obj( function( doc, enc, next ){
153+
t.equal(doc.getName('default'), 'name', 'correctly mapped');
154+
t.deepEqual(doc.getNameAliases('default'), [
155+
'name2','name3',
156+
], 'correctly mapped');
157+
158+
t.equal(doc.getName('de'), 'ding', 'correctly mapped');
159+
t.deepEqual(doc.getNameAliases('de'), [
160+
'ding2','ding3',
161+
], 'correctly mapped');
162+
163+
t.end(); // test will fail if not called (or called twice).
164+
next();
165+
}));
166+
167+
stream.write(doc);
168+
});
169+
170+
test('maps - semi-colon delimited names - no "name" tag', function(t) {
171+
var doc = new Document('a','b',1);
172+
doc.setMeta('tags', {
173+
'name:de': 'ding ;ding2; ding3',
174+
});
175+
var stream = mapper();
176+
stream.pipe( through.obj( function( doc, enc, next ){
177+
t.equal(doc.getName('default'), 'ding', 'correctly mapped');
178+
t.deepEqual(doc.getNameAliases('default'), [], 'correctly mapped');
179+
180+
t.equal(doc.getName('de'), 'ding', 'correctly mapped');
181+
t.deepEqual(doc.getNameAliases('de'), [
182+
'ding2','ding3',
183+
], 'correctly mapped');
184+
185+
t.end(); // test will fail if not called (or called twice).
186+
next();
187+
}));
188+
189+
stream.write(doc);
190+
});
113191
};
114192

115193
// Cover the case of a tag key being 'name:' eg. { 'name:': 'foo' }

‎test/util/parseSemicolonDelimitedValues.js

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
const parseSemicolonDelimitedValues = require('../../util/parseSemicolonDelimitedValues');
2+
3+
module.exports.tests = {};
4+
5+
// test exports
6+
module.exports.tests.smoke = function (test, common) {
7+
test('interface', t => {
8+
t.equal(typeof parseSemicolonDelimitedValues, 'function', 'function');
9+
t.end();
10+
});
11+
test('parse - invalid', t => {
12+
t.deepEqual(parseSemicolonDelimitedValues(1), []);
13+
t.deepEqual(parseSemicolonDelimitedValues(['a']), []);
14+
t.deepEqual(parseSemicolonDelimitedValues([{'a': 'b'}]), []);
15+
t.deepEqual(parseSemicolonDelimitedValues(undefined), []);
16+
t.deepEqual(parseSemicolonDelimitedValues(null), []);
17+
t.deepEqual(parseSemicolonDelimitedValues(''), []);
18+
t.end();
19+
});
20+
test('parse - examples', t => {
21+
t.deepEqual(parseSemicolonDelimitedValues(''), []);
22+
t.deepEqual(parseSemicolonDelimitedValues(' '), []);
23+
t.deepEqual(parseSemicolonDelimitedValues(' ;; ; ; ; ; ;; ; ;; '), []);
24+
t.deepEqual(parseSemicolonDelimitedValues(' a; b ;;; ; '), ['a', 'b']);
25+
t.end();
26+
});
27+
};
28+
29+
module.exports.all = function (tape, common) {
30+
31+
function test(name, testFunction) {
32+
return tape('parseSemicolonDelimitedValues: ' + name, testFunction);
33+
}
34+
35+
for (var testCase in module.exports.tests) {
36+
module.exports.tests[testCase](test, common);
37+
}
38+
};

‎util/parseSemicolonDelimitedValues.js

+4-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,10 @@ const _ = require('lodash');
33
// Split multi-value OSM tags into an Array
44
// https://wiki.openstreetmap.org/wiki/Talk:Semi-colon_value_separator
55
function parseSemicolonDelimitedValues(value) {
6-
return (_.isString(value) ? value : '').split(';').map(Function.prototype.call, String.prototype.trim);
6+
return (_.isString(value) ? value : '')
7+
.split(';')
8+
.map(Function.prototype.call, String.prototype.trim)
9+
.filter(Boolean);
710
}
811

912
module.exports = parseSemicolonDelimitedValues;

0 commit comments

Comments
 (0)