Skip to content

Commit 80449a8

Browse files
BukhtawarBukhtawar
authored
Merge branch 'main' into decommission-api/pr
2 parents 534a3de + ff2d5be commit 80449a8

17 files changed

+214
-29
lines changed

‎CHANGELOG.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
3838
- Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
3939
- Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
4040
- Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
41+
- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4 ([#4644](https://github.com/opensearch-project/OpenSearch/pull/4644))
4142

4243
### Changed
4344
- Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
@@ -53,9 +54,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
5354
- Add DecommissionService and helper to execute awareness attribute decommissioning ([#4084](https://github.com/opensearch-project/OpenSearch/pull/4084))
5455
- Further simplification of the ZIP publication implementation ([#4360](https://github.com/opensearch-project/OpenSearch/pull/4360))
5556
- Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
57+
- Load the deprecated master role in a dedicated method instead of in setAdditionalRoles() ([#4582](https://github.com/opensearch-project/OpenSearch/pull/4582))
5658
- Add APIs (GET/PUT) to decommission awareness attribute ([#4261](https://github.com/opensearch-project/OpenSearch/pull/4261))
5759

58-
5960
### Deprecated
6061

6162
### Removed
@@ -88,6 +89,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
8889
- [Segment Replication] Ignore lock file when testing cleanupAndPreserveLatestCommitPoint ([#4544](https://github.com/opensearch-project/OpenSearch/pull/4544))
8990
- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
9091
- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))
92+
- [Bug]: Alias filter lost after rollover ([#4499](https://github.com/opensearch-project/OpenSearch/pull/4499))
9193

9294
### Security
9395
- CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))

‎plugins/repository-hdfs/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ opensearchplugin {
4848
}
4949

5050
versions << [
51-
'hadoop3': '3.3.3'
51+
'hadoop3': '3.3.4'
5252
]
5353

5454
testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"

‎plugins/repository-hdfs/licenses/hadoop-client-api-3.3.3.jar.sha1

-1
This file was deleted.

‎plugins/repository-hdfs/licenses/hadoop-client-api-3.3.4.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
6339a8f7279310c8b1f7ef314b592d8c71ca72ef

‎plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.3.jar.sha1

-1
This file was deleted.

‎plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.4.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0

‎plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.3.jar.sha1

-1
This file was deleted.

‎plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.4.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
036ef2f86dc44410d2bb5d54ce40435d2484d9a5

‎server/src/main/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverService.java

+31-5
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ private RolloverResult rolloverAlias(
188188
ClusterState newState = createIndexService.applyCreateIndexRequest(currentState, createIndexClusterStateRequest, silent);
189189
newState = indexAliasesService.applyAliasActions(
190190
newState,
191-
rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata.isHidden(), aliasName)
191+
rolloverAliasToNewIndex(sourceIndexName, rolloverIndexName, explicitWriteIndex, aliasMetadata, aliasName)
192192
);
193193

194194
RolloverInfo rolloverInfo = new RolloverInfo(aliasName, metConditions, threadPool.absoluteTimeInMillis());
@@ -309,20 +309,46 @@ static List<AliasAction> rolloverAliasToNewIndex(
309309
String oldIndex,
310310
String newIndex,
311311
boolean explicitWriteIndex,
312-
@Nullable Boolean isHidden,
312+
AliasMetadata aliasMetadata,
313313
String alias
314314
) {
315+
String filterAsString = aliasMetadata.getFilter() != null ? aliasMetadata.getFilter().string() : null;
316+
315317
if (explicitWriteIndex) {
316318
return Collections.unmodifiableList(
317319
Arrays.asList(
318-
new AliasAction.Add(newIndex, alias, null, null, null, true, isHidden),
319-
new AliasAction.Add(oldIndex, alias, null, null, null, false, isHidden)
320+
new AliasAction.Add(
321+
newIndex,
322+
alias,
323+
filterAsString,
324+
aliasMetadata.getIndexRouting(),
325+
aliasMetadata.getSearchRouting(),
326+
true,
327+
aliasMetadata.isHidden()
328+
),
329+
new AliasAction.Add(
330+
oldIndex,
331+
alias,
332+
filterAsString,
333+
aliasMetadata.getIndexRouting(),
334+
aliasMetadata.getSearchRouting(),
335+
false,
336+
aliasMetadata.isHidden()
337+
)
320338
)
321339
);
322340
} else {
323341
return Collections.unmodifiableList(
324342
Arrays.asList(
325-
new AliasAction.Add(newIndex, alias, null, null, null, null, isHidden),
343+
new AliasAction.Add(
344+
newIndex,
345+
alias,
346+
filterAsString,
347+
aliasMetadata.getIndexRouting(),
348+
aliasMetadata.getSearchRouting(),
349+
null,
350+
aliasMetadata.isHidden()
351+
),
326352
new AliasAction.Remove(oldIndex, alias, null)
327353
)
328354
);

‎server/src/main/java/org/opensearch/cluster/metadata/AliasAction.java

+12
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,18 @@ public String getAlias() {
138138
return alias;
139139
}
140140

141+
public String getFilter() {
142+
return filter;
143+
}
144+
145+
public String getSearchRouting() {
146+
return searchRouting;
147+
}
148+
149+
public String getIndexRouting() {
150+
return indexRouting;
151+
}
152+
141153
public Boolean writeIndex() {
142154
return writeIndex;
143155
}

‎server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

+12-5
Original file line numberDiff line numberDiff line change
@@ -616,11 +616,18 @@ public static void setAdditionalRoles(final Set<DiscoveryNodeRole> additionalRol
616616
+ "], roles by name abbreviation ["
617617
+ roleNameAbbreviationToPossibleRoles
618618
+ "]";
619-
// TODO: Remove the Map 'roleNameToPossibleRolesWithMaster' and let 'roleMap = roleNameToPossibleRoles', after removing MASTER_ROLE.
620-
// It's used to allow CLUSTER_MANAGER_ROLE that introduced in 2.0, having the same abbreviation name with MASTER_ROLE.
621-
final Map<String, DiscoveryNodeRole> roleNameToPossibleRolesWithMaster = new HashMap<>(roleNameToPossibleRoles);
622-
roleNameToPossibleRolesWithMaster.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
623-
roleMap = Collections.unmodifiableMap(roleNameToPossibleRolesWithMaster);
619+
roleMap = roleNameToPossibleRoles;
620+
}
621+
622+
/**
623+
* Load the deprecated {@link DiscoveryNodeRole#MASTER_ROLE}.
624+
* Master role is not added into BUILT_IN_ROLES, because {@link #setAdditionalRoles(Set)} check role name abbreviation duplication,
625+
* and CLUSTER_MANAGER_ROLE has the same abbreviation name with MASTER_ROLE.
626+
*/
627+
public static void setDeprecatedMasterRole() {
628+
final Map<String, DiscoveryNodeRole> modifiableRoleMap = new HashMap<>(roleMap);
629+
modifiableRoleMap.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), DiscoveryNodeRole.MASTER_ROLE);
630+
roleMap = Collections.unmodifiableMap(modifiableRoleMap);
624631
}
625632

626633
public static Set<String> getPossibleRoleNames() {

‎server/src/main/java/org/opensearch/node/Node.java

+2
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,8 @@ protected Node(
429429
.collect(Collectors.toSet());
430430
DiscoveryNode.setAdditionalRoles(additionalRoles);
431431

432+
DiscoveryNode.setDeprecatedMasterRole();
433+
432434
/*
433435
* Create the environment based on the finalized view of the settings. This is to ensure that components get the same setting
434436
* values, no matter they ask for them from.

‎server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

+143-4
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,13 @@ public void testRolloverAliasActions() {
126126
String sourceIndex = randomAlphaOfLength(10);
127127
String targetIndex = randomAlphaOfLength(10);
128128

129-
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, false, null, sourceAlias);
129+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
130+
sourceIndex,
131+
targetIndex,
132+
false,
133+
createDefaultAliasMetadata(sourceAlias, null),
134+
sourceAlias
135+
);
130136
assertThat(actions, hasSize(2));
131137
boolean foundAdd = false;
132138
boolean foundRemove = false;
@@ -149,7 +155,13 @@ public void testRolloverAliasActionsWithExplicitWriteIndex() {
149155
String sourceAlias = randomAlphaOfLength(10);
150156
String sourceIndex = randomAlphaOfLength(10);
151157
String targetIndex = randomAlphaOfLength(10);
152-
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, true, null, sourceAlias);
158+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
159+
sourceIndex,
160+
targetIndex,
161+
true,
162+
createDefaultAliasMetadata(sourceAlias, null),
163+
sourceAlias
164+
);
153165

154166
assertThat(actions, hasSize(2));
155167
boolean foundAddWrite = false;
@@ -172,11 +184,64 @@ public void testRolloverAliasActionsWithExplicitWriteIndex() {
172184
assertTrue(foundRemoveWrite);
173185
}
174186

187+
public void testRolloverAliasActionsWithFilterAndExplicitWriteIndex() {
188+
String sourceAlias = randomAlphaOfLength(10);
189+
String sourceIndex = randomAlphaOfLength(10);
190+
String targetIndex = randomAlphaOfLength(10);
191+
String indexRouting = randomAlphaOfLength(10);
192+
String sourceRouting = randomAlphaOfLength(10);
193+
AliasMetadata aliasMetadata = createAliasMetadata(
194+
sourceAlias,
195+
Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)),
196+
indexRouting,
197+
sourceRouting,
198+
true
199+
);
200+
201+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
202+
sourceIndex,
203+
targetIndex,
204+
true,
205+
aliasMetadata,
206+
sourceAlias
207+
);
208+
209+
assertThat(actions, hasSize(2));
210+
boolean foundAddWrite = false;
211+
boolean foundRemoveWrite = false;
212+
for (AliasAction action : actions) {
213+
AliasAction.Add addAction = (AliasAction.Add) action;
214+
if (action.getIndex().equals(targetIndex)) {
215+
assertEquals(sourceAlias, addAction.getAlias());
216+
assertEquals(aliasMetadata.filter().string(), addAction.getFilter());
217+
assertEquals(indexRouting, addAction.getIndexRouting());
218+
assertEquals(sourceRouting, addAction.getSearchRouting());
219+
220+
assertTrue(addAction.writeIndex());
221+
foundAddWrite = true;
222+
} else if (action.getIndex().equals(sourceIndex)) {
223+
assertEquals(sourceAlias, addAction.getAlias());
224+
assertFalse(addAction.writeIndex());
225+
foundRemoveWrite = true;
226+
} else {
227+
throw new AssertionError("Unknown index [" + action.getIndex() + "]");
228+
}
229+
}
230+
assertTrue(foundAddWrite);
231+
assertTrue(foundRemoveWrite);
232+
}
233+
175234
public void testRolloverAliasActionsWithHiddenAliasAndExplicitWriteIndex() {
176235
String sourceAlias = randomAlphaOfLength(10);
177236
String sourceIndex = randomAlphaOfLength(10);
178237
String targetIndex = randomAlphaOfLength(10);
179-
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, true, true, sourceAlias);
238+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
239+
sourceIndex,
240+
targetIndex,
241+
true,
242+
createDefaultAliasMetadata(sourceAlias, true),
243+
sourceAlias
244+
);
180245

181246
assertThat(actions, hasSize(2));
182247
boolean foundAddWrite = false;
@@ -202,11 +267,66 @@ public void testRolloverAliasActionsWithHiddenAliasAndExplicitWriteIndex() {
202267
assertTrue(foundRemoveWrite);
203268
}
204269

270+
public void testRolloverAliasActionsWithFilterAndHiddenAliasAndImplicitWriteIndex() {
271+
String sourceAlias = randomAlphaOfLength(10);
272+
String sourceIndex = randomAlphaOfLength(10);
273+
String targetIndex = randomAlphaOfLength(10);
274+
String indexRouting = randomAlphaOfLength(10);
275+
String sourceRouting = randomAlphaOfLength(10);
276+
AliasMetadata aliasMetadata = createAliasMetadata(
277+
sourceAlias,
278+
Collections.singletonMap(randomAlphaOfLength(2), randomAlphaOfLength(2)),
279+
indexRouting,
280+
sourceRouting,
281+
true
282+
);
283+
284+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
285+
sourceIndex,
286+
targetIndex,
287+
false,
288+
aliasMetadata,
289+
sourceAlias
290+
);
291+
292+
assertThat(actions, hasSize(2));
293+
boolean foundAddWrite = false;
294+
boolean foundRemoveWrite = false;
295+
for (AliasAction action : actions) {
296+
if (action.getIndex().equals(targetIndex)) {
297+
assertThat(action, instanceOf(AliasAction.Add.class));
298+
AliasAction.Add addAction = (AliasAction.Add) action;
299+
assertEquals(sourceAlias, addAction.getAlias());
300+
assertThat(addAction.writeIndex(), nullValue());
301+
assertTrue(addAction.isHidden());
302+
assertEquals(aliasMetadata.filter().string(), addAction.getFilter());
303+
assertEquals(indexRouting, addAction.getIndexRouting());
304+
assertEquals(sourceRouting, addAction.getSearchRouting());
305+
foundAddWrite = true;
306+
} else if (action.getIndex().equals(sourceIndex)) {
307+
assertThat(action, instanceOf(AliasAction.Remove.class));
308+
AliasAction.Remove removeAction = (AliasAction.Remove) action;
309+
assertEquals(sourceAlias, removeAction.getAlias());
310+
foundRemoveWrite = true;
311+
} else {
312+
throw new AssertionError("Unknown index [" + action.getIndex() + "]");
313+
}
314+
}
315+
assertTrue(foundAddWrite);
316+
assertTrue(foundRemoveWrite);
317+
}
318+
205319
public void testRolloverAliasActionsWithHiddenAliasAndImplicitWriteIndex() {
206320
String sourceAlias = randomAlphaOfLength(10);
207321
String sourceIndex = randomAlphaOfLength(10);
208322
String targetIndex = randomAlphaOfLength(10);
209-
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(sourceIndex, targetIndex, false, true, sourceAlias);
323+
List<AliasAction> actions = MetadataRolloverService.rolloverAliasToNewIndex(
324+
sourceIndex,
325+
targetIndex,
326+
false,
327+
createDefaultAliasMetadata(sourceAlias, true),
328+
sourceAlias
329+
);
210330

211331
assertThat(actions, hasSize(2));
212332
boolean foundAddWrite = false;
@@ -1010,4 +1130,23 @@ private static IndexMetadata createMetadata(String indexName) {
10101130
.settings(settings)
10111131
.build();
10121132
}
1133+
1134+
private static AliasMetadata createDefaultAliasMetadata(String alias, Boolean isHidden) {
1135+
return AliasMetadata.builder(alias).isHidden(isHidden).build();
1136+
}
1137+
1138+
private static AliasMetadata createAliasMetadata(
1139+
String alias,
1140+
Map filter,
1141+
String indexRouting,
1142+
String searchRouting,
1143+
Boolean isHidden
1144+
) {
1145+
return AliasMetadata.builder(alias)
1146+
.isHidden(isHidden)
1147+
.filter(filter)
1148+
.indexRouting(indexRouting)
1149+
.searchRouting(searchRouting)
1150+
.build();
1151+
}
10131152
}

‎server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeRoleSettingTests.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,7 @@ public void testIsIngestNode() {
5656
}
5757

5858
public void testIsMasterNode() {
59-
// It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
60-
DiscoveryNode.setAdditionalRoles(Collections.emptySet());
59+
DiscoveryNode.setDeprecatedMasterRole();
6160
runRoleTest(DiscoveryNode::isClusterManagerNode, DiscoveryNodeRole.MASTER_ROLE);
6261
}
6362

‎server/src/test/java/org/opensearch/cluster/node/DiscoveryNodeTests.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -176,11 +176,11 @@ public void testDiscoveryNodeIsRemoteClusterClientUnset() {
176176
}
177177

178178
// Added in 2.0 temporarily, validate the MASTER_ROLE is in the list of known roles.
179-
// MASTER_ROLE was removed from BUILT_IN_ROLES and is imported by setAdditionalRoles(),
179+
// MASTER_ROLE was removed from BUILT_IN_ROLES and is imported by setDeprecatedMasterRole(),
180180
// as a workaround for making the new CLUSTER_MANAGER_ROLE has got the same abbreviation 'm'.
181181
// The test validate this behavior.
182-
public void testSetAdditionalRolesCanAddDeprecatedMasterRole() {
183-
DiscoveryNode.setAdditionalRoles(Collections.emptySet());
182+
public void testSetDeprecatedMasterRoleCanAddMasterRole() {
183+
DiscoveryNode.setDeprecatedMasterRole();
184184
assertTrue(DiscoveryNode.getPossibleRoleNames().contains(DiscoveryNodeRole.MASTER_ROLE.roleName()));
185185
}
186186

‎server/src/test/java/org/opensearch/node/NodeRoleSettingsTests.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ public class NodeRoleSettingsTests extends OpenSearchTestCase {
2626
* Remove the test after removing MASTER_ROLE.
2727
*/
2828
public void testClusterManagerAndMasterRoleCanNotCoexist() {
29-
// It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
30-
DiscoveryNode.setAdditionalRoles(Collections.emptySet());
29+
DiscoveryNode.setDeprecatedMasterRole();
3130
Settings roleSettings = Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "cluster_manager, master").build();
3231
Exception exception = expectThrows(IllegalArgumentException.class, () -> NodeRoleSettings.NODE_ROLES_SETTING.get(roleSettings));
3332
assertThat(exception.getMessage(), containsString("[master, cluster_manager] can not be assigned together to a node"));
@@ -49,8 +48,7 @@ public void testClusterManagerAndDataNodeRoles() {
4948
* Remove the test after removing MASTER_ROLE.
5049
*/
5150
public void testMasterRoleDeprecationMessage() {
52-
// It's used to add MASTER_ROLE into 'roleMap', because MASTER_ROLE is removed from DiscoveryNodeRole.BUILT_IN_ROLES in 2.0.
53-
DiscoveryNode.setAdditionalRoles(Collections.emptySet());
51+
DiscoveryNode.setDeprecatedMasterRole();
5452
Settings roleSettings = Settings.builder().put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master").build();
5553
assertEquals(Collections.singletonList(DiscoveryNodeRole.MASTER_ROLE), NodeRoleSettings.NODE_ROLES_SETTING.get(roleSettings));
5654
assertWarnings(DiscoveryNodeRole.MASTER_ROLE_DEPRECATION_MESSAGE);

0 commit comments

Comments
 (0)