1111import org .apache .logging .log4j .LogManager ;
1212import org .apache .logging .log4j .Logger ;
1313import org .apache .logging .log4j .message .ParameterizedMessage ;
14+ import org .apache .lucene .codecs .CodecUtil ;
15+ import org .apache .lucene .index .IndexFileNames ;
16+ import org .apache .lucene .index .SegmentInfos ;
1417import org .apache .lucene .search .ReferenceManager ;
1518import org .apache .lucene .store .Directory ;
19+ import org .apache .lucene .store .FilterDirectory ;
1620import org .apache .lucene .store .IOContext ;
21+ import org .apache .lucene .store .IndexInput ;
22+ import org .opensearch .common .concurrent .GatedCloseable ;
23+ import org .opensearch .index .engine .EngineException ;
24+ import org .opensearch .index .store .RemoteSegmentStoreDirectory ;
1725
1826import java .io .IOException ;
19- import java .nio .file .NoSuchFileException ;
20- import java .util .Arrays ;
21- import java .util .HashSet ;
27+ import java .util .Collection ;
28+ import java .util .Comparator ;
29+ import java .util .HashMap ;
30+ import java .util .List ;
31+ import java .util .Map ;
32+ import java .util .Optional ;
2233import java .util .Set ;
34+ import java .util .concurrent .atomic .AtomicBoolean ;
35+ import java .util .stream .Collectors ;
2336
2437/**
2538 * RefreshListener implementation to upload newly created segment files to the remote store
39+ *
40+ * @opensearch.internal
2641 */
27- public class RemoteStoreRefreshListener implements ReferenceManager .RefreshListener {
42+ public final class RemoteStoreRefreshListener implements ReferenceManager .RefreshListener {
43+ // Visible for testing
44+ static final Set <String > EXCLUDE_FILES = Set .of ("write.lock" );
45+ // Visible for testing
46+ static final int LAST_N_METADATA_FILES_TO_KEEP = 10 ;
2847
48+ private final IndexShard indexShard ;
2949 private final Directory storeDirectory ;
30- private final Directory remoteDirectory ;
31- // ToDo: This can be a map with metadata of the uploaded file as value of the map (GitHub #3398)
32- private final Set < String > filesUploadedToRemoteStore ;
50+ private final RemoteSegmentStoreDirectory remoteDirectory ;
51+ private final Map < String , String > localSegmentChecksumMap ;
52+ private long primaryTerm ;
3353 private static final Logger logger = LogManager .getLogger (RemoteStoreRefreshListener .class );
3454
35- public RemoteStoreRefreshListener (Directory storeDirectory , Directory remoteDirectory ) throws IOException {
36- this .storeDirectory = storeDirectory ;
37- this .remoteDirectory = remoteDirectory ;
38- // ToDo: Handle failures in reading list of files (GitHub #3397)
39- this .filesUploadedToRemoteStore = new HashSet <>(Arrays .asList (remoteDirectory .listAll ()));
55+ public RemoteStoreRefreshListener (IndexShard indexShard ) {
56+ this .indexShard = indexShard ;
57+ this .storeDirectory = indexShard .store ().directory ();
58+ this .remoteDirectory = (RemoteSegmentStoreDirectory ) ((FilterDirectory ) ((FilterDirectory ) indexShard .remoteStore ().directory ())
59+ .getDelegate ()).getDelegate ();
60+ this .primaryTerm = indexShard .getOperationPrimaryTerm ();
61+ localSegmentChecksumMap = new HashMap <>();
4062 }
4163
4264 @ Override
@@ -46,42 +68,112 @@ public void beforeRefresh() throws IOException {
4668
4769 /**
4870 * Upload new segment files created as part of the last refresh to the remote segment store.
49- * The method also deletes segment files from remote store which are not part of local filesystem .
71+ * This method also uploads remote_segments_metadata file which contains metadata of each segment file uploaded .
5072 * @param didRefresh true if the refresh opened a new reference
51- * @throws IOException in case of I/O error in reading list of local files
5273 */
5374 @ Override
54- public void afterRefresh (boolean didRefresh ) throws IOException {
55- if (didRefresh ) {
56- Set <String > localFiles = Set .of (storeDirectory .listAll ());
57- localFiles .stream ().filter (file -> !filesUploadedToRemoteStore .contains (file )).forEach (file -> {
58- try {
59- remoteDirectory .copyFrom (storeDirectory , file , file , IOContext .DEFAULT );
60- filesUploadedToRemoteStore .add (file );
61- } catch (NoSuchFileException e ) {
62- logger .info (
63- () -> new ParameterizedMessage ("The file {} does not exist anymore. It can happen in case of temp files" , file ),
64- e
65- );
66- } catch (IOException e ) {
67- // ToDO: Handle transient and permanent un-availability of the remote store (GitHub #3397)
68- logger .warn (() -> new ParameterizedMessage ("Exception while uploading file {} to the remote segment store" , file ), e );
69- }
70- });
75+ public void afterRefresh (boolean didRefresh ) {
76+ synchronized (this ) {
77+ try {
78+ if (indexShard .shardRouting .primary ()) {
79+ if (this .primaryTerm != indexShard .getOperationPrimaryTerm ()) {
80+ this .primaryTerm = indexShard .getOperationPrimaryTerm ();
81+ this .remoteDirectory .init ();
82+ }
83+ try {
84+ String lastCommittedLocalSegmentFileName = SegmentInfos .getLastCommitSegmentsFileName (storeDirectory );
85+ if (!remoteDirectory .containsFile (
86+ lastCommittedLocalSegmentFileName ,
87+ getChecksumOfLocalFile (lastCommittedLocalSegmentFileName )
88+ )) {
89+ deleteStaleCommits ();
90+ }
91+ try (GatedCloseable <SegmentInfos > segmentInfosGatedCloseable = indexShard .getSegmentInfosSnapshot ()) {
92+ SegmentInfos segmentInfos = segmentInfosGatedCloseable .get ();
93+ Collection <String > refreshedLocalFiles = segmentInfos .files (true );
94+
95+ List <String > segmentInfosFiles = refreshedLocalFiles .stream ()
96+ .filter (file -> file .startsWith (IndexFileNames .SEGMENTS ))
97+ .collect (Collectors .toList ());
98+ Optional <String > latestSegmentInfos = segmentInfosFiles .stream ()
99+ .max (Comparator .comparingLong (IndexFileNames ::parseGeneration ));
71100
72- Set <String > remoteFilesToBeDeleted = new HashSet <>();
73- // ToDo: Instead of deleting files in sync, mark them and delete in async/periodic flow (GitHub #3142)
74- filesUploadedToRemoteStore .stream ().filter (file -> !localFiles .contains (file )).forEach (file -> {
75- try {
76- remoteDirectory .deleteFile (file );
77- remoteFilesToBeDeleted .add (file );
78- } catch (IOException e ) {
79- // ToDO: Handle transient and permanent un-availability of the remote store (GitHub #3397)
80- logger .warn (() -> new ParameterizedMessage ("Exception while deleting file {} from the remote segment store" , file ), e );
101+ if (latestSegmentInfos .isPresent ()) {
102+ refreshedLocalFiles .addAll (SegmentInfos .readCommit (storeDirectory , latestSegmentInfos .get ()).files (true ));
103+ segmentInfosFiles .stream ()
104+ .filter (file -> !file .equals (latestSegmentInfos .get ()))
105+ .forEach (refreshedLocalFiles ::remove );
106+
107+ boolean uploadStatus = uploadNewSegments (refreshedLocalFiles );
108+ if (uploadStatus ) {
109+ remoteDirectory .uploadMetadata (
110+ refreshedLocalFiles ,
111+ storeDirectory ,
112+ indexShard .getOperationPrimaryTerm (),
113+ segmentInfos .getGeneration ()
114+ );
115+ localSegmentChecksumMap .keySet ()
116+ .stream ()
117+ .filter (file -> !refreshedLocalFiles .contains (file ))
118+ .collect (Collectors .toSet ())
119+ .forEach (localSegmentChecksumMap ::remove );
120+ }
121+ }
122+ } catch (EngineException e ) {
123+ logger .warn ("Exception while reading SegmentInfosSnapshot" , e );
124+ }
125+ } catch (IOException e ) {
126+ // We don't want to fail refresh if upload of new segments fails. The missed segments will be re-tried
127+ // in the next refresh. This should not affect durability of the indexed data after remote trans-log integration.
128+ logger .warn ("Exception while uploading new segments to the remote segment store" , e );
129+ }
81130 }
82- });
131+ } catch (Throwable t ) {
132+ logger .error ("Exception in RemoteStoreRefreshListener.afterRefresh()" , t );
133+ }
134+ }
135+ }
136+
137+ // Visible for testing
138+ boolean uploadNewSegments (Collection <String > localFiles ) throws IOException {
139+ AtomicBoolean uploadSuccess = new AtomicBoolean (true );
140+ localFiles .stream ().filter (file -> !EXCLUDE_FILES .contains (file )).filter (file -> {
141+ try {
142+ return !remoteDirectory .containsFile (file , getChecksumOfLocalFile (file ));
143+ } catch (IOException e ) {
144+ logger .info (
145+ "Exception while reading checksum of local segment file: {}, ignoring the exception and re-uploading the file" ,
146+ file
147+ );
148+ return true ;
149+ }
150+ }).forEach (file -> {
151+ try {
152+ remoteDirectory .copyFrom (storeDirectory , file , file , IOContext .DEFAULT );
153+ } catch (IOException e ) {
154+ uploadSuccess .set (false );
155+ // ToDO: Handle transient and permanent un-availability of the remote store (GitHub #3397)
156+ logger .warn (() -> new ParameterizedMessage ("Exception while uploading file {} to the remote segment store" , file ), e );
157+ }
158+ });
159+ return uploadSuccess .get ();
160+ }
161+
162+ private String getChecksumOfLocalFile (String file ) throws IOException {
163+ if (!localSegmentChecksumMap .containsKey (file )) {
164+ try (IndexInput indexInput = storeDirectory .openInput (file , IOContext .DEFAULT )) {
165+ String checksum = Long .toString (CodecUtil .retrieveChecksum (indexInput ));
166+ localSegmentChecksumMap .put (file , checksum );
167+ }
168+ }
169+ return localSegmentChecksumMap .get (file );
170+ }
83171
84- remoteFilesToBeDeleted .forEach (filesUploadedToRemoteStore ::remove );
172+ private void deleteStaleCommits () {
173+ try {
174+ remoteDirectory .deleteStaleSegments (LAST_N_METADATA_FILES_TO_KEEP );
175+ } catch (IOException e ) {
176+ logger .info ("Exception while deleting stale commits from remote segment store, will retry delete post next commit" , e );
85177 }
86178 }
87179}
0 commit comments