Reduce/Increase throughtput based on monitoring segments #14
Labels
enhancement
New feature or request
Comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Reducing server indexing overhead
Indexing could start failing when
solrcopywould be used to aggressively.For improving this one could look a data in the segments handler in admin API.
The strategy can be monitoring the size and age of merge candidate segments.
Example
The address
http://localhost:8983/solr/corename/admin/segments?wt=jsonwould return a json response like:{ "responseHeader":{ "status":0, "QTime":3 }, "segments":{ "_dnc":{ "name":"_dnc", "delCount":0, "sizeInBytes":3720192411, "size":1352743, "age":"2020-05-15T04:08:23.499Z", "source":"merge", "version":"6.6.6" }, "_d4q":{ "name":"_d4q", "delCount":538, "sizeInBytes":694162377, "size":214347, "age":"2020-05-15T03:51:56.820Z", "source":"merge", "version":"6.6.6" }, "_eju":{ "name":"_eju", "delCount":6070, "sizeInBytes":855393299, "size":265206, "age":"2020-05-15T04:57:48.340Z", "source":"merge", "version":"6.6.6" }, "_dsx":{ "name":"_dsx", "delCount":0, "sizeInBytes":503792200, "size":149989, "age":"2020-05-15T04:13:45.125Z", "source":"merge", "version":"6.6.6" }, "_e8s":{ "name":"_e8s", "delCount":47776, "sizeInBytes":648225090, "size":196282, "age":"2020-05-15T04:27:55.925Z", "source":"merge", "version":"6.6.6" }, "_e43":{ "name":"_e43", "delCount":0, "sizeInBytes":520414059, "size":155389, "age":"2020-05-15T04:23:38.009Z", "source":"merge", "version":"6.6.6" }, "_fef":{ "name":"_fef", "delCount":0, "sizeInBytes":725333779, "size":222873, "age":"2020-05-15T05:24:41.320Z", "source":"merge", "version":"6.6.6" }, "_es7":{ "name":"_es7", "delCount":0, "sizeInBytes":513762350, "size":159011, "age":"2020-05-15T05:05:08.497Z", "source":"merge", "version":"6.6.6" }, "_f0u":{ "name":"_f0u", "delCount":0, "sizeInBytes":573389084, "size":172592, "age":"2020-05-15T05:12:38.196Z", "source":"merge", "version":"6.6.6" }, "_g0p":{ "name":"_g0p", "delCount":45940, "sizeInBytes":1024868184, "size":322497, "age":"2020-05-15T05:44:38.097Z", "source":"merge", "version":"6.6.6" }, "_f8w":{ "name":"_f8w", "delCount":0, "sizeInBytes":497816681, "size":147568, "age":"2020-05-15T05:19:41.182Z", "source":"merge", "version":"6.6.6" }, "_gaz":{ "name":"_gaz", "delCount":0, "sizeInBytes":593980446, "size":180392, "age":"2020-05-15T05:52:56.207Z", "source":"merge", "version":"6.6.6" }, "_fkn":{ "name":"_fkn", "delCount":3275, "sizeInBytes":12405668, "size":3451, "age":"2020-05-15T05:29:46.512Z", "source":"merge", "version":"6.6.6", "mergeCandidate":true }, "_g42":{ "name":"_g42", "delCount":123, "sizeInBytes":133100032, "size":37831, "age":"2020-05-15T05:47:16.496Z", "source":"merge", "version":"6.6.6" }, "_gf8":{ "name":"_gf8", "delCount":0, "sizeInBytes":276107099, "size":79162, "age":"2020-05-15T05:56:58.199Z", "source":"merge", "version":"6.6.6" }, "_gp6":{ "name":"_gp6", "delCount":0, "sizeInBytes":101241802, "size":29038, "age":"2020-05-15T06:06:45.908Z", "source":"merge", "version":"6.6.6" }, "_gkg":{ "name":"_gkg", "delCount":0, "sizeInBytes":593948603, "size":179423, "age":"2020-05-15T06:02:00.224Z", "source":"merge", "version":"6.6.6" }, "_gw6":{ "name":"_gw6", "delCount":0, "sizeInBytes":227529468, "size":65338, "age":"2020-05-15T06:13:29.840Z", "source":"merge", "version":"6.6.6" }, "_grn":{ "name":"_grn", "delCount":0, "sizeInBytes":549726114, "size":167534, "age":"2020-05-15T06:09:19.418Z", "source":"merge", "version":"6.6.6" }, "_h1e":{ "name":"_h1e", "delCount":0, "sizeInBytes":521633536, "size":156034, "age":"2020-05-15T06:19:10.628Z", "source":"merge", "version":"6.6.6" }, "_gv2":{ "name":"_gv2", "delCount":0, "sizeInBytes":78213126, "size":22363, "age":"2020-05-15T06:12:37.908Z", "source":"merge", "version":"6.6.6" }, "_gxh":{ "name":"_gxh", "delCount":0, "sizeInBytes":96406063, "size":26971, "age":"2020-05-15T06:15:24.332Z", "source":"merge", "version":"6.6.6" }, "_h6p":{ "name":"_h6p", "delCount":0, "sizeInBytes":171039601, "size":48299, "age":"2020-05-15T06:23:30.900Z", "source":"merge", "version":"6.6.6" }, "_h20":{ "name":"_h20", "delCount":0, "sizeInBytes":65716159, "size":19190, "age":"2020-05-15T06:19:40.164Z", "source":"merge", "version":"6.6.6" }, "_h3n":{ "name":"_h3n", "delCount":0, "sizeInBytes":84534197, "size":24092, "age":"2020-05-15T06:21:01.420Z", "source":"merge", "version":"6.6.6" }, "_h34":{ "name":"_h34", "delCount":0, "sizeInBytes":43293118, "size":12481, "age":"2020-05-15T06:20:35.132Z", "source":"merge", "version":"6.6.6" }, "_h52":{ "name":"_h52", "delCount":0, "sizeInBytes":59819241, "size":17372, "age":"2020-05-15T06:22:06.211Z", "source":"merge", "version":"6.6.6" }, "_h5v":{ "name":"_h5v", "delCount":0, "sizeInBytes":72945656, "size":21005, "age":"2020-05-15T06:22:53.226Z", "source":"merge", "version":"6.6.6" }, "_h7j":{ "name":"_h7j", "delCount":0, "sizeInBytes":71065724, "size":20723, "age":"2020-05-15T06:24:14.704Z", "source":"merge", "version":"6.6.6" }, "_h6q":{ "name":"_h6q", "delCount":0, "sizeInBytes":2253642, "size":654, "age":"2020-05-15T06:23:39.117Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h6s":{ "name":"_h6s", "delCount":0, "sizeInBytes":4712345, "size":1316, "age":"2020-05-15T06:23:48.008Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h78":{ "name":"_h78", "delCount":0, "sizeInBytes":3324666, "size":910, "age":"2020-05-15T06:24:07.306Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h7s":{ "name":"_h7s", "delCount":0, "sizeInBytes":4712917, "size":1312, "age":"2020-05-15T06:24:33.932Z", "source":"merge", "version":"6.6.6", "mergeCandidate":true }, "_h7i":{ "name":"_h7i", "delCount":0, "sizeInBytes":4906686, "size":1389, "age":"2020-05-15T06:24:22.213Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h7k":{ "name":"_h7k", "delCount":0, "sizeInBytes":3548433, "size":1027, "age":"2020-05-15T06:24:21.420Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h7h":{ "name":"_h7h", "delCount":0, "sizeInBytes":4420269, "size":1299, "age":"2020-05-15T06:24:22.029Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h7q":{ "name":"_h7q", "delCount":0, "sizeInBytes":8349820, "size":2400, "age":"2020-05-15T06:24:33.616Z", "source":"flush", "version":"6.6.6" }, "_h7p":{ "name":"_h7p", "delCount":0, "sizeInBytes":8634659, "size":2468, "age":"2020-05-15T06:24:33.823Z", "source":"flush", "version":"6.6.6" }, "_h7r":{ "name":"_h7r", "delCount":0, "sizeInBytes":390310, "size":109, "age":"2020-05-15T06:24:34.421Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true }, "_h7t":{ "name":"_h7t", "delCount":0, "sizeInBytes":4729144, "size":1395, "age":"2020-05-15T06:24:37.886Z", "source":"flush", "version":"6.6.6", "mergeCandidate":true } } }The text was updated successfully, but these errors were encountered: