Add incoming/indexer/storage information to V1 API's status endpoint

Author: leo-naeka

gnocchi/incoming/__init__.py

@@ -257,6 +257,10 @@ def finish_sack_processing(sack):
         """Mark sack processing has finished."""
         pass
 
+    @staticmethod
+    def get_health_status():
+        raise exceptions.NotImplementedError
+
 
 @utils.retry_on_exception_and_log("Unable to initialize incoming driver")
 def get_driver(conf):

gnocchi/incoming/ceph.py

@@ -23,6 +23,7 @@
 
 from gnocchi.common import ceph
 from gnocchi import incoming
+from gnocchi.status import get_ceph_health_status
 
 rados = ceph.rados
 
@@ -232,3 +233,6 @@ def process_measures_for_sack(self, sack):
                 self.ioctx.remove_omap_keys(op, tuple(processed_keys))
                 self.ioctx.operate_write_op(op, str(sack),
                                             flags=self.OMAP_WRITE_FLAGS)
+
+    def get_health_status(self):
+        return get_ceph_health_status(self)

gnocchi/incoming/file.py

@@ -25,6 +25,7 @@
 import six
 
 from gnocchi import incoming
+from gnocchi.status import get_file_health_status
 from gnocchi import utils
 
 LOG = daiquiri.getLogger(__name__)
@@ -206,3 +207,6 @@ def process_measures_for_sack(self, sack):
 
         for metric_id, files in six.iteritems(processed_files):
             self._delete_measures_files_for_metric(metric_id, files)
+
+    def get_health_status(self):
+        return get_file_health_status(self)

gnocchi/incoming/redis.py

@@ -21,6 +21,7 @@
 
 from gnocchi.common import redis
 from gnocchi import incoming
+from gnocchi.status import get_redis_health_status
 
 
 LOG = daiquiri.getLogger(__name__)
@@ -193,3 +194,6 @@ def finish_sack_processing(self, sack):
         # Delete the sack key which handles no data but is used to get a SET
         # notification in iter_on_sacks_to_process
         self._client.delete(str(sack))
+
+    def get_health_status(self):
+        return get_redis_health_status(self)

gnocchi/incoming/s3.py

@@ -24,6 +24,7 @@
 
 from gnocchi.common import s3
 from gnocchi import incoming
+from gnocchi.status import get_s3_health_status
 
 boto3 = s3.boto3
 botocore = s3.botocore
@@ -42,21 +43,21 @@ def __init__(self, conf, greedy=True):
             s3.get_connection(conf)
         )
 
-        self._bucket_name_measures = (
+        self._bucket_name = (
             self._bucket_prefix + "-" + self.MEASURE_PREFIX
         )
 
     def __str__(self):
-        return "%s: %s" % (self.__class__.__name__, self._bucket_name_measures)
+        return "%s: %s" % (self.__class__.__name__, self._bucket_name)
 
     def _get_storage_sacks(self):
-        response = self.s3.get_object(Bucket=self._bucket_name_measures,
+        response = self.s3.get_object(Bucket=self._bucket_name,
                                       Key=self.CFG_PREFIX)
         return json.loads(response['Body'].read().decode())[self.CFG_SACKS]
 
     def set_storage_settings(self, num_sacks):
         data = {self.CFG_SACKS: num_sacks}
-        self.s3.put_object(Bucket=self._bucket_name_measures,
+        self.s3.put_object(Bucket=self._bucket_name,
                            Key=self.CFG_PREFIX,
                            Body=json.dumps(data).encode())
 
@@ -67,7 +68,7 @@ def remove_sacks(num_sacks):
 
     def upgrade(self, num_sacks):
         try:
-            s3.create_bucket(self.s3, self._bucket_name_measures,
+            s3.create_bucket(self.s3, self._bucket_name,
                              self._region_name)
         except botocore.exceptions.ClientError as e:
             if e.response['Error'].get('Code') not in (
@@ -80,7 +81,7 @@ def upgrade(self, num_sacks):
     def _store_new_measures(self, metric_id, data):
         now = datetime.datetime.utcnow().strftime("_%Y%m%d_%H:%M:%S")
         self.s3.put_object(
-            Bucket=self._bucket_name_measures,
+            Bucket=self._bucket_name,
             Key="/".join((str(self.sack_for_metric(metric_id)),
                           str(metric_id),
                           str(uuid.uuid4()) + now)),
@@ -97,7 +98,7 @@ def _build_report(self, details):
             else:
                 kwargs = {}
             response = self.s3.list_objects_v2(
-                Bucket=self._bucket_name_measures,
+                Bucket=self._bucket_name,
                 **kwargs)
             for c in response.get('Contents', ()):
                 if c['Key'] != self.CFG_PREFIX:
@@ -118,7 +119,7 @@ def _list_files(self, path_items, **kwargs):
                 except KeyError:
                     pass
             response = self.s3.list_objects_v2(
-                Bucket=self._bucket_name_measures,
+                Bucket=self._bucket_name,
                 Prefix="/".join(path_items) + "/",
                 **kwargs)
             yield response
@@ -136,7 +137,7 @@ def _list_measure_files_for_metric(self, sack, metric_id):
     def delete_unprocessed_measures_for_metric(self, metric_id):
         sack = self.sack_for_metric(metric_id)
         files = self._list_measure_files_for_metric(sack, metric_id)
-        s3.bulk_delete(self.s3, self._bucket_name_measures, files)
+        s3.bulk_delete(self.s3, self._bucket_name, files)
 
     def has_unprocessed(self, metric_id):
         sack = self.sack_for_metric(metric_id)
@@ -152,7 +153,7 @@ def process_measure_for_metrics(self, metric_ids):
             all_files.extend(files)
             for f in files:
                 response = self.s3.get_object(
-                    Bucket=self._bucket_name_measures,
+                    Bucket=self._bucket_name,
                     Key=f)
                 measures[metric_id] = numpy.concatenate((
                     measures[metric_id],
@@ -162,7 +163,7 @@ def process_measure_for_metrics(self, metric_ids):
         yield measures
 
         # Now clean objects
-        s3.bulk_delete(self.s3, self._bucket_name_measures, all_files)
+        s3.bulk_delete(self.s3, self._bucket_name, all_files)
 
     @contextlib.contextmanager
     def process_measures_for_sack(self, sack):
@@ -177,7 +178,7 @@ def process_measures_for_sack(self, sack):
                 continue
 
             response = self.s3.get_object(
-                Bucket=self._bucket_name_measures,
+                Bucket=self._bucket_name,
                 Key=f)
             measures[metric_id] = numpy.concatenate((
                 measures[metric_id],
@@ -187,4 +188,7 @@ def process_measures_for_sack(self, sack):
         yield measures
 
         # Now clean objects
-        s3.bulk_delete(self.s3, self._bucket_name_measures, files)
+        s3.bulk_delete(self.s3, self._bucket_name, files)
+
+    def get_health_status(self):
+        return get_s3_health_status(self)

gnocchi/incoming/swift.py

@@ -22,6 +22,7 @@
 
 from gnocchi.common import swift
 from gnocchi import incoming
+from gnocchi.status import get_swift_health_status
 from gnocchi import utils
 
 swclient = swift.swclient
@@ -144,3 +145,6 @@ def process_measures_for_sack(self, sack):
         yield measures
 
         swift.bulk_delete(self.swift, sack_name, files)
+
+    def get_health_status(self):
+        return get_swift_health_status(self)

gnocchi/indexer/__init__.py

@@ -477,3 +477,7 @@ def get_resource_attributes_schemas():
     @staticmethod
     def get_resource_type_schema():
         raise exceptions.NotImplementedError
+
+    @staticmethod
+    def get_health_status():
+        raise exceptions.NotImplementedError

gnocchi/indexer/sqlalchemy.py

@@ -50,6 +50,7 @@
 from gnocchi.indexer import sqlalchemy_base as base
 from gnocchi.indexer import sqlalchemy_types as types
 from gnocchi import resource_type
+from gnocchi.status import get_sqlalchemy_health_status
 from gnocchi import utils
 
 Base = base.Base
@@ -1176,6 +1177,9 @@ def _build_sort_keys(sorts, unique_keys):
 
         return sort_keys, sort_dirs
 
+    def get_health_status(self):
+        return get_sqlalchemy_health_status(self)
+
 
 def _operator_in(field_name, value):
     # Do not generate empty IN comparison

gnocchi/rest/api.py

@@ -2092,42 +2092,68 @@ class StatusController(rest.RestController):
     @pecan.expose('json')
     def get(details=True):
         enforce("get status", {})
-        try:
-            members_req = pecan.request.coordinator.get_members(
-                metricd.MetricProcessor.GROUP_ID)
-        except tooz.NotImplemented:
-            members_req = None
+
+        # Add status for incoming/indexer/storage
+        response_data = {
+            'incoming': {
+                'status': pecan.request.incoming.get_health_status(),
+            },
+            'indexer': {
+                'status': pecan.request.indexer.get_health_status(),
+            },
+            'storage': {
+                'status': pecan.request.storage.get_health_status(),
+            }
+        }
+
+        # Always return the detail, but set status code to 503
+        # if a component is not available
+        pecan.response.status = 200 if all([
+            component['status']['is_available']
+            for component in response_data.values()]) else 503
+
+        # Add storage measures to process
         try:
             report = pecan.request.incoming.measures_report(
-                strtobool("details", details))
+                strtobool('details', details))
         except incoming.ReportGenerationError:
-            abort(503, 'Unable to generate status. Please retry.')
-        report_dict = {"storage": {"summary": report['summary']}}
+            abort(503, "Unable to generate status. Please retry.")
+        response_data['storage']['summary'] = report['summary']
         if 'details' in report:
-            report_dict["storage"]["measures_to_process"] = report['details']
-        report_dict['metricd'] = {}
-        if members_req:
+            response_data['storage']['measures_to_process'] = report['details']
+
+        # Add metricd status
+        try:
+            members_req = pecan.request.coordinator.get_members(
+                metricd.MetricProcessor.GROUP_ID)
+        except tooz.NotImplemented:
+            response_data['metricd'] = {
+                'processors': None,
+                'statistics': {}
+            }
+        else:
             members = members_req.get()
             caps = [
                 pecan.request.coordinator.get_member_capabilities(
                     metricd.MetricProcessor.GROUP_ID, member)
                 for member in members
             ]
-            report_dict['metricd']['processors'] = [
-                member.decode() for member in members
-            ]
             members_data = {}
             for member, cap in six.moves.zip(members, caps):
                 caps_data = {
                     six.ensure_str(k): v
                     for k, v in six.iteritems(cap.get())
                 }
                 members_data[member.decode()] = caps_data
-            report_dict['metricd']['statistics'] = members_data
-        else:
-            report_dict['metricd']['processors'] = None
-            report_dict['metricd']['statistics'] = {}
-        return report_dict
+
+            response_data['metricd'] = {
+                'processors': [
+                    member.decode() for member in members
+                ],
+                'statistics': members_data
+            }
+
+        return response_data
 
 
 class MetricsBatchController(object):