Recording components logs and generate component yaml for smoke tests framework (#5333)

Signed-off-by: Zelin Hao <zelinhao@amazon.com>

Author: zelinh

src/test_workflow/smoke_test/smoke_test_runner.py

@@ -75,7 +75,7 @@ def run(self) -> Any:
             test_cluster.__start_cluster__(os.path.join(work_dir.path))
             is_cluster_ready = False
             for i in range(10):
-                logging.info(f"Attempt {i} of 10 to check cluster.")
+                logging.info(f"Attempt {i + 1} of 10 to check cluster.")
                 if test_cluster.__check_cluster_ready__():
                     is_cluster_ready = True
                     break

src/test_workflow/smoke_test/smoke_test_runner_opensearch.py

@@ -5,10 +5,12 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import io
 import logging
 import os
+from logging import Handler
 from pathlib import Path
-from typing import Any
+from typing import Any, Tuple
 
 import requests
 from openapi_core import Spec, validate_request, validate_response
@@ -17,6 +19,7 @@
 from manifests.test_manifest import TestManifest
 from test_workflow.smoke_test.smoke_test_runner import SmokeTestRunner
 from test_workflow.test_args import TestArgs
+from test_workflow.test_recorder.test_result_data import TestResultData
 from test_workflow.test_result.test_component_results import TestComponentResults
 from test_workflow.test_result.test_result import TestResult
 from test_workflow.test_result.test_suite_results import TestSuiteResults
@@ -64,6 +67,31 @@ def validate_response_swagger(self, response: Any) -> None:
         validate_response(response=response, spec=self.spec_, request=request)
         logging.info("Response is validated.")
 
+    def record_test_result(self, component: str, test_api: str, api_action: str, stdout: str, stderr: str) -> None:
+        test_config = f"{api_action}_{test_api.replace('/', '_')}"
+        logging.info(f"Recording test result for {component} component, config {test_config}")
+        file_path = self.test_recorder._create_base_folder_structure(component, test_config)
+        self.test_recorder._generate_std_files(stdout, stderr, file_path)
+
+    def setup_logging_buffers(self) -> Tuple[io.StringIO, io.StringIO, logging.StreamHandler, logging.StreamHandler]:
+        info_buffer = io.StringIO()
+        error_buffer = io.StringIO()
+
+        info_handler = logging.StreamHandler(info_buffer)
+        error_handler = logging.StreamHandler(error_buffer)
+
+        info_handler.setLevel(logging.INFO)
+        error_handler.setLevel(logging.ERROR)
+
+        logging.getLogger().addHandler(info_handler)
+        logging.getLogger().addHandler(error_handler)
+
+        return info_buffer, error_buffer, info_handler, error_handler
+
+    def cleanup_logging_handlers(self, info_handler: Handler, error_handler: Handler) -> None:
+        logging.getLogger().removeHandler(info_handler)
+        logging.getLogger().removeHandler(error_handler)
+
     def start_test(self, work_dir: Path) -> TestSuiteResults:
         url = "https://localhost:9200"
 
@@ -76,9 +104,11 @@ def start_test(self, work_dir: Path) -> TestSuiteResults:
                 test_results = TestComponentResults()
                 for api_requests, api_details in component_spec.items():
                     request_url = ''.join([url, api_requests])
-                    logging.info(f"Validating api request {api_requests}")
-                    logging.info(f"API request URL is {request_url}")
                     for method in api_details.keys():  # Iterates over each method, e.g., "GET", "POST"
+                        info_buffer, error_buffer, info_handler, error_handler = self.setup_logging_buffers()
+
+                        logging.info(f"Validating api request {api_requests}")
+                        logging.info(f"API request URL is {request_url}")
                         requests_method = getattr(requests, method.lower())
                         parameters_data = self.convert_parameter_json(api_details.get(method).get("parameters"))
                         header = api_details.get(method).get("header", self.mimetype)
@@ -87,15 +117,27 @@ def start_test(self, work_dir: Path) -> TestSuiteResults:
                         status = 0
                         try:
                             response = requests_method(request_url, verify=False, auth=("admin", "myStrongPassword123!"), headers=header, data=parameters_data)
-                            logging.info(f"Response is {response.text}")
+                            logging.info(f"Response is \n{response.text}")
                             self.validate_response_swagger(response)
                         except Exception as e:
                             status = 1
                             logging.error(f"Unexpected Error type is {type(e)}")
                             logging.error(e)
                             logging.info("Response is not validated. Please check the response output text above.")
                         finally:
+                            self.record_test_result(component.name, api_requests, method, info_buffer.getvalue(), error_buffer.getvalue())
                             test_result = TestResult(component.name, ' '.join([api_requests, method]), status)  # type: ignore
+                            test_result_data_local = TestResultData(
+                                component.name,
+                                f"{method}_{api_requests.replace('/', '_')}",
+                                status,
+                                info_buffer.getvalue(),
+                                error_buffer.getvalue(),
+                                {}
+                            )
+                            self.test_recorder.test_results_logs.generate_component_yml(test_result_data_local)
+                            self.cleanup_logging_handlers(info_handler, error_handler)
                             test_results.append(test_result)
+
                 all_results.append(component.name, test_results)
         return all_results

tests/tests_test_workflow/test_smoke_workflow/smoke_test/test_smoke_test_runner_opensearch.py

@@ -4,6 +4,7 @@
 # The OpenSearch Contributors require contributions made to
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
+import logging
 import os.path
 import unittest
 from pathlib import Path
@@ -149,3 +150,55 @@ def test_download_spec_https_fail(self, mock_file: Mock, mock_get: Mock, mock_te
         mock_file().write.assert_not_called()
 
         self.assertTrue(runner.spec_path.endswith(os.path.join("smoke_tests_spec", "opensearch-openapi-local.yaml")))
+
+    @patch("requests.get")
+    @patch("test_workflow.smoke_test.smoke_test_runner.TestRecorder")
+    def test_record_test_result(self, mock_recorder: Mock, mock_requests: Mock) -> None:
+        runner = SmokeTestRunnerOpenSearch(MagicMock(), MagicMock())
+
+        runner.test_recorder = mock_recorder
+
+        component = "test_component"
+        test_api = "/api/test"
+        api_action = "GET"
+        stdout = "test output"
+        stderr = "test error"
+
+        runner.record_test_result(component, test_api, api_action, stdout, stderr)
+
+        test_config = f"{api_action}_{test_api.replace('/', '_')}"
+        mock_recorder._create_base_folder_structure.assert_called_once_with(component, test_config)
+        mock_recorder._generate_std_files.assert_called_once()
+
+    @patch("requests.get")
+    @patch("test_workflow.smoke_test.smoke_test_runner.TestRecorder")
+    def test_setup_logging_buffers(self, mock_recorder: Mock, mock_requests: Mock) -> None:
+        runner = SmokeTestRunnerOpenSearch(MagicMock(), MagicMock())
+
+        info_buffer, error_buffer, info_handler, error_handler = runner.setup_logging_buffers()
+
+        logger = logging.getLogger()
+        logger.setLevel(logging.INFO)
+        assert info_handler in logger.handlers
+        assert error_handler in logger.handlers
+
+        logging.info("Test info message")
+        logging.error("Test error message")
+
+        assert "Test info message" in info_buffer.getvalue()
+        assert "Test error message" in error_buffer.getvalue()
+
+    @patch("requests.get")
+    @patch("test_workflow.smoke_test.smoke_test_runner.TestRecorder")
+    def test_cleanup_logging_handlers(self, mock_recorder: Mock, mock_requests: Mock) -> None:
+        runner = SmokeTestRunnerOpenSearch(MagicMock(), MagicMock())
+        info_buffer, error_buffer, info_handler, error_handler = runner.setup_logging_buffers()
+
+        logger = logging.getLogger()
+        assert info_handler in logger.handlers
+        assert error_handler in logger.handlers
+
+        runner.cleanup_logging_handlers(info_handler, error_handler)
+
+        assert info_handler not in logger.handlers
+        assert error_handler not in logger.handlers